Skip to main content

datumesh_http/
mesh.rs

1//! [`Mesh`]: the engine, its store and everything the endpoints share; the blocking
2//! operations each endpoint is built on, usable as library calls.
3
4use std::collections::{BTreeMap, HashMap, HashSet};
5use std::path::PathBuf;
6use std::sync::{Arc, Mutex, RwLock};
7use std::time::{Duration, Instant};
8
9use datumesh_core::json::{self, Value};
10use datumesh_core::{keys_from_response, verify, ErrorCode, FunctionDescriptor, KeyDoc, Keys, Receipt, SigningKey};
11use datumesh_engine::{Answer, Clock, Engine, EngineError, Runtime, SystemClock};
12use datumesh_formula::runtime::FormulaRuntime;
13use datumesh_store::Store;
14
15use crate::apikeys::Principal;
16use crate::call::{self, Bindings, Bound, Rejection, Target};
17use crate::config::{Config, ConfigError, Rule};
18use crate::envelope::{self, Cache, Reply};
19use crate::foreign::{host_of, CrossMesh, Foreign, ForeignError, KeyCache};
20use crate::jobs::Jobs;
21use crate::limit::Limiter;
22use crate::metrics::Metrics;
23use crate::pool::Pool;
24
25/// Why a mesh could not be built.
26#[derive(Debug)]
27#[non_exhaustive]
28pub enum BuildError {
29    /// The configuration.
30    Config(ConfigError),
31    /// The engine's configuration (the key, its publication in the store, the runtimes).
32    Engine(datumesh_engine::ConfigError),
33    /// The base URL ([`Config::base_url`]). Since 0.2.0.
34    BaseUrl(crate::config::BaseUrlError),
35}
36
37impl std::fmt::Display for BuildError {
38    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
39        match self {
40            BuildError::Config(e) => write!(f, "{e}"),
41            BuildError::Engine(e) => write!(f, "engine: {e}"),
42            BuildError::BaseUrl(e) => write!(f, "configuration: base URL: {e}"),
43        }
44    }
45}
46
47impl std::error::Error for BuildError {}
48
49/// Builds a [`Mesh`].
50pub struct MeshBuilder {
51    config: Config,
52    store: Arc<dyn Store + Send + Sync>,
53    key: Option<SigningKey>,
54    key_file: Option<PathBuf>,
55    runtimes: Vec<Arc<dyn Runtime>>,
56    formula: bool,
57    clock: Arc<dyn Clock>,
58    fuel: Option<u64>,
59    licence: Option<String>,
60    memo_capacity: Option<usize>,
61}
62
63impl std::fmt::Debug for MeshBuilder {
64    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
65        f.debug_struct("MeshBuilder").field("host", &self.config.host).finish_non_exhaustive()
66    }
67}
68
69impl MeshBuilder {
70    /// The mesh's signing key.
71    pub fn signing_key(mut self, key: SigningKey) -> Self {
72        self.key = Some(key);
73        self.key_file = None;
74        self
75    }
76
77    /// The mesh's signing key, from a file of its 32-byte seed (32 bytes or 64 hex digits),
78    /// which must have mode 0600.
79    pub fn signing_key_file(mut self, path: impl Into<PathBuf>) -> Self {
80        self.key_file = Some(path.into());
81        self.key = None;
82        self
83    }
84
85    /// Adds a runtime (the formula runtime is added unless [`MeshBuilder::without_formula`]).
86    pub fn runtime(mut self, runtime: Arc<dyn Runtime>) -> Self {
87        self.runtimes.push(runtime);
88        self
89    }
90
91    /// No formula runtime.
92    pub fn without_formula(mut self) -> Self {
93        self.formula = false;
94        self
95    }
96
97    /// The clock (default: the system clock).
98    pub fn clock(mut self, clock: Arc<dyn Clock>) -> Self {
99        self.clock = clock;
100        self
101    }
102
103    /// The engine's fuel per request.
104    pub fn fuel(mut self, fuel: u64) -> Self {
105        self.fuel = Some(fuel);
106        self
107    }
108
109    /// The licence of the receipts the engine signs.
110    pub fn licence(mut self, licence: impl Into<String>) -> Self {
111        self.licence = Some(licence.into());
112        self
113    }
114
115    /// The engine's memo capacity.
116    pub fn memo_capacity(mut self, n: usize) -> Self {
117        self.memo_capacity = Some(n);
118        self
119    }
120
121    /// Checks the configuration, builds the engine (its key must be published in the store's
122    /// namespace record for the host) and the mesh.
123    pub fn build(self) -> Result<Mesh, BuildError> {
124        if let Some(u) = &self.config.base_url {
125            crate::config::check_base_url(&self.config.host, u).map_err(BuildError::BaseUrl)?;
126        }
127        self.config.check().map_err(BuildError::Config)?;
128        let own_key_id = match (&self.key, &self.key_file) {
129            (Some(k), _) => Some(k.key_id().to_string()),
130            (None, Some(p)) => std::fs::read(p)
131                .ok()
132                .and_then(|b| crate::keynames::seed_of_file(&b))
133                .map(|s| datumesh_core::SigningKey::from_seed(&s).key_id().to_string()),
134            (None, None) => None,
135        };
136        let keys = Arc::new(KeyCache::default());
137        let foreign = Arc::new(Foreign::new(&self.config, self.store.clone(), keys.clone(), self.clock.clone()));
138        let mut b = Engine::builder(self.store.clone(), self.config.host.clone()).clock(self.clock.clone());
139        b = match (self.key, &self.key_file) {
140            (Some(k), _) => b.signing_key(k),
141            (None, Some(p)) => {
142                crate::config::read_secret_file(p).map_err(BuildError::Config)?;
143                b.signing_key_file(p.clone())
144            }
145            (None, None) => b,
146        };
147        let mut runtimes = self.runtimes;
148        if self.formula {
149            runtimes.push(Arc::new(FormulaRuntime::new(self.store.clone())));
150        }
151        if self.config.test_cells {
152            crate::testrun::install_test_cases(&*self.store, &self.config.host)
153                .map_err(|e| BuildError::Config(ConfigError::new("DATUMESH_TEST_CELLS", e)))?;
154            runtimes.push(Arc::new(crate::testrun::TestCases { store: self.store.clone() }));
155        }
156        for rt in runtimes {
157            b = b.runtime(Arc::new(CrossMesh::new(rt, foreign.clone())));
158        }
159        if let Some(f) = self.fuel {
160            b = b.fuel(f);
161        }
162        if let Some(l) = self.licence {
163            b = b.licence(l);
164        }
165        if let Some(n) = self.memo_capacity {
166            b = b.memo_capacity(n);
167        }
168        let engine = b.build().map_err(BuildError::Engine)?;
169        let config = self.config;
170        let embed = config.embed_url.as_deref().map(crate::embed::EmbedClient::new);
171        let inner = Inner {
172            generic: RwLock::new(Arc::new(crate::find::stopwords())),
173            embed,
174            reindex: Mutex::new(()),
175            publish_limits: Mutex::new(crate::bundles::Limits::default()),
176            relays: Mutex::new(HashMap::new()),
177            takeover: Mutex::new(None),
178            find: Mutex::new(None),
179            value_counts_at: Mutex::new(None),
180            unnamed: Mutex::new(HashSet::new()),
181            series: Mutex::new(crate::series::Memo::default()),
182            challenges: Mutex::new(crate::keynames::Challenges::default()),
183            keys_lock: Mutex::new(()),
184            own_key_id,
185            pool: Arc::new(Pool::new(config.pool_size)),
186            jobs: Jobs::new(config.ttl.job, config.max_jobs),
187            limiter: Limiter::new(config.rate.clone()),
188            metrics: Metrics::default(),
189            engine: Arc::new(engine),
190            store: self.store,
191            clock: self.clock,
192            keys,
193            foreign,
194            descriptors: Mutex::new(HashMap::new()),
195            last_good: Mutex::new(HashMap::new()),
196            infos: (0..32).map(|_| Mutex::new(HashMap::new())).collect(),
197            by_hash: (0..32).map(|_| Mutex::new(HashMap::new())).collect(),
198            latest: Latest::new(),
199            targets: Targets::new(),
200            slices: Mutex::new(Vec::new()),
201            known: Mutex::new(crate::describe::Index::default()),
202            started: Instant::now(),
203            config,
204        };
205        let mesh = Mesh { inner: Arc::new(inner) };
206        // The operator's catalogue, then the index: now, and every 30 s (a bundle or a publish
207        // from another process reaches the index within that). The thread stops with the mesh.
208        // On a thread of its own: the store's synchronous client must not run on a runtime's thread.
209        std::thread::scope(|s| {
210            let inner = mesh.inner.clone();
211            let _ = s.spawn(move || crate::find::sync_operator_catalog(&inner)).join();
212        });
213        // The object index (describe of an object id, CONTRACT-0.4.9 H3): on a thread of its own, after its first
214        // delay, then when it is due (dirty, or older than 30 s). Requests read the last build.
215        let weak = Arc::downgrade(&mesh.inner);
216        let delay = mesh.inner.config.index_first_build_delay;
217        if delay.is_zero() {
218            on_scoped_thread(&mesh.inner, Inner::refresh_index);
219        }
220        let spawned = std::thread::Builder::new().name("datumesh-object-index".into()).spawn(move || {
221            let started = Instant::now();
222            loop {
223                let Some(inner) = weak.upgrade() else { break };
224                if started.elapsed() >= delay {
225                    inner.refresh_index();
226                }
227                drop(inner);
228                std::thread::sleep(crate::describe::INDEX_TICK);
229            }
230        });
231        if let Err(e) = spawned {
232            tracing::warn!(error = %e, "the object index thread is not started; describe of an object id answers 503");
233        }
234        // The takeover lists (CONTRACT-0.4.9 H2): built after the series' first delay, then every series refresh
235        // interval (the series' knobs, as the takeover lists are read with the series overview).
236        let weak = Arc::downgrade(&mesh.inner);
237        let (delay, refresh) = (mesh.inner.config.series_first_build_delay, mesh.inner.config.series_refresh);
238        let mut last: Option<Instant> = None;
239        if delay.is_zero() {
240            on_scoped_thread(&mesh.inner, crate::takeover::refresh);
241            last = Some(Instant::now());
242        }
243        let spawned = std::thread::Builder::new().name("datumesh-takeover".into()).spawn(move || {
244            let started = Instant::now();
245            loop {
246                let Some(inner) = weak.upgrade() else { break };
247                if started.elapsed() >= delay && last.is_none_or(|t| t.elapsed() >= refresh) {
248                    crate::takeover::refresh(&inner);
249                    last = Some(Instant::now());
250                }
251                drop(inner);
252                std::thread::sleep(Duration::from_millis(500));
253            }
254        });
255        if let Err(e) = spawned {
256            tracing::warn!(error = %e, "the takeover thread is not started; the takeover pages answer 503");
257        }
258        // The find answers (CONTRACT-0.4.9 H4): the summary and the empty query's listing, built after the first
259        // delay, then every 300 s. Requests read the last build.
260        let weak = Arc::downgrade(&mesh.inner);
261        let delay = mesh.inner.config.find_first_build_delay;
262        let mut last: Option<Instant> = None;
263        if delay.is_zero() {
264            on_scoped_thread(&mesh.inner, crate::find::refresh);
265            last = Some(Instant::now());
266        }
267        let spawned = std::thread::Builder::new().name("datumesh-find".into()).spawn(move || {
268            let started = Instant::now();
269            loop {
270                let Some(inner) = weak.upgrade() else { break };
271                if started.elapsed() >= delay && last.is_none_or(|t| t.elapsed() >= FIND_REFRESH) {
272                    crate::find::refresh(&inner);
273                    last = Some(Instant::now());
274                }
275                drop(inner);
276                std::thread::sleep(Duration::from_millis(500));
277            }
278        });
279        if let Err(e) = spawned {
280            tracing::warn!(error = %e, "the find thread is not started; the summary and the empty query answer 503");
281        }
282        // The names of the stored tests written before 0.4.9 (CONTRACT-0.4.9 H1): once, in batches, on a thread of
283        // its own; the rows written from now on are named when they are written.
284        let weak = Arc::downgrade(&mesh.inner);
285        let spawned = std::thread::Builder::new().name("datumesh-test-names".into()).spawn(move || {
286            if let Some(inner) = weak.upgrade() {
287                crate::testrun::name_old_tests(&inner);
288            }
289        });
290        if let Err(e) = spawned {
291            tracing::warn!(error = %e, "the test naming thread is not started; old tests stay unnamed");
292        }
293        let weak = Arc::downgrade(&mesh.inner);
294        let spawned = std::thread::Builder::new().name("datumesh-index".into()).spawn(move || loop {
295            let Some(inner) = weak.upgrade() else { break };
296            crate::find::reindex(&inner);
297            drop(inner);
298            std::thread::sleep(Duration::from_secs(30));
299        });
300        if let Err(e) = spawned {
301            tracing::warn!(error = %e, "the index thread is not started; find sees the catalogue as it is at start");
302        }
303        // The series overview (G5): the first build at start (unless a delay is set, then after it), then every
304        // refresh interval, on a thread of its own (the store's client is synchronous: the first build runs on a
305        // scoped thread the caller joins, as the catalogue's does). Requests read the last build; the thread stops
306        // with the mesh.
307        if !mesh.inner.config.series.is_empty() {
308            let weak = Arc::downgrade(&mesh.inner);
309            let (delay, refresh) = (mesh.inner.config.series_first_build_delay, mesh.inner.config.series_refresh);
310            if delay.is_zero() {
311                std::thread::scope(|s| {
312                    let inner = mesh.inner.clone();
313                    let _ = s.spawn(move || crate::series::rebuild(&inner, "background")).join();
314                });
315            }
316            let mut wait = if delay.is_zero() { refresh } else { delay };
317            let spawned = std::thread::Builder::new().name("datumesh-series".into()).spawn(move || loop {
318                std::thread::sleep(wait);
319                let Some(inner) = weak.upgrade() else { break };
320                crate::series::rebuild(&inner, "background");
321                drop(inner);
322                wait = refresh;
323            });
324            if let Err(e) = spawned {
325                tracing::warn!(error = %e, "the series thread is not started; the series overview is never built");
326            }
327        }
328        Ok(mesh)
329    }
330}
331
332/// Runs `f` on a thread of its own and waits for it: the first build of a background answer at start, when no
333/// delay is set (the store's client is synchronous, so it does not run on the caller's thread).
334fn on_scoped_thread(inner: &Arc<Inner>, f: fn(&Inner)) {
335    std::thread::scope(|s| {
336        let inner = inner.clone();
337        let _ = s.spawn(move || f(&inner)).join();
338    });
339}
340
341/// How often the find thread builds its answers again (CONTRACT-0.4.9 H4).
342const FIND_REFRESH: Duration = Duration::from_secs(300);
343
344/// The last `ok` receipt served for a cell, for `stale` answers.
345struct LastGood {
346    info: Arc<ReceiptInfo>,
347    at: Instant,
348}
349
350/// What the mesh needs of a receipt it serves, kept by its bytes (content-addressed: never
351/// stale): its hash, `asof`, trust and its `ok` envelope.
352pub(crate) struct ReceiptInfo {
353    pub(crate) hash: String,
354    pub(crate) asof: String,
355    pub(crate) receipt: Vec<u8>,
356    pub(crate) parsed: Receipt,
357    pub(crate) trust: datumesh_core::TrustLabel,
358    /// The `ok` envelope, the function hash echoed.
359    pub(crate) ok: crate::envelope::Bytes,
360}
361
362pub(crate) struct Inner {
363    pub(crate) config: Config,
364    pub(crate) store: Arc<dyn Store + Send + Sync>,
365    pub(crate) engine: Arc<Engine>,
366    pub(crate) clock: Arc<dyn Clock>,
367    pub(crate) pool: Arc<Pool>,
368    pub(crate) jobs: Jobs,
369    pub(crate) limiter: Limiter,
370    pub(crate) metrics: Metrics,
371    pub(crate) keys: Arc<KeyCache>,
372    pub(crate) foreign: Arc<Foreign>,
373    descriptors: Mutex<HashMap<String, Arc<FunctionDescriptor>>>,
374    last_good: Mutex<HashMap<String, LastGood>>,
375    infos: Vec<Mutex<HashMap<u64, Arc<ReceiptInfo>>>>,
376    by_hash: Vec<Mutex<HashMap<String, Arc<ReceiptInfo>>>>,
377    latest: Latest,
378    targets: Targets,
379    #[allow(clippy::type_complexity)]
380    pub(crate) slices: Mutex<Vec<(String, String, Arc<Vec<datumesh_store::SliceValue>>)>>,
381    /// The index of the objects the mesh knows (describe, short ids).
382    pub(crate) known: Mutex<crate::describe::Index>,
383    pub(crate) started: Instant,
384    /// The generic words `find` never matches values by (CONTRACT-build2-part2-read R3).
385    pub(crate) generic: RwLock<Arc<HashSet<String>>>,
386    /// The embedding service, when `DATUMESH_EMBED_URL` is set (R2).
387    pub(crate) embed: Option<crate::embed::EmbedClient>,
388    /// One index pass at a time.
389    pub(crate) reindex: Mutex<()>,
390    /// The attempts at `POST /bundles`, per host and for the mesh (Build 2 Part 4, P1).
391    pub(crate) publish_limits: Mutex<crate::bundles::Limits>,
392    /// Where a relayed function's source was read, by function name and hash (Build 2 Part 5, F5).
393    pub(crate) relays: Mutex<HashMap<String, (Instant, Option<crate::fnpage::Relay>)>>,
394    /// The last build of the takeover thread: the relayed functions of each source domain (CONTRACT-0.4.9 H2).
395    pub(crate) takeover: Mutex<Option<Arc<crate::takeover::Lists>>>,
396    /// The last build of the find thread: the catalogue summary and the empty query's listing (CONTRACT-0.4.9 H4).
397    pub(crate) find: Mutex<Option<Arc<crate::find::Built>>>,
398    /// When the value counts of the catalogue were last refreshed (H4: once per `DATUMESH_VALUE_COUNTS_SECS`).
399    pub(crate) value_counts_at: Mutex<Option<Instant>>,
400    /// The hashes whose stored tests are not all named yet (the naming thread, CONTRACT-0.4.9 H1): their pages read
401    /// their tests by call, not by name, until it is done.
402    pub(crate) unnamed: Mutex<HashSet<String>>,
403    /// The overviews and dictionary orders of the series pages (Build 2 Part 6, G5).
404    pub(crate) series: Mutex<crate::series::Memo>,
405    /// The challenges of `POST /x/keys` issued and not yet used (QUESTIONS-build2 D39).
406    pub(crate) challenges: Mutex<crate::keynames::Challenges>,
407    /// One registration at a time: the key document is read and written as a whole.
408    pub(crate) keys_lock: Mutex<()>,
409    /// The id of the mesh's own signing key, when the mesh knows it (a key of another id is a registered one).
410    pub(crate) own_key_id: Option<String>,
411}
412
413/// A Datumesh mesh: the engine behind the endpoints. Cheap to clone (shared).
414#[derive(Clone)]
415pub struct Mesh {
416    pub(crate) inner: Arc<Inner>,
417}
418
419impl std::fmt::Debug for Mesh {
420    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
421        f.debug_struct("Mesh").field("host", &self.inner.config.host).finish_non_exhaustive()
422    }
423}
424
425/// Bindings looked up once per name and asof within one batch.
426pub(crate) struct Memo<'a> {
427    inner: &'a Inner,
428    seen: std::borrow::Cow<'a, Seen>,
429}
430
431/// A binding a batch looks up: the lookup's result once one succeeded, and the lock held
432/// during a lookup, so the batch's chunks, which start together and ask for the same names,
433/// make one store round trip per name and asof, not one each.
434#[derive(Default)]
435struct SeenEntry {
436    found: std::sync::OnceLock<Option<Arc<Bound>>>,
437    lookup: Mutex<()>,
438}
439
440/// The bindings a batch has looked up (shared by its chunks), by name, then asof.
441#[derive(Default)]
442pub(crate) struct Seen(RwLock<HashMap<String, SeenAsofs>>);
443
444/// A name's entries in [`Seen`], by asof.
445type SeenAsofs = Vec<(String, Arc<SeenEntry>)>;
446
447impl Clone for Seen {
448    fn clone(&self) -> Seen {
449        let m = self.0.read().unwrap_or_else(|p| p.into_inner());
450        let copy = m
451            .iter()
452            .map(|(name, v)| {
453                let v = v
454                    .iter()
455                    .map(|(a, e)| (a.clone(), Arc::new(SeenEntry { found: e.found.clone(), lookup: Mutex::new(()) })))
456                    .collect();
457                (name.clone(), v)
458            })
459            .collect();
460        Seen(RwLock::new(copy))
461    }
462}
463
464/// What [`Seen`] has of a name as of an asof.
465enum Held {
466    Found(Option<Arc<Bound>>),
467    Entry(Arc<SeenEntry>),
468    Full,
469}
470
471impl Seen {
472    /// The binding of `fn_name` as of `asof` if found, else its entry (made if there is room:
473    /// 64 asofs per name).
474    fn held(&self, fn_name: &str, asof: &str) -> Held {
475        if let Some(v) = self.0.read().unwrap_or_else(|p| p.into_inner()).get(fn_name) {
476            if let Some((_, e)) = v.iter().find(|(a, _)| a == asof) {
477                return match e.found.get() {
478                    Some(b) => Held::Found(b.clone()),
479                    None => Held::Entry(e.clone()),
480                };
481            }
482        }
483        let mut m = self.0.write().unwrap_or_else(|p| p.into_inner());
484        let v = m.entry(fn_name.to_string()).or_default();
485        if let Some((_, e)) = v.iter().find(|(a, _)| a == asof) {
486            return Held::Entry(e.clone());
487        }
488        if v.len() >= 64 {
489            return Held::Full;
490        }
491        let e = Arc::new(SeenEntry::default());
492        v.push((asof.to_string(), e.clone()));
493        Held::Entry(e)
494    }
495
496    /// The binding of `fn_name` as of `asof`: as found before, else looked up by `look`, at
497    /// most one lookup at a time per name and asof (the others wait for its result). A failed
498    /// lookup is not kept: the next request looks the name up again.
499    pub(crate) fn lookup(
500        &self,
501        fn_name: &str,
502        asof: &str,
503        look: impl FnOnce() -> Result<Option<Arc<Bound>>, ErrorCode>,
504    ) -> Result<Option<Arc<Bound>>, ErrorCode> {
505        let e = match self.held(fn_name, asof) {
506            Held::Found(b) => return Ok(b),
507            Held::Full => return look(),
508            Held::Entry(e) => e,
509        };
510        let _looking = e.lookup.lock().unwrap_or_else(|p| p.into_inner());
511        if let Some(b) = e.found.get() {
512            return Ok(b.clone());
513        }
514        let b = look()?;
515        let _ = e.found.set(b.clone());
516        Ok(b)
517    }
518}
519
520impl<'a> Memo<'a> {
521    pub(crate) fn new(inner: &'a Inner) -> Memo<'a> {
522        Memo { inner, seen: std::borrow::Cow::Owned(Seen::default()) }
523    }
524
525    pub(crate) fn shared(inner: &'a Inner, seen: &'a Seen) -> Memo<'a> {
526        Memo { inner, seen: std::borrow::Cow::Borrowed(seen) }
527    }
528}
529
530impl Bindings for Memo<'_> {
531    fn pinned(&self, fn_hash: &str) -> Result<Option<Arc<Bound>>, ErrorCode> {
532        Bindings::pinned(self.inner, fn_hash)
533    }
534
535    fn alias(&self, alias: &str) -> Option<String> {
536        Bindings::alias(self.inner, alias)
537    }
538
539    fn class(&self, name: &str, asof: &str) -> Result<Option<Arc<call::BoundClass>>, ErrorCode> {
540        Bindings::class(self.inner, name, asof)
541    }
542
543    fn aliases_tx(&self) -> Option<String> {
544        Bindings::aliases_tx(self.inner)
545    }
546
547    fn lookup(&self, class: &str, label: &str, asof: &str) -> Result<Option<String>, ErrorCode> {
548        Bindings::lookup(self.inner, class, label, asof)
549    }
550
551    fn cached(&self, fn_hash: &str, rest: &str) -> Option<Arc<Target>> {
552        self.inner.targets.get(fn_hash, rest)
553    }
554
555    fn remember(&self, rest: &str, target: &Arc<Target>) {
556        self.inner.targets.put(rest, target);
557    }
558
559    fn binding(&self, fn_name: &str, asof: &str) -> Result<Option<Arc<Bound>>, ErrorCode> {
560        self.seen.lookup(fn_name, asof, || Bindings::binding(self.inner, fn_name, asof))
561    }
562}
563
564impl Bindings for Inner {
565    fn pinned(&self, fn_hash: &str) -> Result<Option<Arc<Bound>>, ErrorCode> {
566        match self.descriptor(fn_hash) {
567            Ok(d) => Ok(Some(Arc::new(Bound::new(fn_hash, "", d)))),
568            Err(ErrorCode::FnUnavailable) => Ok(None),
569            Err(c) => Err(c),
570        }
571    }
572
573    fn alias(&self, alias: &str) -> Option<String> {
574        self.config.aliases.get(alias).cloned()
575    }
576
577    fn class(&self, name: &str, asof: &str) -> Result<Option<Arc<call::BoundClass>>, ErrorCode> {
578        let c = self.class_binding(name, asof)?;
579        if c.is_some() {
580            self.hear_of_class(name);
581        }
582        Ok(c)
583    }
584
585    /// The mesh's configured alias map has no snapshot.
586    fn aliases_tx(&self) -> Option<String> {
587        None
588    }
589
590    fn lookup(&self, class: &str, label: &str, asof: &str) -> Result<Option<String>, ErrorCode> {
591        let Some(c) = Bindings::class(self, class, asof)? else { return Ok(None) };
592        let Some(method) = &c.descriptor.lookup else { return Ok(None) };
593        let name = format!("{class}/{method}");
594        let Some(b) = Bindings::binding(self, &name, asof)? else { return Ok(None) };
595        let (ns, fn_name) = name.rsplit_once('/').unwrap_or((&name, ""));
596        let mut args = datumesh_core::Args::new();
597        args.insert("label".into(), datumesh_core::Arg::Value(datumesh_core::Scalar::String(label.to_string())));
598        let cell = datumesh_core::Cell { ns: ns.into(), fn_name: fn_name.into(), fn_hash: b.fn_hash.clone(), args };
599        match self.engine.get_cell(&cell, Some(asof)) {
600            Ok(Answer::Receipt(bytes)) => match Receipt::from_json(&bytes).map(|r| r.value().clone()) {
601                Ok(datumesh_core::Scalar::String(k)) => Ok(Some(k)),
602                _ => Ok(None),
603            },
604            Err(EngineError::Store(_)) => Err(ErrorCode::InputUnavailable),
605            _ => Ok(None),
606        }
607    }
608
609    fn cached(&self, fn_hash: &str, rest: &str) -> Option<Arc<Target>> {
610        self.targets.get(fn_hash, rest)
611    }
612
613    fn remember(&self, rest: &str, target: &Arc<Target>) {
614        self.targets.put(rest, target);
615    }
616
617    fn binding(&self, fn_name: &str, asof: &str) -> Result<Option<Arc<Bound>>, ErrorCode> {
618        let mut failed = None;
619        if self.foreign.is_foreign(fn_name) {
620            if let Err(e) = self.foreign.ensure_name(fn_name) {
621                tracing::warn!(target: "datumesh_http", name = fn_name, detail = %e.detail, "cross-mesh binding");
622                // `Bindings::binding` answers a bare code: the detail of E_FOREIGN_UNAVAILABLE
623                // waits on this thread for `rejection`, which answers on the same thread.
624                FOREIGN_DETAIL.with(|d| d.set(e.foreign_detail()));
625                failed = Some(e.code);
626            }
627        }
628        let b = self.store.resolve_binding(fn_name, asof).map_err(|e| {
629            tracing::warn!(target: "datumesh_http", error = %e, "resolve_binding");
630            ErrorCode::InputUnavailable
631        })?;
632        // The other mesh is unreachable: a binding imported before still names the cell
633        // (whose answer is then stale or an error), else the name cannot be resolved.
634        if let (Some(code), None) = (failed, &b) {
635            return Err(code);
636        }
637        match b {
638            None => Ok(None),
639            Some(b) => {
640                let d = self.descriptor(&b.fn_hash)?;
641                Ok(Some(Arc::new(Bound::new(b.fn_hash, b.tx, d))))
642            }
643        }
644    }
645}
646
647thread_local! {
648    /// Why the other mesh could not be reached, from the last cross-mesh name resolution on
649    /// this thread (see `Inner::binding` and `Inner::rejection`).
650    static FOREIGN_DETAIL: std::cell::Cell<Option<datumesh_core::ForeignDetail>> = const { std::cell::Cell::new(None) };
651}
652
653/// Latest answers by key: the receipt, the cell id and the asof answered at.
654type LatestShard = RwLock<HashMap<String, (Arc<ReceiptInfo>, String, String)>>;
655
656/// The latest answers of local stored cells, as of when they were answered: such an answer
657/// stands for every later asof until a version of the cell is added, which the mesh is told
658/// of ([`Mesh::stored_version_added`]), as the engine's memo is. It saves the engine's work
659/// (the cell id, the memo) on the hottest path: warm gets of stored cells.
660struct Latest {
661    shards: Vec<LatestShard>,
662    by_cell: Mutex<HashMap<String, Vec<String>>>,
663}
664
665impl Latest {
666    fn new() -> Latest {
667        Latest { shards: (0..32).map(|_| RwLock::new(HashMap::new())).collect(), by_cell: Mutex::new(HashMap::new()) }
668    }
669
670    fn shard(&self, key: &str) -> &LatestShard {
671        use std::hash::{Hash, Hasher};
672        let mut h = std::collections::hash_map::DefaultHasher::new();
673        key.hash(&mut h);
674        &self.shards[(h.finish() % self.shards.len() as u64) as usize]
675    }
676
677    /// The answer for `key` as of `a`, if one answered as of `a` or earlier is held.
678    fn get(&self, key: &str, a: &str) -> Option<Arc<ReceiptInfo>> {
679        let m = self.shard(key).read().unwrap_or_else(|p| p.into_inner());
680        let (info, _, at) = m.get(key)?;
681        (a >= at.as_str()).then(|| info.clone())
682    }
683
684    fn put(&self, key: String, cell_id: String, info: Arc<ReceiptInfo>, at: String) {
685        {
686            let mut b = self.by_cell.lock().unwrap_or_else(|p| p.into_inner());
687            if b.len() > 200_000 {
688                b.clear();
689                for s in &self.shards {
690                    s.write().unwrap_or_else(|p| p.into_inner()).clear();
691                }
692            }
693            b.entry(cell_id.clone()).or_default().push(key.clone());
694        }
695        self.shard(&key).write().unwrap_or_else(|p| p.into_inner()).insert(key, (info, cell_id, at));
696    }
697
698    fn forget(&self, cell_id: &str) {
699        let keys = self.by_cell.lock().unwrap_or_else(|p| p.into_inner()).remove(cell_id).unwrap_or_default();
700        for k in keys {
701            self.shard(&k).write().unwrap_or_else(|p| p.into_inner()).remove(&k);
702        }
703    }
704
705    fn clear(&self) {
706        self.by_cell.lock().unwrap_or_else(|p| p.into_inner()).clear();
707        for s in &self.shards {
708            s.write().unwrap_or_else(|p| p.into_inner()).clear();
709        }
710    }
711}
712
713/// Cell URI rests already parsed, by function hash (parsing depends on the signature
714/// alone): resolving a hot cell costs a binding lookup and no parse.
715struct Targets {
716    #[allow(clippy::type_complexity)]
717    shards: Vec<RwLock<HashMap<String, HashMap<String, Arc<Target>>>>>,
718}
719
720impl Targets {
721    fn new() -> Targets {
722        Targets { shards: (0..32).map(|_| RwLock::new(HashMap::new())).collect() }
723    }
724
725    #[allow(clippy::type_complexity)]
726    fn shard(&self, rest: &str) -> &RwLock<HashMap<String, HashMap<String, Arc<Target>>>> {
727        use std::hash::{Hash, Hasher};
728        let mut h = std::collections::hash_map::DefaultHasher::new();
729        rest.hash(&mut h);
730        &self.shards[(h.finish() % self.shards.len() as u64) as usize]
731    }
732
733    fn get(&self, fn_hash: &str, rest: &str) -> Option<Arc<Target>> {
734        self.shard(rest).read().unwrap_or_else(|p| p.into_inner()).get(rest)?.get(fn_hash).cloned()
735    }
736
737    fn put(&self, rest: &str, t: &Arc<Target>) {
738        let mut m = self.shard(rest).write().unwrap_or_else(|p| p.into_inner());
739        if m.len() >= 4_096 {
740            m.clear();
741        }
742        let v = m.entry(rest.to_string()).or_default();
743        if v.len() >= 8 {
744            v.clear();
745        }
746        v.insert(t.bound.fn_hash.clone(), t.clone());
747    }
748}
749
750/// The key of a cell's last good answer: the function hash and the URI (without asof: the
751/// last good answer is kept for requests about now only).
752fn last_good_key(t: &Target) -> String {
753    t.cell_key().to_string()
754}
755
756/// The engine's error as an envelope code (the engine's table, "For the HTTP layer").
757pub(crate) fn engine_code(e: &EngineError) -> (bool, ErrorCode) {
758    match e {
759        EngineError::Invalid(e) if envelope::is_request_code(e.code()) => (true, e.code()),
760        EngineError::Invalid(e) if envelope::is_error_code(e.code()) => (false, e.code()),
761        EngineError::NameUnresolved { .. } => (false, ErrorCode::FnUnavailable),
762        EngineError::UndeclaredInput { .. } => (false, ErrorCode::InputUndeclared),
763        EngineError::Sign(e) if envelope::is_error_code(e.code()) => (false, e.code()),
764        EngineError::Store(_) => (false, ErrorCode::InputUnavailable),
765        _ => (false, ErrorCode::EvalUnavailable),
766    }
767}
768
769impl Inner {
770    pub(crate) fn now(&self) -> String {
771        self.clock.now()
772    }
773
774    /// Whether the stored tests of `fn_hash` are not all named yet (see [`Inner::unnamed`]).
775    pub(crate) fn is_unnamed(&self, fn_hash: &str) -> bool {
776        self.unnamed.lock().unwrap_or_else(|p| p.into_inner()).contains(fn_hash)
777    }
778
779    /// A descriptor by hash (cached: content-addressed).
780    pub(crate) fn descriptor(&self, fn_hash: &str) -> Result<Arc<FunctionDescriptor>, ErrorCode> {
781        if let Some(d) = self.descriptors.lock().unwrap_or_else(|p| p.into_inner()).get(fn_hash) {
782            return Ok(d.clone());
783        }
784        let bytes = self.store.descriptor(fn_hash).ok_or(ErrorCode::FnUnavailable)?;
785        let d = Arc::new(FunctionDescriptor::from_json(&bytes).map_err(|_| ErrorCode::FnUnavailable)?);
786        let mut m = self.descriptors.lock().unwrap_or_else(|p| p.into_inner());
787        if m.len() > 10_000 {
788            m.clear();
789        }
790        m.insert(fn_hash.to_string(), d.clone());
791        Ok(d)
792    }
793
794    /// The `ok` reply with a receipt, remembered for `stale` answers.
795    fn ok_reply(
796        &self,
797        target: &Target,
798        info: Arc<ReceiptInfo>,
799        a: &str,
800        now: &str,
801        private: bool,
802        link: bool,
803    ) -> Reply {
804        let mut reply = Reply::json(200, info.ok.clone(), Cache::Immutable);
805        if target.asof().is_none() {
806            let mut lg = self.last_good.lock().unwrap_or_else(|p| p.into_inner());
807            if lg.len() > 100_000 {
808                lg.clear();
809            }
810            lg.insert(last_good_key(target), LastGood { info, at: Instant::now() });
811        }
812        self.cache_rules(&mut reply, target, a, now, private, link);
813        reply
814    }
815
816    /// Remembers the answer as of now `a` for a local stored cell, when it is the cell's
817    /// latest version (a version with a later `tx` would answer later asofs).
818    fn remember_latest(&self, target: &Target, info: &Arc<ReceiptInfo>, a: &str) {
819        match self.store.latest(target.cell_id()) {
820            Ok(Some(v)) if v.receipt_hash == info.hash => {
821                self.latest.put(
822                    target.cell_key().to_string(),
823                    target.cell_id().to_string(),
824                    info.clone(),
825                    a.to_string(),
826                );
827            }
828            _ => {}
829        }
830    }
831
832    /// A stored version was added for `cell_id`: the engine's memo, the latest answers and
833    /// the slice cache forget what rests on it.
834    pub(crate) fn version_added(&self, cell_id: &str) {
835        self.engine.stored_version_added(cell_id);
836        self.latest.forget(cell_id);
837        self.slices.lock().unwrap_or_else(|p| p.into_inner()).clear();
838        self.known.lock().unwrap_or_else(|p| p.into_inner()).announce(cell_id);
839        self.index_stored_now(cell_id);
840    }
841
842    /// The receipt's hash, asof, trust and `ok` envelope (cached by its bytes).
843    fn receipt_info(&self, bytes: &[u8]) -> Option<Arc<ReceiptInfo>> {
844        use std::hash::{Hash, Hasher};
845        let mut h = std::collections::hash_map::DefaultHasher::new();
846        bytes.hash(&mut h);
847        let key = h.finish();
848        let shard = &self.infos[(key % self.infos.len() as u64) as usize];
849        if let Some(i) = shard.lock().unwrap_or_else(|p| p.into_inner()).get(&key) {
850            if i.receipt == bytes {
851                return Some(i.clone());
852            }
853        }
854        let r = Receipt::from_json(bytes).ok()?;
855        let trust = envelope::trust_label_of(&r, &self.config.host);
856        let ok = envelope::with_fn_hash(datumesh_core::Answer::ok(r.clone(), trust), Some(&r.cell().fn_hash)).to_json();
857        let info = Arc::new(ReceiptInfo {
858            hash: r.hash().to_string(),
859            asof: r.asof().to_string(),
860            receipt: bytes.to_vec(),
861            parsed: r,
862            trust,
863            ok: ok.into(),
864        });
865        let mut m = shard.lock().unwrap_or_else(|p| p.into_inner());
866        if m.len() >= 4_096 {
867            m.clear();
868        }
869        m.insert(key, info.clone());
870        Some(info)
871    }
872
873    /// A receipt's info by its hash: from the cache, else the store.
874    pub(crate) fn receipt_info_by_hash(&self, hash: &str) -> Option<Arc<ReceiptInfo>> {
875        use std::hash::{Hash, Hasher};
876        let mut h = std::collections::hash_map::DefaultHasher::new();
877        hash.hash(&mut h);
878        let shard = &self.by_hash[(h.finish() % self.by_hash.len() as u64) as usize];
879        if let Some(i) = shard.lock().unwrap_or_else(|p| p.into_inner()).get(hash) {
880            return Some(i.clone());
881        }
882        let bytes = self.store.receipt(hash)?;
883        let info = self.receipt_info(&bytes)?;
884        let mut m = shard.lock().unwrap_or_else(|p| p.into_inner());
885        if m.len() >= 2_048 {
886            m.clear();
887        }
888        m.insert(hash.to_string(), info.clone());
889        Some(info)
890    }
891
892    /// The reply to a request the call parser rejected.
893    pub(crate) fn rejection(&self, r: Rejection, asof: Option<&str>) -> Reply {
894        match r {
895            Rejection::BadRequest(c) => envelope::bad_request_reply(c),
896            Rejection::NotCovered => {
897                let _ = asof;
898                Reply::json(404, envelope::not_covered(None), Cache::Public(self.config.ttl.latest_max_age.as_secs()))
899            }
900            Rejection::Error(ErrorCode::NameUnbound) => {
901                envelope::name_unbound_reply(self.config.ttl.latest_max_age.as_secs())
902            }
903            Rejection::Error(ErrorCode::ForeignUnavailable) => {
904                // The other mesh could not be reached while resolving the name (spec-0.4.3).
905                let d = FOREIGN_DETAIL.with(|d| d.take()).unwrap_or(datumesh_core::ForeignDetail::Network);
906                envelope::answer_reply(&datumesh_core::Answer::foreign_unavailable(d), Cache::NoStore)
907            }
908            Rejection::Error(c) => envelope::error_reply(c),
909        }
910    }
911
912    /// The licence and payment rules for a target and principal: a refusal, and whether the
913    /// answer depends on who asks.
914    pub(crate) fn policy(&self, target: &Target, principal: Option<&Principal>) -> (Option<Reply>, bool) {
915        if self.config.policy.rules.is_empty() {
916            return (None, false);
917        }
918        match self.config.policy.rule(&target.fn_name()) {
919            None => (None, false),
920            Some(Rule::Principals(ps)) => {
921                if principal.is_some_and(|p| ps.contains(&p.id)) {
922                    (None, true)
923                } else {
924                    let a = envelope::with_fn_hash(datumesh_core::Answer::unlicensed(), Some(&target.bound.fn_hash));
925                    (Some(envelope::answer_reply(&a, Cache::Private)), true)
926                }
927            }
928            Some(Rule::Price { amount, currency }) => {
929                let authorise = format!("{}/pay/{}", self.config.origin(), target.cell_id());
930                let quote = datumesh_core::Quote { amount: amount.clone(), currency: currency.clone(), authorise };
931                let a =
932                    envelope::with_fn_hash(datumesh_core::Answer::payment_required(quote), Some(&target.bound.fn_hash));
933                (Some(envelope::answer_reply(&a, Cache::Private)), true)
934            }
935        }
936    }
937
938    /// Answers a `/cell` request (blocking; no budget: see [`Mesh::cell`]). `on_target` is
939    /// told the request description as soon as it is known (for `pending`).
940    pub(crate) fn answer(
941        &self,
942        rest: &str,
943        principal: Option<&Principal>,
944        bindings: &dyn Bindings,
945        now: &str,
946        on_target: &mut dyn FnMut(&Target),
947    ) -> Reply {
948        self.answer_with(rest, principal, bindings, now, on_target, true)
949    }
950
951    /// As [`Inner::answer`]; without the canonical link when `link` is false (a batch's
952    /// envelopes have no headers).
953    pub(crate) fn answer_with(
954        &self,
955        rest: &str,
956        principal: Option<&Principal>,
957        bindings: &dyn Bindings,
958        now: &str,
959        on_target: &mut dyn FnMut(&Target),
960        link: bool,
961    ) -> Reply {
962        let target = match call::resolve(rest, now, &self.config.aliases, bindings, false) {
963            Ok(t) => t,
964            Err(r) => return self.rejection(r, None),
965        };
966        on_target(&target);
967        let (refused, private) = self.policy(&target, principal);
968        let reply = match refused {
969            Some(r) => r,
970            None => self.answer_target(&target, now, private, link),
971        };
972        // What the request resolved is per request: the answers are built from cached bytes.
973        match &target.resolved {
974            Some(r) => envelope::with_resolved(reply, r),
975            None => reply,
976        }
977    }
978
979    /// Answers a resolved target as of its asof (or `now`).
980    pub(crate) fn answer_target(&self, target: &Target, now: &str, private: bool, link: bool) -> Reply {
981        let a = target.asof().unwrap_or(now);
982        let stored = target.bound.descriptor.lang == "stored";
983        let local = stored && !self.foreign.is_foreign(&target.cell.ns);
984        if local {
985            if let Some(info) = self.latest.get(target.cell_key(), a) {
986                let mut reply = Reply::json(200, info.ok.clone(), Cache::Immutable);
987                self.cache_rules(&mut reply, target, a, now, private, link);
988                return reply;
989            }
990        }
991        if let Err(e) = self.foreign.prepare(&target.cell, stored, target.asof()) {
992            return self.unavailable(target, e);
993        }
994        // Since 0.2.1: before computing another mesh's function, its publisher's key document
995        // (its declarations, SPEC 8.5), fetched again after its TTL.
996        if !stored {
997            if let Err(e) = self.foreign.ensure_publisher(&target.cell.ns) {
998                return self.unavailable(target, e);
999            }
1000        }
1001        let mut attempt = 0;
1002        loop {
1003            let r = self.engine.get_cell(&target.cell, Some(a));
1004            let inval = self.foreign.take_invalidated();
1005            for id in &inval {
1006                self.version_added(id);
1007            }
1008            if !inval.is_empty() && attempt < 2 {
1009                attempt += 1;
1010                continue;
1011            }
1012            // A formula cell answered: the describe index learns of it (CONTRACT-1.8b H4).
1013            if !stored && matches!(r, Ok(Answer::Receipt(_))) {
1014                self.known.lock().unwrap_or_else(|p| p.into_inner()).note_computed(&target.cell);
1015                self.index_cell_now(&target.cell);
1016            }
1017            return match r {
1018                Ok(Answer::Receipt(bytes)) => {
1019                    let Some(info) = self.receipt_info(&bytes) else {
1020                        return envelope::error_answer_reply(ErrorCode::EvalUnavailable, Some(&target.bound.fn_hash));
1021                    };
1022                    if attempt < 2 {
1023                        match self.foreign.refresh_deps(&info.hash, a) {
1024                            Ok(true) => {
1025                                for id in self.foreign.take_invalidated() {
1026                                    self.version_added(&id);
1027                                }
1028                                attempt += 1;
1029                                continue;
1030                            }
1031                            Ok(false) => {}
1032                            Err(e) => return self.unavailable(target, e),
1033                        }
1034                    }
1035                    if local && target.asof().is_none() {
1036                        self.remember_latest(target, &info, a);
1037                    }
1038                    self.ok_reply(target, info, a, now, private, link)
1039                }
1040                Ok(Answer::NotCovered { fault }) => {
1041                    let mut ans =
1042                        envelope::with_fn_hash(datumesh_core::Answer::not_covered(), Some(&target.bound.fn_hash));
1043                    if let Some(f) = fault {
1044                        ans = ans.with_fault(f);
1045                    }
1046                    let mut reply = envelope::answer_reply(&ans, Cache::Public(0));
1047                    self.cache_rules(&mut reply, target, a, now, private, link);
1048                    if target.asof().is_some() {
1049                        reply.cache = if private {
1050                            Cache::Private
1051                        } else {
1052                            Cache::Public(self.config.ttl.latest_max_age.as_secs())
1053                        };
1054                    }
1055                    reply
1056                }
1057                Ok(_) => envelope::error_answer_reply(ErrorCode::EvalUnavailable, Some(&target.bound.fn_hash)),
1058                Err(EngineError::Unlicensed { cell_id, reason }) => {
1059                    // Licence inheritance refuses the cell (SPEC 8.5, spec-0.4.3): `unlicensed`,
1060                    // no new code. Not immutable: a declaration may come later.
1061                    tracing::info!(target: "datumesh_http", cell_id = %cell_id, reason = reason.as_str(), "unlicensed");
1062                    let a = envelope::with_fn_hash(datumesh_core::Answer::unlicensed(), Some(&target.bound.fn_hash));
1063                    let cache =
1064                        if private { Cache::Private } else { Cache::Public(self.config.ttl.latest_max_age.as_secs()) };
1065                    envelope::answer_reply(&a, cache)
1066                }
1067                Err(EngineError::Cycle { path }) => {
1068                    return self.cycle_reply(&path, Some(a), Some(&target.bound.fn_hash))
1069                }
1070                Err(e) => {
1071                    let (bad, code) = engine_code(&e);
1072                    tracing::info!(target: "datumesh_http", error = %e, code = code.as_str(), "engine");
1073                    if code == ErrorCode::ForeignUnavailable {
1074                        // A cross-mesh fetch inside the evaluation got no response (ForeignError::eval).
1075                        let detail = match &e {
1076                            EngineError::Invalid(c) => {
1077                                c.detail().strip_prefix("cross-mesh: ").unwrap_or(c.detail()).to_string()
1078                            }
1079                            other => other.to_string(),
1080                        };
1081                        return self.unavailable(target, ForeignError { code, detail });
1082                    }
1083                    if bad {
1084                        envelope::bad_request_reply(code)
1085                    } else if code == ErrorCode::InputUnavailable {
1086                        self.unavailable(target, ForeignError { code, detail: e.to_string() })
1087                    } else {
1088                        envelope::error_answer_reply(code, Some(&target.bound.fn_hash))
1089                    }
1090                }
1091            };
1092        }
1093    }
1094
1095    fn cache_rules(&self, reply: &mut Reply, target: &Target, a: &str, now: &str, private: bool, link: bool) {
1096        let latest = self.config.ttl.latest_max_age.as_secs();
1097        reply.cache = if private {
1098            Cache::Private
1099        } else {
1100            match target.asof() {
1101                Some(x) if x <= now => Cache::Immutable,
1102                Some(_) => Cache::Public(latest),
1103                None => Cache::Public(latest),
1104            }
1105        };
1106        if link && target.asof().is_none() {
1107            reply.canonical = Some(format!("{}/cell/{}", self.config.origin(), target.rest_asof(a)));
1108        }
1109    }
1110
1111    /// The answer when something the mesh depends on failed: `stale` with the last receipt
1112    /// it served for the cell, when it has one that answers the request; else `error`.
1113    fn unavailable(&self, target: &Target, e: ForeignError) -> Reply {
1114        tracing::info!(target: "datumesh_http", code = e.code.as_str(), detail = %e.detail, "unavailable");
1115        // A suspended namespace (9.9, 5.5): the import is refused, so `not_covered` with its error,
1116        // and not the last receipt served (a cell served before is served as before, not imported).
1117        if e.code == crate::foreign::NAMESPACE_SUSPENDED {
1118            if let Ok(a) = datumesh_core::Answer::not_covered().with_error(ErrorCode::NamespaceSuspended) {
1119                return envelope::answer_reply(&envelope::with_fn_hash(a, Some(&target.bound.fn_hash)), Cache::NoStore);
1120            }
1121        }
1122        let lg = self.last_good.lock().unwrap_or_else(|p| p.into_inner());
1123        if let Some(g) = lg.get(&last_good_key(target)) {
1124            if target.asof().is_none_or(|a| g.info.asof.as_str() <= a) {
1125                let age = g.at.elapsed().as_secs();
1126                let a = datumesh_core::Answer::stale(g.info.parsed.clone(), g.info.trust, age);
1127                let a = envelope::with_fn_hash(a, Some(&g.info.parsed.cell().fn_hash));
1128                return envelope::answer_reply(&a, Cache::NoStore);
1129            }
1130        }
1131        if let Some(d) = e.foreign_detail() {
1132            let a = envelope::with_fn_hash(datumesh_core::Answer::foreign_unavailable(d), Some(&target.bound.fn_hash));
1133            return envelope::answer_reply(&a, Cache::NoStore);
1134        }
1135        envelope::error_answer_reply(e.code, Some(&target.bound.fn_hash))
1136    }
1137
1138    /// The keys to verify `receipt` with: the documents of its signer and of the signers of
1139    /// its inputs and slice members held in the store, from the cache or the store's
1140    /// namespace records.
1141    pub(crate) fn keys_for_receipt(&self, receipt: &[u8]) -> Keys {
1142        let mut hosts = std::collections::BTreeSet::new();
1143        if let Ok(v) = json::parse(receipt) {
1144            if let Some(h) = v.get("computed_by").and_then(Value::as_str) {
1145                hosts.insert(h.split('#').next().unwrap_or("").to_string());
1146            }
1147            if let Some(Value::Array(ins)) = v.get("inputs") {
1148                let mut members = Vec::new();
1149                for i in ins.iter().filter_map(Value::as_str).take(10_000) {
1150                    if i.starts_with("slice:") {
1151                        if let Some(m) =
1152                            self.store.manifest(i).and_then(|m| datumesh_core::Manifest::from_json(&m).ok())
1153                        {
1154                            members.extend(m.members().iter().take(10_000).cloned());
1155                        }
1156                    } else {
1157                        members.push(i.to_string());
1158                    }
1159                }
1160                for h in members.iter().take(20_000) {
1161                    if let Some(r) = self.store.receipt(h).and_then(|b| Receipt::from_json(&b).ok()) {
1162                        hosts.insert(r.signer_host().to_string());
1163                    }
1164                }
1165            }
1166        }
1167        let mut keys = Keys::new();
1168        for h in hosts {
1169            if let Some(r) = self.keys.result(&h) {
1170                keys.insert_result(&h, r);
1171            } else if let Some(doc) = self.key_doc(&h) {
1172                keys.insert(doc);
1173            }
1174        }
1175        keys
1176    }
1177
1178    /// A host's key document: the cache's, else the store's namespace record.
1179    fn key_doc(&self, host: &str) -> Option<KeyDoc> {
1180        if let Some((d, _, _)) = self.keys.get(host) {
1181            return Some(d);
1182        }
1183        let ns = self.store.namespace(host).ok()??;
1184        keys_from_response(host, 200, Some(&ns.keydoc)).ok()
1185    }
1186}
1187
1188fn verified(r: datumesh_core::VerifyResult) -> Verified {
1189    let s = |v: Option<&str>| v.map(|x| Value::String(x.to_string())).unwrap_or(Value::Null);
1190    let body = Value::Object(vec![
1191        ("blob_checked".into(), Value::Bool(r.blob_checked)),
1192        ("code".into(), s(r.code.map(|c| c.as_str()))),
1193        ("computed_by".into(), s(r.computed_by.as_deref())),
1194        ("detail".into(), Value::String(r.detail.clone())),
1195        ("hash".into(), s(r.hash.as_deref())),
1196        ("level".into(), s(r.level.map(|l| l.as_str()))),
1197        ("ok".into(), Value::Bool(r.ok)),
1198        ("path".into(), Value::String(r.path.clone())),
1199    ]);
1200    Verified { body: body.to_jcs() }
1201}
1202
1203/// What `POST /verify` reports.
1204#[derive(Clone, Debug, PartialEq, Eq)]
1205pub struct Verified {
1206    /// The result as JSON (`ok`, `level`, `code`, `hash`, `computed_by`, `path`, `detail`).
1207    pub body: Vec<u8>,
1208}
1209
1210impl Mesh {
1211    /// A builder for a mesh on `store` with `config`.
1212    pub fn builder(config: Config, store: Arc<dyn Store + Send + Sync>) -> MeshBuilder {
1213        MeshBuilder {
1214            config,
1215            store,
1216            key: None,
1217            key_file: None,
1218            runtimes: Vec::new(),
1219            formula: true,
1220            clock: Arc::new(SystemClock),
1221            fuel: None,
1222            licence: None,
1223            memo_capacity: None,
1224        }
1225    }
1226
1227    /// The configuration.
1228    pub fn config(&self) -> &Config {
1229        &self.inner.config
1230    }
1231
1232    /// The engine.
1233    pub fn engine(&self) -> &Arc<Engine> {
1234        &self.inner.engine
1235    }
1236
1237    /// Forgets every memoised answer (the engine's memo and the mesh's), for a process that
1238    /// wrote versions without telling the mesh.
1239    pub fn clear_memo(&self) {
1240        self.inner.engine.clear_memo();
1241        self.inner.latest.clear();
1242        self.inner.slices.lock().unwrap_or_else(|p| p.into_inner()).clear();
1243    }
1244
1245    /// The cross-mesh resolver.
1246    pub fn foreign(&self) -> &Foreign {
1247        &self.inner.foreign
1248    }
1249
1250    /// The store.
1251    pub fn store(&self) -> &Arc<dyn Store + Send + Sync> {
1252        &self.inner.store
1253    }
1254
1255    /// The key documents the mesh holds for other hosts (fetched, or inserted here): what
1256    /// `POST /verify` verifies with, besides the store's namespace records.
1257    pub fn keys(&self) -> &KeyCache {
1258        &self.inner.keys
1259    }
1260
1261    /// Tells the engine that a stored version was added for `cell_id` outside the mesh.
1262    pub fn stored_version_added(&self, cell_id: &str) {
1263        self.inner.version_added(cell_id);
1264    }
1265
1266    /// The envelope for `rest` (a call or a cell URI's rest, query included, as received at
1267    /// `/cell/`), for `principal`. Blocking, and without a budget: what a `/cell` request
1268    /// gets when it finishes within its budget, or its job's result.
1269    pub fn cell(&self, rest: &str, principal: Option<&Principal>) -> Reply {
1270        let now = self.inner.now();
1271        self.inner.answer(rest, principal, &*self.inner, &now, &mut |_| {})
1272    }
1273
1274    /// The envelopes of a batch, in order (blocking, no budget).
1275    pub fn cells(&self, rests: &[String], principal: Option<&Principal>) -> Vec<Reply> {
1276        let now = self.inner.now();
1277        let memo = Memo::new(&self.inner);
1278        rests.iter().map(|r| self.inner.answer(r, principal, &memo, &now, &mut |_| {})).collect()
1279    }
1280
1281    /// Resolves a request to its cell, as `/cell` does (or to its slice, with `slice`).
1282    pub fn resolve(&self, rest: &str, slice: bool) -> Result<Arc<Target>, Rejection> {
1283        let now = self.inner.now();
1284        call::resolve(rest, &now, &self.inner.config.aliases, &*self.inner, slice)
1285    }
1286
1287    /// The describe answer (SPEC 16.8, 9.6) for `id`: a class name, an object's key encoding, an
1288    /// object id, a cell id or a short id (12 hex digits). A class or an object: the body is a
1289    /// `describe-class` or `describe-object` document, `200`; an id the mesh does not know, `404`;
1290    /// a short id naming more than one object or cell the mesh knows, `400` with
1291    /// `E_SHORT_ID_AMBIGUOUS`. A class hash is not an id (9.6). Blocking. Since 0.2.2.
1292    pub fn describe(&self, id: &str) -> Reply {
1293        self.inner.describe(id)
1294    }
1295
1296    /// `POST /whatif` (5.9): the scenario answer to the request's body (see [`crate::scenario`]).
1297    pub(crate) fn whatif(&self, body: &[u8], principal: Option<&Principal>) -> Reply {
1298        self.inner.whatif(body, principal)
1299    }
1300
1301    /// `POST /x/calc` (Build 2 Part 3, `calc`): see [`crate::calc`].
1302    pub(crate) fn calc(&self, body: &[u8], principal: Option<&Principal>) -> Reply {
1303        self.inner.calc(body, principal)
1304    }
1305
1306    /// Verifies a receipt (SPEC 10.1) with the mesh's keys; at level `recomputed` with the
1307    /// engine as the resolver unless `signed_only`; `blob` is content for step 7.7.
1308    pub fn verify(&self, receipt: &[u8], signed_only: bool, blob: Option<&[u8]>) -> Verified {
1309        let keys = self.inner.keys_for_receipt(receipt);
1310        let resolver: Option<&dyn datumesh_core::Resolver> =
1311            if signed_only { None } else { Some(&*self.inner.engine as &dyn datumesh_core::Resolver) };
1312        verified(verify(receipt, &keys, resolver, blob))
1313    }
1314
1315    /// Verifies a receipt with the mesh's keys and what the caller offers for step 7: a
1316    /// `recompute` object as SPEC 11 describes it (`descriptor`, `inputs`, `manifests`,
1317    /// `lineage`, `value`), the core's `MapResolver` over it. Without one, steps 1-6 only.
1318    pub fn verify_offered(&self, receipt: &[u8], recompute: Option<&Value>, blob: Option<&[u8]>) -> Verified {
1319        let mut keys = self.inner.keys_for_receipt(receipt);
1320        // The signers of the receipts offered for step 7 too.
1321        let mut hosts = std::collections::BTreeSet::new();
1322        for (_, r) in recompute.and_then(|rc| rc.get("inputs")).and_then(Value::as_object).unwrap_or(&[]) {
1323            if let Some(h) = r
1324                .as_str()
1325                .and_then(|t| json::parse(t.as_bytes()).ok())
1326                .and_then(|v| v.get("computed_by").and_then(Value::as_str).map(str::to_string))
1327            {
1328                hosts.insert(h.split('#').next().unwrap_or("").to_string());
1329            }
1330        }
1331        for h in hosts {
1332            if let Some(r) = self.inner.keys.result(&h) {
1333                keys.insert_result(&h, r);
1334            } else if let Some(doc) = self.inner.key_doc(&h) {
1335                keys.insert(doc);
1336            }
1337        }
1338        let Some(rc) = recompute else {
1339            return verified(verify(receipt, &keys, None, blob));
1340        };
1341        let mut m = datumesh_core::MapResolver::new();
1342        let cell = json::parse(receipt).ok().and_then(|r| r.get("cell").cloned());
1343        if let (Some(d), Some(h)) =
1344            (rc.get("descriptor"), cell.as_ref().and_then(|c| c.get("fn_hash")).and_then(Value::as_str))
1345        {
1346            m.insert_descriptor(h, d.to_jcs());
1347        }
1348        for (h, r) in rc.get("inputs").and_then(Value::as_object).unwrap_or(&[]) {
1349            m.insert_receipt(h.as_str(), r.as_str().unwrap_or("").as_bytes());
1350        }
1351        for (e, t) in rc.get("manifests").and_then(Value::as_object).unwrap_or(&[]) {
1352            m.insert_manifest(e.as_str(), t.as_str().unwrap_or("").as_bytes());
1353        }
1354        let value = rc.get("value").and_then(|v| datumesh_core::Scalar::from_json(&v.to_jcs()).ok());
1355        if let (Some(value), Some(c)) = (value, cell.as_ref().and_then(|c| datumesh_core::Cell::from_value(c).ok())) {
1356            m.insert_value(c.id(), value);
1357            if let Some(Value::Array(l)) = rc.get("lineage") {
1358                m.insert_lineage(c.id(), l.iter().filter_map(Value::as_str));
1359            }
1360        }
1361        verified(verify(receipt, &keys, Some(&m), blob))
1362    }
1363
1364    /// The mesh document (SPEC 9.6) this mesh serves: its URLs on the base URL when one is
1365    /// set ([`Config::base_url`]), else on `https://<host>`.
1366    pub fn mesh_document(&self) -> Vec<u8> {
1367        let o = self.inner.config.mesh_document_base();
1368        let mut ep: BTreeMap<&str, String> = [
1369            ("blob", format!("{o}/blob/{{sha256}}")),
1370            ("cell", format!("{o}/cell/{{cell}}")),
1371            ("cells", format!("{o}/cells")),
1372            ("x_bundles", format!("{o}/bundles")),
1373            ("x_calc", format!("{o}/x/calc")),
1374            ("x_interval", format!("{o}/x/interval/{{function}}")),
1375            ("describe", format!("{o}/describe/{{id}}")),
1376            ("descriptor", format!("{o}/descriptor/{{hash}}")),
1377            ("explain", format!("{o}/explain/{{cell}}")),
1378            ("find", format!("{o}/find")),
1379            ("jobs", format!("{o}/jobs/{{job}}")),
1380            ("manifest", format!("{o}/manifest/{{slice_id}}/{{members_hash}}")),
1381            ("receipt", format!("{o}/receipt/{{hash}}")),
1382            ("slice", format!("{o}/slice/{{cell}}")),
1383            ("verify", format!("{o}/verify")),
1384            ("whatif", format!("{o}/whatif")),
1385            ("versions", format!("{o}/versions/{{cell}}")),
1386        ]
1387        .into_iter()
1388        .collect();
1389        if self.inner.config.key_names {
1390            ep.insert("x_keys", format!("{o}/x/keys"));
1391        }
1392        Value::Object(vec![
1393            ("bindings".into(), Value::String(format!("{o}/bindings"))),
1394            (
1395                "endpoints".into(),
1396                Value::Object(ep.into_iter().map(|(k, v)| (k.to_string(), Value::String(v))).collect()),
1397            ),
1398            ("host".into(), Value::String(self.inner.config.host.clone())),
1399            ("proto".into(), Value::String("datumesh/1".into())),
1400        ])
1401        .to_jcs()
1402    }
1403
1404    /// The key document this mesh serves: the store's namespace record for its host.
1405    pub fn key_document(&self) -> Option<Vec<u8>> {
1406        self.inner.store.namespace(&self.inner.config.host).ok().flatten().map(|n| n.keydoc)
1407    }
1408
1409    /// The Prometheus text of `/metrics`.
1410    pub fn metrics(&self) -> String {
1411        crate::http::metrics_text(&self.inner)
1412    }
1413
1414    /// The number of requests counted for a route template (e.g. `/cell/{cell}`).
1415    pub fn requests_of(&self, route: &str) -> u64 {
1416        self.inner.metrics.requests_of(route)
1417    }
1418
1419    /// The host of a function name (its first segment).
1420    pub fn host_of(name: &str) -> &str {
1421        host_of(name)
1422    }
1423
1424    /// How long the mesh has run.
1425    pub fn uptime(&self) -> Duration {
1426        self.inner.started.elapsed()
1427    }
1428}
1429
1430impl Inner {
1431    /// The class `name` is bound to as of `asof`, with its flattened descriptor (not noted for the
1432    /// describe index: see [`Bindings::class`]).
1433    pub(crate) fn class_binding(&self, name: &str, asof: &str) -> Result<Option<Arc<call::BoundClass>>, ErrorCode> {
1434        let store_failed = |e: datumesh_store::Error| {
1435            tracing::warn!(target: "datumesh_http", error = %e, "resolve_class_binding");
1436            ErrorCode::InputUnavailable
1437        };
1438        let Some(b) = self.store.resolve_class_binding(name, asof).map_err(store_failed)? else { return Ok(None) };
1439        let Some(bytes) = self.store.class(&b.class_hash).map_err(store_failed)? else {
1440            return Err(ErrorCode::InputUnavailable);
1441        };
1442        let d = datumesh_core::ClassDescriptor::from_json(&bytes).map_err(|_| ErrorCode::InputUnavailable)?;
1443        Ok(Some(Arc::new(call::BoundClass { class_hash: b.class_hash, since: b.tx, descriptor: Arc::new(d) })))
1444    }
1445}
1446
1447#[cfg(test)]
1448mod tests {
1449    use super::*;
1450    use std::sync::atomic::{AtomicUsize, Ordering};
1451
1452    /// A batch's chunks start together and ask for the same names: one store round trip per
1453    /// name and asof for the whole batch, not one per chunk.
1454    #[test]
1455    fn a_batch_looks_each_binding_up_once() {
1456        let seen = Seen::default();
1457        let lookups = AtomicUsize::new(0);
1458        let start = std::sync::Barrier::new(6);
1459        let look = || {
1460            lookups.fetch_add(1, Ordering::SeqCst);
1461            std::thread::sleep(Duration::from_millis(50));
1462            Ok(None)
1463        };
1464        std::thread::scope(|s| {
1465            for _ in 0..6 {
1466                s.spawn(|| {
1467                    start.wait();
1468                    assert!(matches!(seen.lookup("a.example/f", "2026-01-01T00:00:00Z", look), Ok(None)));
1469                });
1470            }
1471        });
1472        assert_eq!(lookups.load(Ordering::SeqCst), 1, "one lookup for six chunks");
1473        // Found: no lookup.
1474        assert!(matches!(seen.lookup("a.example/f", "2026-01-01T00:00:00Z", look), Ok(None)));
1475        assert_eq!(lookups.load(Ordering::SeqCst), 1);
1476        // Another asof, another name: looked up.
1477        assert!(matches!(seen.lookup("a.example/f", "2026-01-02T00:00:00Z", look), Ok(None)));
1478        assert!(matches!(seen.lookup("a.example/g", "2026-01-01T00:00:00Z", look), Ok(None)));
1479        assert_eq!(lookups.load(Ordering::SeqCst), 3);
1480        // A failed lookup is not kept.
1481        let fail = || Err(ErrorCode::InputUnavailable);
1482        assert!(matches!(seen.lookup("a.example/h", "2026-01-01T00:00:00Z", fail), Err(ErrorCode::InputUnavailable)));
1483        assert!(matches!(seen.lookup("a.example/h", "2026-01-01T00:00:00Z", look), Ok(None)));
1484        assert_eq!(lookups.load(Ordering::SeqCst), 4);
1485    }
1486}