Skip to main content

datumesh_http/
http.rs

1//! The endpoints: an axum router over a [`Mesh`].
2
3use std::net::{IpAddr, Ipv4Addr, SocketAddr};
4use std::sync::{Arc, Mutex};
5use std::time::{Duration, Instant};
6
7use axum::body::{Body, Bytes};
8use axum::extract::{ConnectInfo, MatchedPath, Path, Query, Request, State};
9use axum::http::{header, HeaderMap, HeaderValue, StatusCode, Uri};
10use axum::middleware::{self, Next};
11use axum::response::{IntoResponse, Response};
12use axum::routing::{get, post};
13use axum::Router;
14use datumesh_core::json::{self, Value};
15use datumesh_core::{lexical, ErrorCode, Pending, Request as CoreRequest};
16use tracing::Instrument;
17use utoipa::OpenApi;
18
19use crate::apikeys::Principal;
20use crate::envelope::{self, Cache, Reply, JSON};
21use crate::jobs::Poll;
22use crate::limit::Client;
23use crate::mesh::{Inner, Memo, Mesh};
24
25/// The principal of a request, for the access log.
26#[derive(Clone)]
27struct Who(Option<String>);
28
29fn to_response(r: Reply) -> Response {
30    let mut resp = Response::new(Body::from(r.body));
31    *resp.status_mut() = StatusCode::from_u16(r.status).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
32    let h = resp.headers_mut();
33    h.insert(header::CONTENT_TYPE, HeaderValue::from_static(r.content_type));
34    if let Ok(v) = HeaderValue::from_str(&r.cache.header()) {
35        h.insert(header::CACHE_CONTROL, v);
36    }
37    if let Some(c) = r.canonical {
38        if let Ok(v) = HeaderValue::from_str(&format!("<{c}>; rel=\"canonical\"")) {
39            h.insert(header::LINK, v);
40        }
41    }
42    if let Some(s) = r.retry_after {
43        h.insert(header::RETRY_AFTER, HeaderValue::from(s));
44    }
45    resp
46}
47
48fn plain_json(status: u16, v: Value, cache: Cache) -> Response {
49    to_response(Reply::json(status, v.to_jcs(), cache))
50}
51
52fn not_found(what: &str) -> Response {
53    plain_json(404, Value::Object(vec![("error".into(), Value::String(format!("no such {what}")))]), Cache::Public(60))
54}
55
56/// The client: its IP and, with a valid API key, its principal. An `Authorization` header
57/// that is not a key of this mesh is answered `unlicensed`.
58fn who(inner: &Inner, headers: &HeaderMap, peer: IpAddr) -> Result<(Client, Option<Principal>), Reply> {
59    let ip = client_ip(inner, headers, peer);
60    match headers.get(header::AUTHORIZATION) {
61        None => Ok((Client::Ip(ip), None)),
62        Some(v) => {
63            let p = v
64                .to_str()
65                .ok()
66                .and_then(|s| s.strip_prefix("Bearer "))
67                .and_then(|k| inner.config.api_keys.principal(k.trim()));
68            match p {
69                Some(p) => Ok((Client::Key(p.id.clone(), p.per_minute), Some(p.clone()))),
70                None => Err(Reply::json(403, envelope::unlicensed(), Cache::Private)),
71            }
72        }
73    }
74}
75
76fn client_ip(inner: &Inner, headers: &HeaderMap, peer: IpAddr) -> IpAddr {
77    if inner.config.trust_proxy {
78        if let Some(ip) = headers
79            .get("x-forwarded-for")
80            .and_then(|v| v.to_str().ok())
81            .and_then(|s| s.rsplit(',').next())
82            .and_then(|s| s.trim().parse::<IpAddr>().ok())
83        {
84            return ip;
85        }
86    }
87    peer
88}
89
90fn peer_of(req: &Request) -> IpAddr {
91    req.extensions().get::<ConnectInfo<SocketAddr>>().map(|c| c.0.ip()).unwrap_or(IpAddr::V4(Ipv4Addr::UNSPECIFIED))
92}
93
94/// `rest` of a `/<prefix>/<rest>` request, query included, as received.
95fn rest_of(uri: &Uri, prefix: &str) -> String {
96    let path = uri.path().strip_prefix(prefix).unwrap_or("");
97    match uri.query() {
98        Some(q) => format!("{path}?{q}"),
99        None => path.to_string(),
100    }
101}
102
103/// Runs `f` on the pool within the request's budget: its reply, or, past the budget, a
104/// `pending` envelope whose job gets the reply when `f` finishes. `f` reports the request
105/// description as soon as it knows it.
106async fn within_budget(
107    inner: Arc<Inner>,
108    f: impl FnOnce(&Inner, &mut dyn FnMut(&crate::call::Target)) -> Reply + Send + 'static,
109) -> Reply {
110    let deadline = tokio::time::Instant::now() + inner.config.budget;
111    let (tx, mut rx) = tokio::sync::watch::channel::<Option<CoreRequest>>(None);
112    let pool = inner.pool.clone();
113    let i2 = inner.clone();
114    let mut task = tokio::spawn(async move {
115        pool.run(move || {
116            let mut on = |t: &crate::call::Target| {
117                if let Some(r) = request_of(t) {
118                    let _ = tx.send(Some(r));
119                }
120            };
121            f(&i2, &mut on)
122        })
123        .await
124    });
125    match tokio::time::timeout_at(deadline, &mut task).await {
126        Ok(Ok(Some(r))) => r,
127        Ok(_) => envelope::error_reply(ErrorCode::EvalUnavailable),
128        Err(_) => {
129            let request = rx.borrow_and_update().clone();
130            let Some(request) = request else {
131                // Past the budget before the request was even resolved (the store or another
132                // mesh is slow): the mesh cannot say what it is computing.
133                return envelope::error_reply(ErrorCode::EvalUnavailable);
134            };
135            pend(inner, request, task)
136        }
137    }
138}
139
140/// The request description of a target: its cell id, asof and pin (spec-0.4.2).
141fn request_of(t: &crate::call::Target) -> Option<CoreRequest> {
142    let r = CoreRequest::new(t.cell_id(), t.asof()).ok()?;
143    match &t.pin {
144        Some(p) => r.with_pin(p).ok(),
145        None => Some(r),
146    }
147}
148
149/// A job for a request still running in `task`.
150fn pend(inner: Arc<Inner>, request: CoreRequest, task: tokio::task::JoinHandle<Option<Reply>>) -> Reply {
151    let Some(id) = inner.jobs.create(request.clone()) else {
152        return envelope::error_reply(ErrorCode::EvalUnavailable);
153    };
154    let poll = format!("{}/jobs/{id}", inner.config.origin());
155    let est = inner.jobs.estimate(inner.config.budget.as_secs());
156    let i2 = inner.clone();
157    let id2 = id.clone();
158    tokio::spawn(async move {
159        let r = match task.await {
160            Ok(Some(r)) if r.status != 202 => r,
161            _ => envelope::error_reply(ErrorCode::EvalUnavailable),
162        };
163        let mut r = r;
164        r.cache = Cache::NoStore;
165        r.canonical = None;
166        i2.jobs.complete(&id2, r);
167    });
168    match Pending::new(&id, est, &poll, request) {
169        Ok(p) => Reply::json(202, envelope::pending(&p), Cache::NoStore),
170        Err(_) => envelope::error_reply(ErrorCode::EvalUnavailable),
171    }
172}
173
174fn admit(inner: &Inner, client: &Client, cost: u32) -> Result<(), Reply> {
175    inner.limiter.admit(client, cost).map_err(envelope::rate_limited_reply)
176}
177
178#[derive(OpenApi)]
179#[openapi(
180    info(
181        title = "datumesh-mesh",
182        description = "The Datumesh engine behind HTTPS (spec-0.4.1): cell envelopes (SPEC 5.5), batches, slices, receipts, descriptors, manifests, blobs, lineage, search, verification, jobs, key and mesh documents (SPEC 9.2, 9.6), bindings (SPEC 9.7).",
183        license(name = "Apache-2.0")
184    ),
185    paths(
186        cell,
187        cells,
188        slice,
189        versions,
190        receipt,
191        descriptor,
192        manifest,
193        blob,
194        explain,
195        find,
196        verify_receipt,
197        job,
198        key_document,
199        mesh_document,
200        bindings,
201        describe,
202        health,
203        metrics,
204        openapi_json
205    )
206)]
207struct ApiDoc;
208
209/// The OpenAPI document (generated from the handlers' annotations).
210pub fn openapi() -> utoipa::openapi::OpenApi {
211    ApiDoc::openapi()
212}
213
214/// The envelope of a cell.
215#[utoipa::path(get, path = "/cell/{cell}",
216    params(("cell" = String, Path, description = "A cell URI's rest (SPEC 5.3), byte for byte, `asof` included; or a call (SPEC 15.3's read syntax, URL-encoded) with an optional `?asof=`.")),
217    responses(
218        (status = 200, description = "`ok` or `stale`", content_type = "application/json"),
219        (status = 202, description = "`pending`: poll the job (SPEC 5.5)", content_type = "application/json"),
220        (status = 400, description = "`bad_request`", content_type = "application/json"),
221        (status = 402, description = "`payment_required`", content_type = "application/json"),
222        (status = 403, description = "`unlicensed`", content_type = "application/json"),
223        (status = 404, description = "`not_covered` (with `fault` when the evaluation faulted)", content_type = "application/json"),
224        (status = 406, description = "the Accept header excludes JSON"),
225        (status = 429, description = "`rate_limited`, with `Retry-After`", content_type = "application/json"),
226        (status = 500, description = "`error`", content_type = "application/json"),
227        (status = 503, description = "`error`: the store or another mesh is unavailable", content_type = "application/json"),
228    ))]
229async fn cell(State(mesh): State<Mesh>, uri: Uri, headers: HeaderMap, req: Request) -> Response {
230    let inner = mesh.inner.clone();
231    let (client, principal) = match who(&inner, &headers, peer_of(&req)) {
232        Ok(x) => x,
233        Err(r) => return to_response(r),
234    };
235    let mut resp = match admit(&inner, &client, 1) {
236        Err(r) => to_response(r),
237        Ok(()) => {
238            let rest = rest_of(&uri, "/cell/");
239            let p = principal.clone();
240            let r = within_budget(inner.clone(), move |i, on| {
241                let now = i.now();
242                i.answer(&rest, p.as_ref(), i, &now, on)
243            })
244            .await;
245            if let Some(o) = r.outcome() {
246                inner.metrics.outcome("/cell/{cell}", &o);
247            }
248            to_response(r)
249        }
250    };
251    resp.extensions_mut().insert(Who(principal.map(|p| p.id)));
252    resp
253}
254
255/// A batch of cells.
256#[utoipa::path(post, path = "/cells",
257    request_body(content = String, description = "A JSON array of requests, each a call or a cell URI (`cell://...`) as a string, or `{\"cell\": <that>}`; at most the configured bound (1,000).", content_type = "application/json"),
258    responses(
259        (status = 200, description = "A JSON array of envelopes, in order; each has the HTTP status its outcome has (`error`: 500)", content_type = "application/json"),
260        (status = 400, description = "Not such an array, or too long", content_type = "application/json"),
261        (status = 429, description = "`rate_limited` (a batch of n cells counts as n / 50 requests)", content_type = "application/json"),
262    ))]
263async fn cells(State(mesh): State<Mesh>, headers: HeaderMap, req: Request) -> Response {
264    let inner = mesh.inner.clone();
265    let peer = peer_of(&req);
266    let (client, principal) = match who(&inner, &headers, peer) {
267        Ok(x) => x,
268        Err(r) => return to_response(r),
269    };
270    let body = match axum::body::to_bytes(req.into_body(), 64 << 20).await {
271        Ok(b) => b,
272        Err(_) => return to_response(envelope::bad_request_reply(ErrorCode::CanonLimit)),
273    };
274    let items = match parse_batch(&body, inner.config.batch_max) {
275        Ok(i) => i,
276        // A refusal of the whole batch (5.7): one envelope.
277        Err((code, detail)) => {
278            tracing::info!(target: "datumesh_http", detail = %detail, "batch refused");
279            return to_response(envelope::bad_request_reply(code));
280        }
281    };
282    if let Err(r) = admit(&inner, &client, inner.limiter.batch_cost(items.len())) {
283        return to_response(r);
284    }
285    let pieces = batch(inner.clone(), items, principal.clone()).await;
286    let mut counts: std::collections::BTreeMap<&str, u64> = std::collections::BTreeMap::new();
287    for w in &pieces {
288        for (o, k) in &w.outcomes {
289            *counts.entry(o).or_default() += k;
290        }
291    }
292    for (o, k) in counts {
293        inner.metrics.outcomes("/cells", o, k);
294    }
295    // 5.7: [{"envelope": <the envelope>, "status": <its status>}] in request order: the
296    // chunks' runs as they wrote them (the first opening the array).
297    let mut out = Vec::with_capacity(pieces.iter().map(|w| w.bytes.len()).sum::<usize>() + 2);
298    for w in &pieces {
299        out.extend_from_slice(&w.bytes);
300    }
301    out.extend_from_slice(if pieces.is_empty() { b"[]" } else { b"]" });
302    let mut resp =
303        to_response(Reply::json(200, out, if principal.is_some() { Cache::Private } else { Cache::NoStore }));
304    resp.extensions_mut().insert(Who(principal.map(|p| p.id)));
305    resp
306}
307
308fn batch_error(e: &str) -> Response {
309    plain_json(400, Value::Object(vec![("error".into(), Value::String(e.to_string()))]), Cache::NoStore)
310}
311
312/// The batch's requests, or the code of its refusal: `E_PARSE_URI` for a body that is not an
313/// array of requests, `E_CANON_LIMIT` for more than the mesh accepts (5.7).
314fn parse_batch(body: &[u8], max: usize) -> Result<Vec<String>, (ErrorCode, String)> {
315    let pu = |m: String| (ErrorCode::ParseUri, m);
316    // The body is a list of requests, not a protocol object: plain JSON.
317    let v: serde_json::Value = serde_json::from_slice(body).map_err(|e| pu(format!("not JSON: {e}")))?;
318    let serde_json::Value::Array(items) = v else { return Err(pu("not a JSON array".into())) };
319    if items.len() > max {
320        return Err((ErrorCode::CanonLimit, format!("{} requests: at most {max}", items.len())));
321    }
322    items
323        .into_iter()
324        .map(|i| {
325            let s = match i {
326                serde_json::Value::String(s) => s,
327                serde_json::Value::Object(mut o) => match o.remove("cell") {
328                    Some(serde_json::Value::String(s)) => s,
329                    _ => return Err(pu("an object without a string cell".to_string())),
330                },
331                _ => return Err(pu("an item that is neither a string nor an object".to_string())),
332            };
333            Ok(match s.strip_prefix("cell://") {
334                Some(r) => r.to_string(),
335                None => s,
336            })
337        })
338        .collect()
339}
340
341/// A run of a batch's answers as 5.7 writes them: the items
342/// `{"envelope": <the envelope>, "status": <its status>}` (JCS member order) joined by commas,
343/// and how many of each outcome they hold. Each chunk writes its own on the pool, in
344/// parallel: reading 1,000 envelopes (each a receipt, the function hash echoed) to copy and
345/// count them is the costliest part of a warm batch after the answers themselves.
346#[derive(Default)]
347struct Written {
348    bytes: Vec<u8>,
349    outcomes: Vec<(String, u64)>,
350}
351
352impl Written {
353    /// The run of `replies`, opening the array when `first`, else continuing it.
354    fn of<'a>(first: bool, replies: impl Iterator<Item = &'a Reply> + Clone) -> Written {
355        let mut w = Written {
356            bytes: Vec::with_capacity(replies.clone().map(|r| r.body.len() + 32).sum::<usize>() + 1),
357            outcomes: vec![],
358        };
359        w.bytes.push(if first { b'[' } else { b',' });
360        for (i, r) in replies.enumerate() {
361            if i > 0 {
362                w.bytes.push(b',');
363            }
364            w.push(r);
365        }
366        w
367    }
368
369    fn push(&mut self, r: &Reply) {
370        self.bytes.extend_from_slice(b"{\"envelope\":");
371        self.bytes.extend_from_slice(&r.body);
372        self.bytes.extend_from_slice(b",\"status\":");
373        push_u16(&mut self.bytes, r.status);
374        self.bytes.push(b'}');
375        if let Some(o) = r.outcome_str() {
376            match self.outcomes.iter_mut().find(|(x, _)| x == o) {
377                Some((_, k)) => *k += 1,
378                None => self.outcomes.push((o.to_string(), 1)),
379            }
380        }
381    }
382}
383
384/// Appends `n` in decimal (a JSON integer).
385fn push_u16(out: &mut Vec<u8>, n: u16) {
386    let mut buf = [0u8; 5];
387    let (mut i, mut n) = (buf.len(), n);
388    loop {
389        i -= 1;
390        buf[i] = b'0' + (n % 10) as u8;
391        n /= 10;
392        if n == 0 {
393            break;
394        }
395    }
396    out.extend_from_slice(&buf[i..]);
397}
398
399/// The machine's parallelism, read once: `available_parallelism` reads the cgroup files on
400/// Linux (hundreds of microseconds in a container), too slow to pay on every batch.
401fn cores() -> usize {
402    static CORES: std::sync::OnceLock<usize> = std::sync::OnceLock::new();
403    *CORES.get_or_init(|| std::thread::available_parallelism().map(|c| c.get()).unwrap_or(4))
404}
405
406/// Answers a batch within the budget: the requests are answered in chunks across the pool
407/// (one chunk per core; bindings looked up once per name and asof within the batch), each
408/// chunk writing its answers ([`Written`]). Those not answered within the budget become
409/// jobs: each is resolved again to the request it describes (a store read), and its chunk's
410/// answer completes the job. The runs are in request order.
411async fn batch(inner: Arc<Inner>, items: Vec<String>, principal: Option<Principal>) -> Vec<Written> {
412    let n = items.len();
413    let deadline = tokio::time::Instant::now() + inner.config.budget;
414    let chunks = n.div_ceil(64).clamp(1, inner.config.pool_size.min(cores()));
415    let per = n.div_ceil(chunks).max(1);
416    let ranges: Vec<(usize, usize)> =
417        (0..chunks).map(|c| (c * per, ((c + 1) * per).min(n))).take_while(|(lo, hi)| lo < hi).collect();
418    let now = inner.now();
419    let items = Arc::new(items);
420    // Each chunk's answers so far; once the budget has passed (`late`), its answers go to
421    // the channel instead, to complete their jobs. A chunk done within the budget leaves
422    // its answers written instead.
423    let shared: Slots = Arc::new(ranges.iter().map(|_| Mutex::new(Slot::default())).collect());
424    let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<(usize, Reply)>();
425    let seen = Arc::new(crate::mesh::Seen::default());
426    let mut tasks = Vec::new();
427    for (c, &(lo, hi)) in ranges.iter().enumerate() {
428        let (i2, items, tx, p, now, shared) =
429            (inner.clone(), items.clone(), tx.clone(), principal.clone(), now.clone(), shared.clone());
430        let seen = seen.clone();
431        let pool = inner.pool.clone();
432        tasks.push(tokio::spawn(async move {
433            pool.run(move || {
434                let memo = Memo::shared(&i2, &seen);
435                for k in lo..hi {
436                    let reply = i2.answer_with(&items[k], p.as_ref(), &memo, &now, &mut |_| {}, false);
437                    let mut slot = shared[c].lock().unwrap_or_else(|p| p.into_inner());
438                    if slot.late {
439                        let _ = tx.send((k, reply));
440                    } else {
441                        slot.answers.push((k, reply));
442                    }
443                }
444                let mut slot = shared[c].lock().unwrap_or_else(|p| p.into_inner());
445                if !slot.late {
446                    let answers = std::mem::take(&mut slot.answers);
447                    slot.written = Some(Written::of(c == 0, answers.iter().map(|(_, r)| r)));
448                }
449            })
450            .await
451        }));
452    }
453    drop(tx);
454    let all = futures_join(&mut tasks, deadline).await;
455    let mut done: Vec<Option<Reply>> = vec![None; n];
456    let mut written: Vec<Option<Written>> = Vec::with_capacity(ranges.len());
457    for slot in shared.iter() {
458        let mut s = slot.lock().unwrap_or_else(|p| p.into_inner());
459        s.late = true;
460        for (k, r) in s.answers.drain(..) {
461            done[k] = Some(r);
462        }
463        written.push(s.written.take());
464    }
465    // A chunk's run: as it wrote it, else its answers (an error for any it has none of: its
466    // task panicked, or the answer completes a job).
467    let runs = |done: Vec<Option<Reply>>, written: Vec<Option<Written>>| -> Vec<Written> {
468        let error = envelope::error_reply(ErrorCode::EvalUnavailable);
469        ranges
470            .iter()
471            .zip(written)
472            .enumerate()
473            .map(|(c, (&(lo, hi), w))| {
474                w.unwrap_or_else(|| Written::of(c == 0, done[lo..hi].iter().map(|d| d.as_ref().unwrap_or(&error))))
475            })
476            .collect()
477    };
478    if all {
479        return runs(done, written);
480    }
481    // Past the budget: every request not answered yet becomes a job, resolved again to the
482    // request it describes.
483    let open: Vec<usize> = ranges
484        .iter()
485        .zip(&written)
486        .filter(|(_, w)| w.is_none())
487        .flat_map(|(&(lo, hi), _)| lo..hi)
488        .filter(|k| done[*k].is_none())
489        .collect();
490    let (i2, items2, open2, now2) = (inner.clone(), items.clone(), open.clone(), now.clone());
491    let requests: Vec<Option<CoreRequest>> = inner
492        .pool
493        .run(move || {
494            let memo = Memo::new(&i2);
495            open2
496                .iter()
497                .map(|k| {
498                    crate::call::resolve(&items2[*k], &now2, &i2.config.aliases, &memo, false)
499                        .ok()
500                        .and_then(|t| request_of(&t))
501                })
502                .collect()
503        })
504        .await
505        .unwrap_or_default();
506    let poll_base = format!("{}/jobs/", inner.config.origin());
507    let est = inner.jobs.estimate(inner.config.budget.as_secs());
508    let mut jobs: Vec<Option<String>> = vec![None; n];
509    for (i, k) in open.iter().enumerate() {
510        let reply = match requests.get(i).cloned().flatten() {
511            Some(req) => match inner.jobs.create(req.clone()) {
512                Some(id) => {
513                    let p = Pending::new(&id, est, &format!("{poll_base}{id}"), req);
514                    jobs[*k] = Some(id);
515                    match p {
516                        Ok(p) => Reply::json(202, envelope::pending(&p), Cache::NoStore),
517                        Err(_) => envelope::error_reply(ErrorCode::EvalUnavailable),
518                    }
519                }
520                None => envelope::error_reply(ErrorCode::EvalUnavailable),
521            },
522            None => envelope::error_reply(ErrorCode::EvalUnavailable),
523        };
524        done[*k] = Some(reply);
525    }
526    let i3 = inner.clone();
527    tokio::spawn(async move {
528        while let Some((k, mut reply)) = rx.recv().await {
529            if let Some(id) = &jobs[k] {
530                reply.cache = Cache::NoStore;
531                reply.canonical = None;
532                i3.jobs.complete(id, reply);
533            }
534        }
535        for t in tasks {
536            let _ = t.await;
537        }
538        // Jobs whose task ended without an answer (a panic) are given up.
539        for id in jobs.into_iter().flatten() {
540            if let Poll::Running(..) = i3.jobs.poll_peek(&id) {
541                i3.jobs.complete(&id, envelope::error_reply(ErrorCode::EvalUnavailable));
542            }
543        }
544    });
545    runs(done, written)
546}
547
548/// A batch chunk's state: whether the budget has passed, its answers so far, and its
549/// answers written once it is done within the budget.
550#[derive(Default)]
551struct Slot {
552    late: bool,
553    answers: Vec<(usize, Reply)>,
554    written: Option<Written>,
555}
556
557/// Each batch chunk's state.
558type Slots = Arc<Vec<Mutex<Slot>>>;
559
560/// Waits for every task until `deadline`: whether all finished.
561async fn futures_join(tasks: &mut [tokio::task::JoinHandle<Option<()>>], deadline: tokio::time::Instant) -> bool {
562    for t in tasks.iter_mut() {
563        if t.is_finished() {
564            continue;
565        }
566        if tokio::time::timeout_at(deadline, t).await.is_err() {
567            return false;
568        }
569    }
570    true
571}
572
573/// The versions of a stored cell (SPEC 9.8, spec-0.4.2).
574#[utoipa::path(get, path = "/versions/{cell}",
575    params(("cell" = String, Path, description = "A cell URI's rest without asof (a pin allowed), then optionally `from=` and `to=` (instants, inclusive bounds on tx) as its last parameters.")),
576    responses(
577        (status = 200, description = "`{\"cell\", \"versions\": [{\"asof\", \"receipt\", \"tx\"}], \"next\"?}` ascending by tx", content_type = "application/json"),
578        (status = 400, description = "`bad_request`", content_type = "application/json"),
579        (status = 404, description = "`not_covered` (with `E_NAME_UNBOUND` for an unbound name)", content_type = "application/json"),
580    ))]
581async fn versions(State(mesh): State<Mesh>, uri: Uri, headers: HeaderMap, req: Request) -> Response {
582    if let Some(r) = ip_admit(&mesh.inner, &headers, &req) {
583        return r;
584    }
585    let inner = mesh.inner.clone();
586    let rest = rest_of(&uri, "/versions/");
587    let r = inner
588        .pool
589        .clone()
590        .run(move || inner.versions(&rest))
591        .await
592        .unwrap_or_else(|| envelope::error_reply(ErrorCode::EvalUnavailable));
593    to_response(r)
594}
595
596/// The most versions in one answer of the versions endpoint.
597const VERSIONS_PAGE: usize = 1000;
598
599impl Inner {
600    fn versions(&self, rest: &str) -> Reply {
601        // `from=` and `to=` are the endpoint's, taken off the end of the query.
602        let (path, query) = match rest.split_once('?') {
603            Some((p, q)) => (p, q),
604            None => (rest, ""),
605        };
606        let mut ps: Vec<&str> = if query.is_empty() { vec![] } else { query.split('&').collect() };
607        let (mut from, mut to) = (None, None);
608        while let Some(last) = ps.last() {
609            if let Some(v) = last.strip_prefix("from=") {
610                from = Some(v.to_string());
611            } else if let Some(v) = last.strip_prefix("to=") {
612                to = Some(v.to_string());
613            } else {
614                break;
615            }
616            ps.pop();
617        }
618        for t in [&from, &to].into_iter().flatten() {
619            if lexical::check_instant(t, "$").is_err() {
620                return envelope::bad_request_reply(ErrorCode::CanonInstant);
621            }
622        }
623        let cell_rest = if ps.is_empty() { path.to_string() } else { format!("{path}?{}", ps.join("&")) };
624        let now = self.now();
625        let target = match crate::call::resolve(&cell_rest, &now, &self.config.aliases, self, false) {
626            Ok(t) => t,
627            Err(r) => return self.rejection(r, None),
628        };
629        if target.asof().is_some() {
630            return envelope::bad_request_reply(ErrorCode::ParseUri);
631        }
632        let lo = from.unwrap_or_else(|| "0001-01-01T00:00:00.000000Z".into());
633        let hi = to.clone().unwrap_or_else(|| "9999-12-31T23:59:59.999999Z".into());
634        let mut vs = match self.store.versions(target.cell_id(), &lo, &hi) {
635            Ok(v) => v,
636            Err(_) => return envelope::error_answer_reply(ErrorCode::InputUnavailable, Some(&target.bound.fn_hash)),
637        };
638        vs.sort_by(|a, b| (&a.tx, &a.receipt_hash).cmp(&(&b.tx, &b.receipt_hash)));
639        let more = vs.len() > VERSIONS_PAGE;
640        vs.truncate(VERSIONS_PAGE);
641        let list: Vec<Value> = vs
642            .iter()
643            .map(|v| {
644                Value::Object(vec![
645                    ("asof".into(), Value::String(v.asof.clone())),
646                    ("receipt".into(), Value::String(v.receipt_hash.clone())),
647                    ("tx".into(), Value::String(v.tx.clone())),
648                ])
649            })
650            .collect();
651        let mut body = vec![("cell".to_string(), target.cell.to_value()), ("versions".to_string(), Value::Array(list))];
652        if more {
653            if let Some(last) = vs.last() {
654                let us = datumesh_formula::calendar::parse_instant(&last.tx).unwrap_or(0) + 1;
655                let next_from = datumesh_formula::calendar::instant_text(us).unwrap_or_default();
656                let mut next = format!("{}/versions/{cell_rest}", self.config.origin());
657                next.push(if cell_rest.contains('?') { '&' } else { '?' });
658                next.push_str(&format!("from={next_from}"));
659                if let Some(t) = &to {
660                    next.push_str(&format!("&to={t}"));
661                }
662                body.push(("next".to_string(), Value::String(next)));
663            }
664        }
665        Reply::json(200, Value::Object(body).to_jcs(), Cache::Public(self.config.ttl.latest_max_age.as_secs()))
666    }
667}
668
669/// The members of a slice.
670#[utoipa::path(get, path = "/slice/{cell}",
671    params(
672        ("cell" = String, Path, description = "A slice: a cell URI's rest or a call with `*` for open arguments, of a stored function; `?asof=`, `limit=` (JSON, default 1,000, at most 10,000), `after=` (the cursor of the previous page)."),
673    ),
674    responses(
675        (status = 200, description = "JSON page of members, or with `Accept: application/vnd.apache.arrow.stream` an Arrow IPC stream", content_type = "application/json"),
676        (status = 400, description = "`bad_request`", content_type = "application/json"),
677        (status = 404, description = "`not_covered`", content_type = "application/json"),
678    ))]
679async fn slice(State(mesh): State<Mesh>, uri: Uri, headers: HeaderMap, req: Request) -> Response {
680    let inner = mesh.inner.clone();
681    let (client, _) = match who(&inner, &headers, peer_of(&req)) {
682        Ok(x) => x,
683        Err(r) => return to_response(r),
684    };
685    if let Err(r) = admit(&inner, &client, 1) {
686        return to_response(r);
687    }
688    // One manifest path since spec-0.4.2: /manifest/<slice id>/<members hash>.
689    let rest = rest_of(&uri, "/slice/");
690    let arrow = accepts(&headers, crate::slice::ARROW);
691    let r = inner
692        .pool
693        .clone()
694        .run(move || {
695            let now = inner.now();
696            inner.slice(&rest, &now, arrow)
697        })
698        .await
699        .unwrap_or_else(|| envelope::error_reply(ErrorCode::EvalUnavailable));
700    to_response(r)
701}
702
703fn accepts(headers: &HeaderMap, media: &str) -> bool {
704    headers
705        .get_all(header::ACCEPT)
706        .iter()
707        .filter_map(|v| v.to_str().ok())
708        .any(|v| v.split(',').any(|p| p.split(';').next().unwrap_or("").trim() == media))
709}
710
711/// Whether the client takes JSON (no `Accept`, `*/*`, `application/*` or
712/// `application/json`); for slices also Arrow.
713fn acceptable(headers: &HeaderMap, also: Option<&str>) -> bool {
714    let vals: Vec<&str> = headers.get_all(header::ACCEPT).iter().filter_map(|v| v.to_str().ok()).collect();
715    if vals.is_empty() {
716        return true;
717    }
718    vals.iter().flat_map(|v| v.split(',')).any(|p| {
719        let mut parts = p.split(';');
720        let m = parts.next().unwrap_or("").trim();
721        let q0 = parts.any(|x| {
722            let x = x.trim();
723            x == "q=0" || x == "q=0.0" || x == "q=0.00" || x == "q=0.000"
724        });
725        !q0 && (m == "*/*" || m == "application/*" || m == JSON || also == Some(m))
726    })
727}
728
729async fn object(
730    inner: Arc<Inner>,
731    what: &'static str,
732    f: impl FnOnce(&Inner) -> Option<Vec<u8>> + Send + 'static,
733) -> Response {
734    let i2 = inner.clone();
735    match inner.pool.run(move || f(&i2)).await.flatten() {
736        Some(bytes) => {
737            let ct = if what == "blob" { "application/octet-stream" } else { JSON };
738            to_response(Reply {
739                status: 200,
740                body: bytes.into(),
741                content_type: ct,
742                cache: Cache::Immutable,
743                canonical: None,
744                retry_after: None,
745            })
746        }
747        None => not_found(what),
748    }
749}
750
751/// The refusal of a request by its client's rate limit (or key), if it is refused.
752fn ip_admit(inner: &Inner, headers: &HeaderMap, req: &Request) -> Option<Response> {
753    let client = match who(inner, headers, peer_of(req)) {
754        Ok((c, _)) => c,
755        Err(r) => return Some(to_response(r)),
756    };
757    admit(inner, &client, 1).err().map(to_response)
758}
759
760/// A receipt by its hash (canonical bytes; gzip when asked).
761#[utoipa::path(get, path = "/receipt/{hash}", params(("hash" = String, Path, description = "The receipt hash (SPEC 8.3).")),
762    responses((status = 200, description = "The receipt's canonical JSON; immutable", content_type = "application/json"), (status = 404, description = "Not held")))]
763async fn receipt(State(mesh): State<Mesh>, Path(hash): Path<String>, headers: HeaderMap, req: Request) -> Response {
764    if let Some(r) = ip_admit(&mesh.inner, &headers, &req) {
765        return r;
766    }
767    if lexical::check_hex(&hash, 64, "$").is_err() {
768        return not_found("receipt");
769    }
770    object(mesh.inner.clone(), "receipt", move |i| i.store.receipt(&hash).map(|c| c.into_owned())).await
771}
772
773/// A descriptor by its function hash.
774#[utoipa::path(get, path = "/descriptor/{hash}", params(("hash" = String, Path, description = "The function hash.")),
775    responses((status = 200, description = "The descriptor's canonical JSON; immutable", content_type = "application/json"), (status = 404, description = "Not held")))]
776async fn descriptor(State(mesh): State<Mesh>, Path(hash): Path<String>, headers: HeaderMap, req: Request) -> Response {
777    if let Some(r) = ip_admit(&mesh.inner, &headers, &req) {
778        return r;
779    }
780    if lexical::check_hex(&hash, 64, "$").is_err() {
781        return not_found("descriptor");
782    }
783    object(mesh.inner.clone(), "descriptor", move |i| i.store.descriptor(&hash).map(|c| c.into_owned())).await
784}
785
786async fn manifest_of(inner: Arc<Inner>, entry: String) -> Response {
787    object(inner, "manifest", move |i| i.store.manifest(&entry).map(|c| c.into_owned())).await
788}
789
790/// A slice manifest (SPEC 15.13), by its entry's slice id and members hash.
791#[utoipa::path(get, path = "/manifest/{slice_id}/{members_hash}",
792    params(("slice_id" = String, Path, description = "The slice id."), ("members_hash" = String, Path, description = "The members hash.")),
793    responses((status = 200, description = "The manifest's canonical JSON; immutable", content_type = "application/json"), (status = 404, description = "Not held")))]
794async fn manifest(
795    State(mesh): State<Mesh>,
796    Path((slice_id, members)): Path<(String, String)>,
797    headers: HeaderMap,
798    req: Request,
799) -> Response {
800    if let Some(r) = ip_admit(&mesh.inner, &headers, &req) {
801        return r;
802    }
803    if lexical::check_hex(&slice_id, 64, "$").is_err() || lexical::check_hex(&members, 64, "$").is_err() {
804        return not_found("manifest");
805    }
806    manifest_of(mesh.inner.clone(), format!("slice:{slice_id}:{members}")).await
807}
808
809/// Blob content by SHA-256.
810#[utoipa::path(get, path = "/blob/{sha256}", params(("sha256" = String, Path, description = "The content's SHA-256.")),
811    responses((status = 200, description = "The bytes; immutable", content_type = "application/octet-stream"), (status = 404, description = "Not held")))]
812async fn blob(State(mesh): State<Mesh>, Path(sha): Path<String>, headers: HeaderMap, req: Request) -> Response {
813    if let Some(r) = ip_admit(&mesh.inner, &headers, &req) {
814        return r;
815    }
816    if lexical::check_hex(&sha, 64, "$").is_err() {
817        return not_found("blob");
818    }
819    object(mesh.inner.clone(), "blob", move |i| i.store.blob(&sha).ok().flatten().map(|c| c.into_owned())).await
820}
821
822/// A cell's lineage as nested envelopes.
823#[utoipa::path(get, path = "/explain/{cell}",
824    params(("cell" = String, Path, description = "As for /cell; `depth=` (default 4, at most 16) bounds the walk.")),
825    responses(
826        (status = 200, description = "`{\"envelope\", \"inputs\": [{\"entry\", \"envelope\", \"inputs\"} | {\"entry\", \"slice\", \"members\"}], \"truncated\"}`", content_type = "application/json"),
827        (status = 202, description = "`pending`: the cell is being computed; ask again", content_type = "application/json"),
828        (status = 400, description = "`bad_request`", content_type = "application/json"),
829        (status = 404, description = "`not_covered`", content_type = "application/json"),
830    ))]
831async fn explain(State(mesh): State<Mesh>, uri: Uri, headers: HeaderMap, req: Request) -> Response {
832    let inner = mesh.inner.clone();
833    let (client, principal) = match who(&inner, &headers, peer_of(&req)) {
834        Ok(x) => x,
835        Err(r) => return to_response(r),
836    };
837    if let Err(r) = admit(&inner, &client, 1) {
838        return to_response(r);
839    }
840    let mut rest = rest_of(&uri, "/explain/");
841    let mut depth = inner.config.explain_depth;
842    // `depth=` is the explain's own parameter: taken off the query.
843    if let Some((p, q)) = rest.clone().split_once('?') {
844        let kept: Vec<&str> = q.split('&').filter(|x| !x.starts_with("depth=")).collect();
845        if let Some(d) = q.split('&').find_map(|x| x.strip_prefix("depth=")) {
846            depth = d.parse::<usize>().unwrap_or(depth);
847        }
848        rest = if kept.is_empty() { p.to_string() } else { format!("{p}?{}", kept.join("&")) };
849    }
850    let depth = depth.min(16);
851    let r = within_budget(inner, move |i, on| {
852        let now = i.now();
853        let r = i.answer(&rest, principal.as_ref(), i, &now, on);
854        if r.status != 200 {
855            return r;
856        }
857        let env = json::parse(&r.body).unwrap_or(Value::Null);
858        let mut budget = 2_000usize;
859        let v = i.explain_node(env, depth, &mut budget);
860        Reply::json(200, v.to_jcs(), Cache::NoStore)
861    })
862    .await;
863    to_response(r)
864}
865
866impl Inner {
867    /// An envelope with its inputs' envelopes, `depth` levels down, at most `budget` nodes.
868    fn explain_node(&self, envelope: Value, depth: usize, budget: &mut usize) -> Value {
869        let inputs: Vec<String> = envelope
870            .get("receipt")
871            .and_then(|r| r.get("inputs"))
872            .and_then(|i| {
873                if let Value::Array(a) = i {
874                    Some(a.iter().filter_map(Value::as_str).map(str::to_string).collect())
875                } else {
876                    None
877                }
878            })
879            .unwrap_or_default();
880        let mut truncated = false;
881        let mut out = Vec::new();
882        for e in inputs {
883            if depth == 0 || *budget == 0 {
884                truncated = true;
885                break;
886            }
887            *budget -= 1;
888            if e.starts_with("slice:") {
889                let m = self.store.manifest(&e).and_then(|m| datumesh_core::Manifest::from_json(&m).ok());
890                let mut node = vec![("entry".to_string(), Value::String(e.clone()))];
891                if let Some(m) = m {
892                    node.push(("members".to_string(), envelope::num(m.members().len() as u64)));
893                    node.push(("slice".to_string(), m.slice().to_value()));
894                }
895                out.push(Value::Object(node));
896                continue;
897            }
898            let child = match self.receipt_info_by_hash(&e) {
899                Some(info) => self.explain_node(json::parse(&info.ok).unwrap_or(Value::Null), depth - 1, budget),
900                None => json::parse(&envelope::not_covered(None)).unwrap_or(Value::Null),
901            };
902            let mut node = vec![("entry".to_string(), Value::String(e.clone()))];
903            if let Value::Object(m) = child {
904                node.extend(m);
905            }
906            out.push(Value::Object(node));
907        }
908        let mut node = vec![("envelope".to_string(), envelope.clone()), ("inputs".to_string(), Value::Array(out))];
909        node.push(("truncated".to_string(), Value::Bool(truncated)));
910        Value::Object(node)
911    }
912}
913
914#[derive(serde::Deserialize)]
915struct FindQuery {
916    q: Option<String>,
917    limit: Option<usize>,
918}
919
920/// The whole of a `/find` match: every word of `words` is a substring of `hay` (lowercased).
921fn find_matches(words: &[String], hay: &str) -> bool {
922    words.iter().all(|w| hay.contains(w.as_str()))
923}
924
925/// A stored descriptor as JSON (`null` when it is not stored).
926fn descriptor_json(store: &dyn datumesh_store::Store, hash: &str) -> Value {
927    store.descriptor(hash).and_then(|d| json::parse(&d).ok()).unwrap_or(Value::Null)
928}
929
930/// The arguments and return of a class method's function (CONTRACT-1.7a H3): the arguments after
931/// the class's keys, each as its declaration without a unit that names another argument, and the
932/// return as declared.
933fn method_call_shape(store: &dyn datumesh_store::Store, fn_hash: &str, keys: &[String]) -> (Value, Value) {
934    let Some(bytes) = store.descriptor(fn_hash) else { return (Value::Null, Value::Null) };
935    let Ok(d) = datumesh_core::FunctionDescriptor::from_json(&bytes) else { return (Value::Null, Value::Null) };
936    let sig = d.signature.to_value();
937    let args = match sig.get("args") {
938        Some(Value::Object(m)) => Value::Object(
939            m.iter()
940                .filter(|(k, _)| !keys.contains(k))
941                .map(|(k, v)| {
942                    let v = match v {
943                        Value::Object(f) => Value::Object(
944                            f.iter()
945                                .filter(|(fk, fv)| fk != "unit" || matches!(fv, Value::String(_)))
946                                .cloned()
947                                .collect(),
948                        ),
949                        other => other.clone(),
950                    };
951                    (k.clone(), v)
952                })
953                .collect(),
954        ),
955        _ => Value::Object(vec![]),
956    };
957    (args, sig.get("returns").cloned().unwrap_or(Value::Null))
958}
959
960/// A class of the catalog as a `/find` result (CONTRACT-1.7a H3), when it matches: its name and
961/// description, its method names and the catalog descriptions of its methods.
962fn class_find(
963    inner: &crate::mesh::Inner,
964    name: &str,
965    desc: &str,
966    class: &datumesh_store::ClassBinding,
967    words: &[String],
968) -> Option<Value> {
969    let store: &dyn datumesh_store::Store = &*inner.store;
970    let catalog = &inner.config.catalog;
971    let bytes = store.class(&class.class_hash).ok().flatten()?;
972    let cd = datumesh_core::ClassDescriptor::from_json(&bytes).ok()?;
973    let keys: Vec<String> = cd.keys.keys().cloned().collect();
974    let mut hay = format!("{} {}", name.to_lowercase(), desc.to_lowercase());
975    let mut methods = Vec::new();
976    for (m, fh) in &cd.methods {
977        let full = format!("{name}/{m}");
978        let mdesc = catalog.get(&full);
979        hay.push_str(&format!(" {} {}", m.to_lowercase(), mdesc.map(|d| d.to_lowercase()).unwrap_or_default()));
980        let (args, returns) = method_call_shape(store, fh, &keys);
981        let mut obj = vec![
982            ("fn".into(), Value::String(full)),
983            ("fn_hash".into(), Value::String(fh.clone())),
984            ("args".into(), args),
985            ("returns".into(), returns),
986        ];
987        if let Some(d) = mdesc {
988            obj.push(("description".into(), Value::String(d.clone())));
989        }
990        methods.push((m.clone(), Value::Object(obj)));
991    }
992    if !find_matches(words, &hay) {
993        return None;
994    }
995    let descriptor = json::parse(&bytes).unwrap_or(Value::Null);
996    Some(Value::Object(vec![
997        ("kind".into(), Value::String("class".into())),
998        ("name".into(), Value::String(name.into())),
999        ("description".into(), Value::String(desc.into())),
1000        ("class_hash".into(), Value::String(class.class_hash.clone())),
1001        ("descriptor".into(), descriptor),
1002        ("methods".into(), Value::Object(methods)),
1003        ("since".into(), Value::String(class.tx.clone())),
1004    ]))
1005}
1006
1007/// Functions, classes and methods of the catalog by keyword (CONTRACT-1.7a H3): each entry the
1008/// catalog names that is bound as of now, with its `kind`. A catalog name that is not bound is
1009/// skipped.
1010#[utoipa::path(get, path = "/find",
1011    params(("q" = String, Query, description = "Keywords, all of which must appear (case-insensitive) in the name or the catalog description of an entry; a class is also matched by its methods' names and their catalog descriptions."), ("limit" = Option<usize>, Query, description = "At most this many results (default 20, at most 100).")),
1012    responses((status = 200, description = "`{\"query\", \"results\": [...]}`: each result has a `kind`. `function`: `{\"kind\", \"name\", \"description\", \"descriptor\", \"fn_hash\", \"since\"}`. `class`: `{\"kind\", \"name\", \"description\", \"class_hash\", \"descriptor\" (the flattened class descriptor), \"methods\": {<m>: {\"fn\": \"<name>/<m>\", \"fn_hash\", \"args\" (the arguments after the keys), \"returns\", \"description\"?}}, \"since\"}`. `method` (a catalog entry `<class>/<m>`): `{\"kind\", \"name\", \"description\", \"class\", \"fn_hash\", \"descriptor\", \"since\"}`.", content_type = "application/json")))]
1013async fn find(State(mesh): State<Mesh>, Query(q): Query<FindQuery>, headers: HeaderMap, req: Request) -> Response {
1014    if let Some(r) = ip_admit(&mesh.inner, &headers, &req) {
1015        return r;
1016    }
1017    let inner = mesh.inner.clone();
1018    let query = q.q.unwrap_or_default();
1019    let limit = q.limit.unwrap_or(20).clamp(1, 100);
1020    let r = inner
1021        .pool
1022        .clone()
1023        .run(move || {
1024            let words: Vec<String> = query.split_whitespace().map(str::to_lowercase).collect();
1025            let now = inner.now();
1026            let store: &dyn datumesh_store::Store = &*inner.store;
1027            let mut results = Vec::new();
1028            for (name, desc) in &inner.config.catalog {
1029                let entry = if let Ok(Some(cb)) = store.resolve_class_binding(name, &now) {
1030                    class_find(&inner, name, desc, &cb, &words)
1031                } else {
1032                    let parent = name.rsplit_once('/').map(|(p, _)| p).unwrap_or("");
1033                    // A method entry: `<class>/<m>` where the class is bound and has method `m`.
1034                    let is_method = match store.resolve_class_binding(parent, &now) {
1035                        Ok(Some(cb)) if !parent.is_empty() => {
1036                            let short = &name[parent.len() + 1..];
1037                            store
1038                                .class(&cb.class_hash)
1039                                .ok()
1040                                .flatten()
1041                                .and_then(|bytes| datumesh_core::ClassDescriptor::from_json(&bytes).ok())
1042                                .is_some_and(|cd| cd.methods.contains_key(short))
1043                        }
1044                        _ => false,
1045                    };
1046                    let hay = format!("{} {}", name.to_lowercase(), desc.to_lowercase());
1047                    if !find_matches(&words, &hay) {
1048                        None
1049                    } else if let Ok(Some(b)) = store.resolve_binding(name, &now) {
1050                        let descriptor = descriptor_json(store, &b.fn_hash);
1051                        let mut m = vec![
1052                            ("kind".into(), Value::String(if is_method { "method" } else { "function" }.into())),
1053                            ("description".into(), Value::String(desc.clone())),
1054                            ("descriptor".into(), descriptor),
1055                            ("fn_hash".into(), Value::String(b.fn_hash)),
1056                            ("name".into(), Value::String(name.clone())),
1057                            ("since".into(), Value::String(b.tx)),
1058                        ];
1059                        if is_method {
1060                            m.push(("class".into(), Value::String(parent.into())));
1061                        }
1062                        Some(Value::Object(m))
1063                    } else {
1064                        None
1065                    }
1066                };
1067                if let Some(e) = entry {
1068                    results.push(e);
1069                    if results.len() >= limit {
1070                        break;
1071                    }
1072                }
1073            }
1074            Value::Object(vec![("query".into(), Value::String(query)), ("results".into(), Value::Array(results))])
1075        })
1076        .await
1077        .unwrap_or(Value::Null);
1078    plain_json(200, r, Cache::Public(60))
1079}
1080
1081#[derive(serde::Deserialize)]
1082struct VerifyQuery {
1083    level: Option<String>,
1084    blob: Option<String>,
1085    form: Option<String>,
1086}
1087
1088/// Verification as a service: the core verifier on a submitted receipt, with this mesh's
1089/// keys; free, unauthenticated, rate-limited per IP.
1090#[utoipa::path(post, path = "/verify",
1091    params(("level" = Option<String>, Query, description = "`recomputed` (default: recompute with the engine as the resolver) or `signed` (steps 1-6)."), ("blob" = Option<String>, Query, description = "Blob content for step 7.7, as hex (at most 1 MiB)."), ("form" = Option<String>, Query, description = "`offered`: the body is `{\"receipt\": <its JSON text>, \"recompute\"?: <SPEC 11's recompute object>, \"blob_hex\"?}` and step 7 uses what is offered instead of this mesh's engine.")),
1092    request_body(content = String, description = "The receipt's JSON text, exactly as received (or the offered form).", content_type = "application/json"),
1093    responses(
1094        (status = 200, description = "`{\"ok\", \"level\", \"code\", \"hash\", \"computed_by\", \"path\", \"detail\", \"blob_checked\"}`", content_type = "application/json"),
1095        (status = 429, description = "`rate_limited`", content_type = "application/json"),
1096    ))]
1097async fn verify_receipt(
1098    State(mesh): State<Mesh>,
1099    Query(q): Query<VerifyQuery>,
1100    headers: HeaderMap,
1101    req: Request,
1102) -> Response {
1103    let inner = mesh.inner.clone();
1104    let ip = client_ip(&inner, &headers, peer_of(&req));
1105    if let Err(s) = inner.limiter.admit_verify(ip) {
1106        return to_response(envelope::rate_limited_reply(s));
1107    }
1108    let body: Bytes = match axum::body::to_bytes(req.into_body(), 4 << 20).await {
1109        Ok(b) => b,
1110        Err(_) => return batch_error("body too large"),
1111    };
1112    if q.form.as_deref() == Some("offered") {
1113        let Ok(v) = json::parse(&body) else { return batch_error("form=offered: not strict JSON") };
1114        let Some(receipt) = v.get("receipt").and_then(Value::as_str).map(|s| s.as_bytes().to_vec()) else {
1115            return batch_error("form=offered: no receipt text");
1116        };
1117        let blob = match v.get("blob_hex").and_then(Value::as_str).map(hex) {
1118            None => None,
1119            Some(Some(b)) if b.len() <= 1 << 20 => Some(b),
1120            Some(_) => return batch_error("blob_hex: hex of at most 1 MiB"),
1121        };
1122        let m = mesh.clone();
1123        let rc = v.get("recompute").cloned();
1124        let r = inner.pool.run(move || m.verify_offered(&receipt, rc.as_ref(), blob.as_deref())).await;
1125        return match r {
1126            Some(v) => to_response(Reply::json(200, v.body, Cache::NoStore)),
1127            None => to_response(envelope::error_reply(ErrorCode::EvalUnavailable)),
1128        };
1129    }
1130    let signed = q.level.as_deref() == Some("signed");
1131    let blob = match q.blob.as_deref().map(hex) {
1132        None => None,
1133        Some(Some(b)) if b.len() <= 1 << 20 => Some(b),
1134        Some(_) => return batch_error("blob: hex of at most 1 MiB"),
1135    };
1136    let m = mesh.clone();
1137    let r = inner.pool.run(move || m.verify(&body, signed, blob.as_deref())).await;
1138    match r {
1139        Some(v) => to_response(Reply::json(200, v.body, Cache::NoStore)),
1140        None => to_response(envelope::error_reply(ErrorCode::EvalUnavailable)),
1141    }
1142}
1143
1144fn hex(s: &str) -> Option<Vec<u8>> {
1145    if !s.len().is_multiple_of(2) {
1146        return None;
1147    }
1148    (0..s.len()).step_by(2).map(|i| u8::from_str_radix(s.get(i..i + 2)?, 16).ok()).collect()
1149}
1150
1151/// A job: `pending` again while it runs, then the envelope of the original request.
1152#[utoipa::path(get, path = "/jobs/{job}", params(("job" = String, Path, description = "The job id of a `pending` envelope.")),
1153    responses(
1154        (status = 202, description = "`pending`, the same job and request", content_type = "application/json"),
1155        (status = 400, description = "`bad_request` with `E_JOB_UNKNOWN`: unknown or expired", content_type = "application/json"),
1156        (status = 200, description = "The completed envelope (any outcome but pending, with its status), bound to the original request", content_type = "application/json"),
1157    ))]
1158async fn job(State(mesh): State<Mesh>, Path(id): Path<String>, headers: HeaderMap, req: Request) -> Response {
1159    let inner = mesh.inner.clone();
1160    let (client, _) = match who(&inner, &headers, peer_of(&req)) {
1161        Ok(x) => x,
1162        Err(r) => return to_response(r),
1163    };
1164    if let Err(r) = admit(&inner, &client, 1) {
1165        return to_response(r);
1166    }
1167    let unknown = || to_response(Reply::json(400, envelope::bad_request(ErrorCode::JobUnknown), Cache::NoStore));
1168    let valid = (32..=64).contains(&id.len()) && id.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b));
1169    if !valid {
1170        return unknown();
1171    }
1172    match inner.jobs.poll(&id) {
1173        Poll::Unknown => unknown(),
1174        Poll::Done(r) => to_response(r),
1175        Poll::Running(request, elapsed) => {
1176            let poll = format!("{}/jobs/{id}", inner.config.origin());
1177            match Pending::new(&id, inner.jobs.estimate(elapsed), &poll, request) {
1178                Ok(p) => to_response(Reply::json(202, envelope::pending(&p), Cache::NoStore)),
1179                Err(_) => to_response(envelope::error_reply(ErrorCode::EvalUnavailable)),
1180            }
1181        }
1182    }
1183}
1184
1185/// The key document (SPEC 9.2): the store's namespace record for this mesh's host.
1186#[utoipa::path(get, path = "/.well-known/datumesh.json", responses((status = 200, description = "The key document", content_type = "application/json"), (status = 404, description = "None stored")))]
1187async fn key_document(State(mesh): State<Mesh>) -> Response {
1188    let m = mesh.clone();
1189    match mesh.inner.pool.run(move || m.key_document()).await.flatten() {
1190        Some(doc) => to_response(Reply::json(200, doc, Cache::Public(300))),
1191        None => not_found("key document"),
1192    }
1193}
1194
1195/// The mesh document (SPEC 9.6).
1196#[utoipa::path(get, path = "/.well-known/datumesh-mesh.json", responses((status = 200, description = "The mesh document", content_type = "application/json")))]
1197async fn mesh_document(State(mesh): State<Mesh>) -> Response {
1198    to_response(Reply::json(200, mesh.mesh_document(), Cache::Public(300)))
1199}
1200
1201/// Which function each name denotes as of an instant (SPEC 9.7).
1202#[utoipa::path(get, path = "/bindings",
1203    params(("name" = String, Query, description = "A function name, percent-encoded as in SPEC 5.1; repeat for several."), ("asof" = Option<String>, Query, description = "An instant (default: now).")),
1204    responses((status = 200, description = "`{\"asof\", \"bindings\": {name: {\"fn_hash\", \"since\"}}}`", content_type = "application/json"), (status = 400, description = "A malformed name or asof", content_type = "application/json")))]
1205async fn bindings(State(mesh): State<Mesh>, uri: Uri, headers: HeaderMap, req: Request) -> Response {
1206    if let Some(r) = ip_admit(&mesh.inner, &headers, &req) {
1207        return r;
1208    }
1209    let inner = mesh.inner.clone();
1210    let mut names = Vec::new();
1211    let mut asof = None;
1212    for p in uri.query().unwrap_or("").split('&').filter(|p| !p.is_empty()) {
1213        let (k, v) = p.split_once('=').unwrap_or((p, ""));
1214        let Some(v) = pct_decode(v) else { return batch_error("percent-encoding (SPEC 5.1)") };
1215        match k {
1216            "name" if lexical::check_function_name(&v, "$").is_ok() => names.push(v),
1217            "asof" if lexical::check_instant(&v, "$").is_ok() && asof.is_none() => asof = Some(v),
1218            _ => return batch_error(&format!("parameter {k}")),
1219        }
1220    }
1221    let r = inner
1222        .pool
1223        .clone()
1224        .run(move || {
1225            let at = asof.unwrap_or_else(|| inner.now());
1226            let mut out = Vec::new();
1227            for n in names {
1228                if !inner.config.owns(crate::foreign::host_of(&n)) {
1229                    continue;
1230                }
1231                // A class name is answered with its class hash (9.7, spec-0.5.0); any other name
1232                // with the function it means.
1233                let found = match inner.store.resolve_class_binding(&n, &at) {
1234                    Ok(Some(c)) => ("class_hash", c.class_hash, c.tx),
1235                    Ok(None) => match inner.store.resolve_binding(&n, &at) {
1236                        Ok(Some(b)) => ("fn_hash", b.fn_hash, b.tx),
1237                        Ok(None) => continue,
1238                        Err(_) => return None,
1239                    },
1240                    Err(_) => return None,
1241                };
1242                out.push((
1243                    n,
1244                    Value::Object(vec![
1245                        (found.0.into(), Value::String(found.1)),
1246                        ("since".into(), Value::String(found.2)),
1247                    ]),
1248                ));
1249            }
1250            Some(Value::Object(vec![("asof".into(), Value::String(at)), ("bindings".into(), Value::Object(out))]))
1251        })
1252        .await
1253        .flatten();
1254    match r {
1255        Some(v) => plain_json(200, v, Cache::Public(inner_latest(&mesh))),
1256        None => to_response(envelope::error_reply(ErrorCode::InputUnavailable)),
1257    }
1258}
1259
1260fn inner_latest(mesh: &Mesh) -> u64 {
1261    mesh.inner.config.ttl.latest_max_age.as_secs()
1262}
1263
1264/// Strict percent-decoding (SPEC 5.1): literal characters as themselves, `%` and two
1265/// uppercase hex digits for other bytes, UTF-8.
1266fn pct_decode(s: &str) -> Option<String> {
1267    let b = s.as_bytes();
1268    let mut out = Vec::new();
1269    let mut i = 0;
1270    while i < b.len() {
1271        let c = b[i];
1272        if c == b'%' {
1273            let h = b.get(i + 1..i + 3)?;
1274            if !h.iter().all(|x| x.is_ascii_digit() || (b'A'..=b'F').contains(x)) {
1275                return None;
1276            }
1277            let v = u8::from_str_radix(std::str::from_utf8(h).ok()?, 16).ok()?;
1278            if v.is_ascii_alphanumeric() || matches!(v, b'-' | b'.' | b'_' | b'~' | b':') {
1279                return None;
1280            }
1281            out.push(v);
1282            i += 3;
1283        } else if c.is_ascii_alphanumeric() || matches!(c, b'-' | b'.' | b'_' | b'~' | b':') {
1284            out.push(c);
1285            i += 1;
1286        } else {
1287            return None;
1288        }
1289    }
1290    String::from_utf8(out).ok()
1291}
1292
1293/// Liveness and the store's health.
1294#[utoipa::path(get, path = "/health", responses((status = 200, description = "`{\"status\": \"ok\"}`", content_type = "application/json"), (status = 503, description = "The store does not answer", content_type = "application/json")))]
1295async fn health(State(mesh): State<Mesh>) -> Response {
1296    let m = mesh.clone();
1297    let ok = tokio::time::timeout(
1298        Duration::from_secs(5),
1299        mesh.inner.pool.run(move || m.inner.store.namespace(&m.inner.config.host).is_ok()),
1300    )
1301    .await
1302    .ok()
1303    .flatten()
1304    .unwrap_or(false);
1305    let v = Value::Object(vec![("status".into(), Value::String(if ok { "ok" } else { "store unavailable" }.into()))]);
1306    plain_json(if ok { 200 } else { 503 }, v, Cache::NoStore)
1307}
1308
1309/// Prometheus metrics.
1310#[utoipa::path(get, path = "/metrics", responses((status = 200, description = "Prometheus text", content_type = "text/plain")))]
1311async fn metrics(State(mesh): State<Mesh>) -> Response {
1312    let mut r = Reply::json(200, metrics_text(&mesh.inner).into_bytes(), Cache::NoStore);
1313    r.content_type = "text/plain; version=0.0.4";
1314    to_response(r)
1315}
1316
1317/// This document.
1318#[utoipa::path(get, path = "/openapi.json", responses((status = 200, description = "The OpenAPI document", content_type = "application/json")))]
1319async fn openapi_json() -> Response {
1320    let mut r = Reply::json(200, openapi().to_json().unwrap_or_default().into_bytes(), Cache::Public(300));
1321    r.content_type = JSON;
1322    to_response(r)
1323}
1324
1325pub(crate) fn metrics_text(inner: &Inner) -> String {
1326    use std::fmt::Write;
1327    use std::sync::atomic::Ordering::Relaxed;
1328    let mut out = String::new();
1329    inner.metrics.write(&mut out);
1330    let p = &inner.pool;
1331    let g = |out: &mut String, name: &str, help: &str, kind: &str, v: String| {
1332        let _ = writeln!(out, "# HELP {name} {help}\n# TYPE {name} {kind}\n{name} {v}");
1333    };
1334    g(&mut out, "datumesh_pool_size", "The blocking pool's size: the concurrency limit.", "gauge", p.size.to_string());
1335    g(&mut out, "datumesh_pool_in_use", "Pool threads busy now.", "gauge", p.in_use.load(Relaxed).to_string());
1336    g(&mut out, "datumesh_pool_peak", "The most pool threads busy at once.", "gauge", p.peak.load(Relaxed).to_string());
1337    g(&mut out, "datumesh_pool_runs_total", "Tasks run on the pool.", "counter", p.runs.load(Relaxed).to_string());
1338    g(
1339        &mut out,
1340        "datumesh_pool_waited_total",
1341        "Tasks that found the pool saturated and waited for a thread.",
1342        "counter",
1343        p.waited.load(Relaxed).to_string(),
1344    );
1345    let (held, running) = inner.jobs.counts();
1346    g(&mut out, "datumesh_jobs_held", "Jobs held (running or done).", "gauge", held.to_string());
1347    g(&mut out, "datumesh_jobs_running", "Jobs running.", "gauge", running.to_string());
1348    g(
1349        &mut out,
1350        "datumesh_jobs_created_total",
1351        "Requests answered pending.",
1352        "counter",
1353        inner.jobs.created.load(Relaxed).to_string(),
1354    );
1355    g(
1356        &mut out,
1357        "datumesh_rate_limited_total",
1358        "Requests refused by a rate limit.",
1359        "counter",
1360        inner.limiter.refused.load(Relaxed).to_string(),
1361    );
1362    g(
1363        &mut out,
1364        "datumesh_peer_fetches_total",
1365        "Requests to other meshes.",
1366        "counter",
1367        inner.foreign.fetches.load(Relaxed).to_string(),
1368    );
1369    g(
1370        &mut out,
1371        "datumesh_peer_failures_total",
1372        "Requests to other meshes that failed.",
1373        "counter",
1374        inner.foreign.failures.load(Relaxed).to_string(),
1375    );
1376    g(
1377        &mut out,
1378        "datumesh_peer_imported_total",
1379        "Foreign versions imported into the store.",
1380        "counter",
1381        inner.foreign.imported.load(Relaxed).to_string(),
1382    );
1383    let m = inner.engine.memo_stats();
1384    g(&mut out, "datumesh_memo_entries", "Engine memo entries.", "gauge", m.entries.to_string());
1385    g(&mut out, "datumesh_memo_bytes", "Engine memo size, approximately.", "gauge", m.approx_bytes.to_string());
1386    g(
1387        &mut out,
1388        "datumesh_uptime_seconds",
1389        "Seconds since the mesh started.",
1390        "gauge",
1391        inner.started.elapsed().as_secs().to_string(),
1392    );
1393    out
1394}
1395
1396/// Request ids, the access log, metrics and `Accept`.
1397async fn around(State(mesh): State<Mesh>, req: Request, next: Next) -> Response {
1398    let start = Instant::now();
1399    let route = req.extensions().get::<MatchedPath>().map(|m| m.as_str().to_string()).unwrap_or_else(|| "other".into());
1400    let route = match route.as_str() {
1401        "/cell/{*rest}" => "/cell/{cell}".to_string(),
1402        "/slice/{*rest}" => "/slice/{cell}".to_string(),
1403        "/explain/{*rest}" => "/explain/{cell}".to_string(),
1404        "/versions/{*rest}" => "/versions/{cell}".to_string(),
1405        _ => route,
1406    };
1407    let id = req
1408        .headers()
1409        .get("x-request-id")
1410        .and_then(|v| v.to_str().ok())
1411        .filter(|s| !s.is_empty() && s.len() <= 64 && s.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-'))
1412        .map(str::to_string)
1413        .unwrap_or_else(|| {
1414            let mut b = [0u8; 8];
1415            let _ = getrandom::fill(&mut b);
1416            lexical::to_hex(&b)
1417        });
1418    let ip = client_ip(&mesh.inner, req.headers(), peer_of(&req));
1419    let method = req.method().clone();
1420    let also = if route == "/slice/{cell}" { Some(crate::slice::ARROW) } else { None };
1421    let mut resp = if route != "/blob/{sha256}" && route != "/metrics" && !acceptable(req.headers(), also) {
1422        let mut r = Reply::json(
1423            406,
1424            Value::Object(vec![("error".into(), Value::String("not acceptable: application/json only".into()))])
1425                .to_jcs(),
1426            Cache::NoStore,
1427        );
1428        r.content_type = JSON;
1429        to_response(r)
1430    } else {
1431        next.run(req).instrument(tracing::info_span!("request", request_id = %id)).await
1432    };
1433    let elapsed = start.elapsed();
1434    let status = resp.status().as_u16();
1435    mesh.inner.metrics.record(&route, status, elapsed);
1436    let principal = resp.extensions().get::<Who>().and_then(|w| w.0.clone());
1437    tracing::info!(target: "datumesh_http::access", request_id = %id, %method, route = %route, status, ms = elapsed.as_secs_f64() * 1000.0, ip = %ip, principal = principal.as_deref().unwrap_or("-"), "request");
1438    if let Ok(v) = HeaderValue::from_str(&id) {
1439        resp.headers_mut().insert("x-request-id", v);
1440    }
1441    resp.headers_mut().insert(header::VARY, HeaderValue::from_static("Accept, Accept-Encoding, Authorization"));
1442    if let Ok(v) = HeaderValue::from_str(&format!("app;dur={:.3}", elapsed.as_secs_f64() * 1000.0)) {
1443        resp.headers_mut().insert("server-timing", v);
1444    }
1445    resp
1446}
1447
1448/// describe (SPEC 16.8): a class or an object, by class name, key encoding, object id, cell id or
1449/// short id (`{id}`, percent-encoded as a path).
1450#[utoipa::path(get, path = "/describe/{id}",
1451    params(("id" = String, Path, description = "A class name, an object's key encoding, an object id, a cell id or a short id (12 hex digits).")),
1452    responses((status = 200, description = "A describe-class or describe-object document (SPEC 16.8)", content_type = "application/json"), (status = 400, description = "`E_KEY_ENCODING`, `E_KEY_NORMALISATION` or `E_SHORT_ID_AMBIGUOUS`", content_type = "application/json"), (status = 404, description = "An id the mesh does not know", content_type = "application/json")))]
1453async fn describe(State(mesh): State<Mesh>, uri: Uri, headers: HeaderMap, req: Request) -> Response {
1454    if let Some(r) = ip_admit(&mesh.inner, &headers, &req) {
1455        return r;
1456    }
1457    let Some(id) = crate::call::url_decode(uri.path().strip_prefix("/describe/").unwrap_or("")) else {
1458        return to_response(envelope::bad_request_reply(ErrorCode::CanonPercent));
1459    };
1460    let inner = mesh.inner.clone();
1461    match inner.pool.clone().run(move || mesh.describe(&id)).await {
1462        Some(r) => to_response(r),
1463        None => to_response(envelope::error_reply(ErrorCode::EvalUnavailable)),
1464    }
1465}
1466
1467async fn fallback() -> Response {
1468    not_found("endpoint")
1469}
1470
1471impl Mesh {
1472    /// The router of every endpoint. Serve it with
1473    /// `into_make_service_with_connect_info::<SocketAddr>()` (the client IP comes from it).
1474    pub fn router(&self) -> Router {
1475        Router::new()
1476            .route("/cell/{*rest}", get(cell))
1477            .route("/cells", post(cells))
1478            .route("/slice/{*rest}", get(slice))
1479            .route("/receipt/{hash}", get(receipt))
1480            .route("/descriptor/{hash}", get(descriptor))
1481            .route("/manifest/{slice_id}/{members_hash}", get(manifest))
1482            .route("/versions/{*rest}", get(versions))
1483            .route("/blob/{sha256}", get(blob))
1484            .route("/explain/{*rest}", get(explain))
1485            .route("/find", get(find))
1486            .route("/verify", post(verify_receipt))
1487            .route("/jobs/{job}", get(job))
1488            .route("/.well-known/datumesh.json", get(key_document))
1489            .route("/.well-known/datumesh-mesh.json", get(mesh_document))
1490            .route("/bindings", get(bindings))
1491            .route("/describe/{*id}", get(describe))
1492            .route("/health", get(health))
1493            .route("/metrics", get(metrics))
1494            .route("/openapi.json", get(openapi_json))
1495            .fallback(fallback)
1496            .layer(middleware::from_fn_with_state(self.clone(), around))
1497            .layer(tower_http::compression::CompressionLayer::new().gzip(true))
1498            .with_state(self.clone())
1499    }
1500
1501    /// Serves the router on `listener` until `shutdown` resolves.
1502    pub async fn serve(
1503        &self,
1504        listener: tokio::net::TcpListener,
1505        shutdown: impl std::future::Future<Output = ()> + Send + 'static,
1506    ) -> std::io::Result<()> {
1507        axum::serve(listener, self.router().into_make_service_with_connect_info::<SocketAddr>())
1508            .with_graceful_shutdown(shutdown)
1509            .await
1510    }
1511}
1512
1513impl IntoResponse for Reply {
1514    fn into_response(self) -> Response {
1515        to_response(self)
1516    }
1517}
1518
1519#[cfg(test)]
1520mod tests {
1521    use super::*;
1522
1523    fn reply(status: u16, body: &str) -> Reply {
1524        Reply::json(status, body.as_bytes().to_vec(), Cache::NoStore)
1525    }
1526
1527    /// The runs the chunks write, concatenated and closed, are the batch as 5.7 writes it, and
1528    /// count its outcomes.
1529    #[test]
1530    fn runs_make_the_batch() {
1531        let h = "a".repeat(64);
1532        let replies = [
1533            reply(200, &format!(r#"{{"fn_hash":"{h}","outcome":"ok","receipt":{{"value":"1"}},"trust":"primary"}}"#)),
1534            reply(400, r#"{"error":"E_CANON_NUMBER","outcome":"bad_request"}"#),
1535            reply(
1536                200,
1537                &format!(r#"{{"age_seconds":3,"fn_hash":"{h}","outcome":"stale","receipt":{{}},"trust":"computed"}}"#),
1538            ),
1539            reply(404, r#"{"error":"E_NAME_UNBOUND","outcome":"not_covered"}"#),
1540            reply(200, &format!(r#"{{"fn_hash":"{h}","outcome":"ok","receipt":{{"value":"2"}},"trust":"primary"}}"#)),
1541        ];
1542        let mut want = b"[".to_vec();
1543        for (i, r) in replies.iter().enumerate() {
1544            if i > 0 {
1545                want.push(b',');
1546            }
1547            want.extend_from_slice(b"{\"envelope\":");
1548            want.extend_from_slice(&r.body);
1549            want.extend_from_slice(format!(",\"status\":{}}}", r.status).as_bytes());
1550        }
1551        want.push(b']');
1552        for split in 1..replies.len() {
1553            let runs = [Written::of(true, replies[..split].iter()), Written::of(false, replies[split..].iter())];
1554            let mut got: Vec<u8> = runs.iter().flat_map(|w| w.bytes.iter().copied()).collect();
1555            got.push(b']');
1556            assert_eq!(String::from_utf8(got).unwrap(), String::from_utf8(want.clone()).unwrap());
1557            let mut counts = std::collections::BTreeMap::new();
1558            for (o, k) in runs.iter().flat_map(|w| w.outcomes.iter()) {
1559                *counts.entry(o.as_str()).or_insert(0) += k;
1560            }
1561            let want_counts: std::collections::BTreeMap<&str, u64> =
1562                [("bad_request", 1), ("not_covered", 1), ("ok", 2), ("stale", 1)].into_iter().collect();
1563            assert_eq!(counts, want_counts);
1564        }
1565        for (n, s) in [(0u16, "0"), (7, "7"), (200, "200"), (404, "404"), (65535, "65535")] {
1566            let mut out = Vec::new();
1567            push_u16(&mut out, n);
1568            assert_eq!(out, s.as_bytes());
1569        }
1570    }
1571
1572    /// The outcome of each envelope, whatever members come before it.
1573    #[test]
1574    fn outcomes() {
1575        let h = "0".repeat(64);
1576        for (body, want) in [
1577            (format!(r#"{{"fn_hash":"{h}","outcome":"ok","receipt":{{"outcome":"x"}}}}"#), Some("ok")),
1578            (r#"{"age_seconds":12,"outcome":"stale"}"#.to_string(), Some("stale")),
1579            (r#"{"error":"E_PARSE_URI","outcome":"bad_request"}"#.to_string(), Some("bad_request")),
1580            (r#"{"outcome":"pending","poll":"https://a.example/jobs/1"}"#.to_string(), Some("pending")),
1581            (r#"{"error":"no such endpoint"}"#.to_string(), None),
1582            (r#"{"outcome""#.to_string(), None),
1583        ] {
1584            assert_eq!(reply(200, &body).outcome_str(), want, "{body}");
1585            assert_eq!(reply(200, &body).outcome(), want.map(str::to_string), "{body}");
1586        }
1587        let mut arrow = reply(200, r#"{"outcome":"ok"}"#);
1588        arrow.content_type = "application/vnd.apache.arrow.stream";
1589        assert_eq!(arrow.outcome(), None);
1590    }
1591}