Skip to main content

datumesh_http/
http.rs

1//! The endpoints: an axum router over a [`Mesh`].
2
3use std::net::{IpAddr, Ipv4Addr, SocketAddr};
4use std::sync::{Arc, Mutex};
5use std::time::{Duration, Instant};
6
7use axum::body::{Body, Bytes};
8use axum::extract::{ConnectInfo, MatchedPath, Path, Query, Request, State};
9use axum::http::{header, HeaderMap, HeaderValue, StatusCode, Uri};
10use axum::middleware::{self, Next};
11use axum::response::{IntoResponse, Response};
12use axum::routing::{get, post};
13use axum::Router;
14use datumesh_core::json::{self, Value};
15use datumesh_core::{lexical, ErrorCode, Pending, Receipt, Request as CoreRequest};
16use tracing::Instrument;
17use utoipa::OpenApi;
18
19use crate::apikeys::Principal;
20use crate::envelope::{self, Cache, Reply, JSON};
21use crate::jobs::Poll;
22use crate::limit::Client;
23use crate::mesh::{Inner, Memo, Mesh};
24
25/// The principal of a request, for the access log.
26#[derive(Clone)]
27struct Who(Option<String>);
28
29fn to_response(r: Reply) -> Response {
30    let mut resp = Response::new(Body::from(r.body));
31    *resp.status_mut() = StatusCode::from_u16(r.status).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
32    let h = resp.headers_mut();
33    h.insert(header::CONTENT_TYPE, HeaderValue::from_static(r.content_type));
34    if let Ok(v) = HeaderValue::from_str(&r.cache.header()) {
35        h.insert(header::CACHE_CONTROL, v);
36    }
37    if let Some(c) = r.canonical {
38        if let Ok(v) = HeaderValue::from_str(&format!("<{c}>; rel=\"canonical\"")) {
39            h.insert(header::LINK, v);
40        }
41    }
42    if let Some(s) = r.retry_after {
43        h.insert(header::RETRY_AFTER, HeaderValue::from(s));
44    }
45    resp
46}
47
48fn plain_json(status: u16, v: Value, cache: Cache) -> Response {
49    to_response(Reply::json(status, v.to_jcs(), cache))
50}
51
52fn not_found(what: &str) -> Response {
53    plain_json(404, Value::Object(vec![("error".into(), Value::String(format!("no such {what}")))]), Cache::Public(60))
54}
55
56/// The client: its IP and, with a valid API key, its principal. An `Authorization` header
57/// that is not a key of this mesh is answered `unlicensed`.
58fn who(inner: &Inner, headers: &HeaderMap, peer: IpAddr) -> Result<(Client, Option<Principal>), Reply> {
59    let ip = client_ip(inner, headers, peer);
60    match headers.get(header::AUTHORIZATION) {
61        None => Ok((Client::Ip(ip), None)),
62        Some(v) => {
63            let p = v
64                .to_str()
65                .ok()
66                .and_then(|s| s.strip_prefix("Bearer "))
67                .and_then(|k| inner.config.api_keys.principal(k.trim()));
68            match p {
69                Some(p) => Ok((Client::Key(p.id.clone(), p.per_minute), Some(p.clone()))),
70                None => Err(Reply::json(403, envelope::unlicensed(), Cache::Private)),
71            }
72        }
73    }
74}
75
76fn client_ip(inner: &Inner, headers: &HeaderMap, peer: IpAddr) -> IpAddr {
77    if inner.config.trust_proxy {
78        if let Some(ip) = headers
79            .get("x-forwarded-for")
80            .and_then(|v| v.to_str().ok())
81            .and_then(|s| s.rsplit(',').next())
82            .and_then(|s| s.trim().parse::<IpAddr>().ok())
83        {
84            return ip;
85        }
86    }
87    peer
88}
89
90fn peer_of(req: &Request) -> IpAddr {
91    req.extensions().get::<ConnectInfo<SocketAddr>>().map(|c| c.0.ip()).unwrap_or(IpAddr::V4(Ipv4Addr::UNSPECIFIED))
92}
93
94/// `rest` of a `/<prefix>/<rest>` request, query included, as received.
95fn rest_of(uri: &Uri, prefix: &str) -> String {
96    let path = uri.path().strip_prefix(prefix).unwrap_or("");
97    match uri.query() {
98        Some(q) => format!("{path}?{q}"),
99        None => path.to_string(),
100    }
101}
102
103/// Runs `f` on the pool within the request's budget: its reply, or, past the budget, a
104/// `pending` envelope whose job gets the reply when `f` finishes. `f` reports the request
105/// description as soon as it knows it.
106async fn within_budget(
107    inner: Arc<Inner>,
108    f: impl FnOnce(&Inner, &mut dyn FnMut(&crate::call::Target)) -> Reply + Send + 'static,
109) -> Reply {
110    let deadline = tokio::time::Instant::now() + inner.config.budget;
111    let (tx, mut rx) = tokio::sync::watch::channel::<Option<CoreRequest>>(None);
112    let pool = inner.pool.clone();
113    let i2 = inner.clone();
114    let mut task = tokio::spawn(async move {
115        pool.run(move || {
116            let mut on = |t: &crate::call::Target| {
117                if let Ok(r) = CoreRequest::new(t.cell_id(), t.asof()) {
118                    let _ = tx.send(Some(r));
119                }
120            };
121            f(&i2, &mut on)
122        })
123        .await
124    });
125    match tokio::time::timeout_at(deadline, &mut task).await {
126        Ok(Ok(Some(r))) => r,
127        Ok(_) => envelope::error_reply(ErrorCode::EvalUnavailable),
128        Err(_) => {
129            let request = rx.borrow_and_update().clone();
130            let Some(request) = request else {
131                // Past the budget before the request was even resolved (the store or another
132                // mesh is slow): the mesh cannot say what it is computing.
133                return envelope::error_reply(ErrorCode::EvalUnavailable);
134            };
135            pend(inner, request, task)
136        }
137    }
138}
139
140/// A job for a request still running in `task`.
141fn pend(inner: Arc<Inner>, request: CoreRequest, task: tokio::task::JoinHandle<Option<Reply>>) -> Reply {
142    let Some(id) = inner.jobs.create(request.clone()) else {
143        return envelope::error_reply(ErrorCode::EvalUnavailable);
144    };
145    let poll = format!("{}/jobs/{id}", inner.config.origin());
146    let est = inner.jobs.estimate(inner.config.budget.as_secs());
147    let i2 = inner.clone();
148    let id2 = id.clone();
149    tokio::spawn(async move {
150        let r = match task.await {
151            Ok(Some(r)) if r.status != 202 => r,
152            _ => envelope::error_reply(ErrorCode::EvalUnavailable),
153        };
154        let mut r = r;
155        r.cache = Cache::NoStore;
156        r.canonical = None;
157        i2.jobs.complete(&id2, r);
158    });
159    match Pending::new(&id, est, &poll, request) {
160        Ok(p) => Reply::json(202, envelope::pending(&p), Cache::NoStore),
161        Err(_) => envelope::error_reply(ErrorCode::EvalUnavailable),
162    }
163}
164
165fn admit(inner: &Inner, client: &Client, cost: u32) -> Result<(), Reply> {
166    inner.limiter.admit(client, cost).map_err(envelope::rate_limited_reply)
167}
168
169#[derive(OpenApi)]
170#[openapi(
171    info(
172        title = "datumesh-mesh",
173        description = "The Datumesh engine behind HTTPS (spec-0.4.1): cell envelopes (SPEC 5.5), batches, slices, receipts, descriptors, manifests, blobs, lineage, search, verification, jobs, key and mesh documents (SPEC 9.2, 9.6), bindings (SPEC 9.7).",
174        license(name = "Apache-2.0")
175    ),
176    paths(
177        cell,
178        cells,
179        slice,
180        receipt,
181        descriptor,
182        manifest,
183        blob,
184        explain,
185        find,
186        verify_receipt,
187        job,
188        key_document,
189        mesh_document,
190        bindings,
191        health,
192        metrics,
193        openapi_json
194    )
195)]
196struct ApiDoc;
197
198/// The OpenAPI document (generated from the handlers' annotations).
199pub fn openapi() -> utoipa::openapi::OpenApi {
200    ApiDoc::openapi()
201}
202
203/// The envelope of a cell.
204#[utoipa::path(get, path = "/cell/{cell}",
205    params(("cell" = String, Path, description = "A cell URI's rest (SPEC 5.3), byte for byte, `asof` included; or a call (SPEC 15.3's read syntax, URL-encoded) with an optional `?asof=`.")),
206    responses(
207        (status = 200, description = "`ok` or `stale`", content_type = "application/json"),
208        (status = 202, description = "`pending`: poll the job (SPEC 5.5)", content_type = "application/json"),
209        (status = 400, description = "`bad_request`", content_type = "application/json"),
210        (status = 402, description = "`payment_required`", content_type = "application/json"),
211        (status = 403, description = "`unlicensed`", content_type = "application/json"),
212        (status = 404, description = "`not_covered` (with `fault` when the evaluation faulted)", content_type = "application/json"),
213        (status = 406, description = "the Accept header excludes JSON"),
214        (status = 429, description = "`rate_limited`, with `Retry-After`", content_type = "application/json"),
215        (status = 500, description = "`error`", content_type = "application/json"),
216        (status = 503, description = "`error`: the store or another mesh is unavailable", content_type = "application/json"),
217    ))]
218async fn cell(State(mesh): State<Mesh>, uri: Uri, headers: HeaderMap, req: Request) -> Response {
219    let inner = mesh.inner.clone();
220    let (client, principal) = match who(&inner, &headers, peer_of(&req)) {
221        Ok(x) => x,
222        Err(r) => return to_response(r),
223    };
224    let mut resp = match admit(&inner, &client, 1) {
225        Err(r) => to_response(r),
226        Ok(()) => {
227            let rest = rest_of(&uri, "/cell/");
228            let p = principal.clone();
229            let r = within_budget(inner.clone(), move |i, on| {
230                let now = i.now();
231                i.answer(&rest, p.as_ref(), i, &now, on)
232            })
233            .await;
234            if let Some(o) = r.outcome() {
235                inner.metrics.outcome("/cell/{cell}", &o);
236            }
237            to_response(r)
238        }
239    };
240    resp.extensions_mut().insert(Who(principal.map(|p| p.id)));
241    resp
242}
243
244/// A batch of cells.
245#[utoipa::path(post, path = "/cells",
246    request_body(content = String, description = "A JSON array of requests, each a call or a cell URI (`cell://...`) as a string, or `{\"cell\": <that>}`; at most the configured bound (1,000).", content_type = "application/json"),
247    responses(
248        (status = 200, description = "A JSON array of envelopes, in order; each has the HTTP status its outcome has (`error`: 500)", content_type = "application/json"),
249        (status = 400, description = "Not such an array, or too long", content_type = "application/json"),
250        (status = 429, description = "`rate_limited` (a batch of n cells counts as n / 50 requests)", content_type = "application/json"),
251    ))]
252async fn cells(State(mesh): State<Mesh>, headers: HeaderMap, req: Request) -> Response {
253    let inner = mesh.inner.clone();
254    let peer = peer_of(&req);
255    let (client, principal) = match who(&inner, &headers, peer) {
256        Ok(x) => x,
257        Err(r) => return to_response(r),
258    };
259    let body = match axum::body::to_bytes(req.into_body(), 64 << 20).await {
260        Ok(b) => b,
261        Err(_) => return batch_error("body too large"),
262    };
263    let items = match parse_batch(&body, inner.config.batch_max) {
264        Ok(i) => i,
265        Err(e) => return batch_error(&e),
266    };
267    if let Err(r) = admit(&inner, &client, inner.limiter.batch_cost(items.len())) {
268        return to_response(r);
269    }
270    let replies = batch(inner.clone(), items, principal.clone()).await;
271    let mut counts: std::collections::BTreeMap<String, u64> = std::collections::BTreeMap::new();
272    for r in &replies {
273        if let Some(o) = r.outcome() {
274            *counts.entry(o).or_default() += 1;
275        }
276    }
277    for (o, k) in counts {
278        inner.metrics.outcomes("/cells", &o, k);
279    }
280    let mut out = Vec::with_capacity(replies.iter().map(|r| r.body.len() + 1).sum::<usize>() + 2);
281    out.push(b'[');
282    for (i, r) in replies.iter().enumerate() {
283        if i > 0 {
284            out.push(b',');
285        }
286        out.extend_from_slice(&r.body);
287    }
288    out.push(b']');
289    let mut resp =
290        to_response(Reply::json(200, out, if principal.is_some() { Cache::Private } else { Cache::NoStore }));
291    resp.extensions_mut().insert(Who(principal.map(|p| p.id)));
292    resp
293}
294
295fn batch_error(e: &str) -> Response {
296    plain_json(400, Value::Object(vec![("error".into(), Value::String(e.to_string()))]), Cache::NoStore)
297}
298
299fn parse_batch(body: &[u8], max: usize) -> Result<Vec<String>, String> {
300    // The body is a list of requests, not a protocol object: plain JSON.
301    let v: serde_json::Value = serde_json::from_slice(body).map_err(|e| format!("not JSON: {e}"))?;
302    let serde_json::Value::Array(items) = v else { return Err("not a JSON array".into()) };
303    if items.len() > max {
304        return Err(format!("{} requests: at most {max}", items.len()));
305    }
306    items
307        .into_iter()
308        .map(|i| {
309            let s = match i {
310                serde_json::Value::String(s) => s,
311                serde_json::Value::Object(mut o) => match o.remove("cell") {
312                    Some(serde_json::Value::String(s)) => s,
313                    _ => return Err("an object without a string cell".to_string()),
314                },
315                _ => return Err("an item that is neither a string nor an object".to_string()),
316            };
317            Ok(match s.strip_prefix("cell://") {
318                Some(r) => r.to_string(),
319                None => s,
320            })
321        })
322        .collect()
323}
324
325/// Answers a batch within the budget: the requests are answered in chunks across the pool
326/// (one chunk per core; bindings looked up once per name and asof within a chunk). Those not
327/// answered within the budget become jobs: each is resolved again to the request it
328/// describes (a store read), and its chunk's answer completes the job.
329async fn batch(inner: Arc<Inner>, items: Vec<String>, principal: Option<Principal>) -> Vec<Reply> {
330    let n = items.len();
331    let deadline = tokio::time::Instant::now() + inner.config.budget;
332    let cores = std::thread::available_parallelism().map(|c| c.get()).unwrap_or(4);
333    let chunks = n.div_ceil(64).clamp(1, inner.config.pool_size.min(cores));
334    let per = n.div_ceil(chunks).max(1);
335    let now = inner.now();
336    let items = Arc::new(items);
337    // Each chunk's answers so far; once the budget has passed (`late`), its answers go to
338    // the channel instead, to complete their jobs.
339    let shared: Slots = Arc::new((0..chunks).map(|_| Mutex::new((false, Vec::new()))).collect());
340    let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<(usize, Reply)>();
341    let seen = Arc::new(crate::mesh::Seen::default());
342    let mut tasks = Vec::new();
343    for c in 0..chunks {
344        let (lo, hi) = (c * per, ((c + 1) * per).min(n));
345        if lo >= hi {
346            break;
347        }
348        let (i2, items, tx, p, now, shared) =
349            (inner.clone(), items.clone(), tx.clone(), principal.clone(), now.clone(), shared.clone());
350        let seen = seen.clone();
351        let pool = inner.pool.clone();
352        tasks.push(tokio::spawn(async move {
353            pool.run(move || {
354                let memo = Memo::shared(&i2, &seen);
355                for k in lo..hi {
356                    let reply = i2.answer_with(&items[k], p.as_ref(), &memo, &now, &mut |_| {}, false);
357                    let mut slot = shared[c].lock().unwrap_or_else(|p| p.into_inner());
358                    if slot.0 {
359                        let _ = tx.send((k, reply));
360                    } else {
361                        slot.1.push((k, reply));
362                    }
363                }
364            })
365            .await
366        }));
367    }
368    drop(tx);
369    let all = futures_join(&mut tasks, deadline).await;
370    let mut done: Vec<Option<Reply>> = vec![None; n];
371    for slot in shared.iter() {
372        let mut s = slot.lock().unwrap_or_else(|p| p.into_inner());
373        s.0 = true;
374        for (k, r) in s.1.drain(..) {
375            done[k] = Some(r);
376        }
377    }
378    if all {
379        return done
380            .into_iter()
381            .map(|d| d.unwrap_or_else(|| envelope::error_reply(ErrorCode::EvalUnavailable)))
382            .collect();
383    }
384    // Past the budget: every request not answered yet becomes a job, resolved again to the
385    // request it describes.
386    let open: Vec<usize> = (0..n).filter(|k| done[*k].is_none()).collect();
387    let (i2, items2, open2, now2) = (inner.clone(), items.clone(), open.clone(), now.clone());
388    let requests: Vec<Option<CoreRequest>> = inner
389        .pool
390        .run(move || {
391            let memo = Memo::new(&i2);
392            open2
393                .iter()
394                .map(|k| {
395                    crate::call::resolve(&items2[*k], &now2, &i2.config.aliases, &memo, false)
396                        .ok()
397                        .and_then(|t| CoreRequest::new(t.cell_id(), t.asof()).ok())
398                })
399                .collect()
400        })
401        .await
402        .unwrap_or_default();
403    let poll_base = format!("{}/jobs/", inner.config.origin());
404    let est = inner.jobs.estimate(inner.config.budget.as_secs());
405    let mut jobs: Vec<Option<String>> = vec![None; n];
406    for (i, k) in open.iter().enumerate() {
407        let reply = match requests.get(i).cloned().flatten() {
408            Some(req) => match inner.jobs.create(req.clone()) {
409                Some(id) => {
410                    let p = Pending::new(&id, est, &format!("{poll_base}{id}"), req);
411                    jobs[*k] = Some(id);
412                    match p {
413                        Ok(p) => Reply::json(202, envelope::pending(&p), Cache::NoStore),
414                        Err(_) => envelope::error_reply(ErrorCode::EvalUnavailable),
415                    }
416                }
417                None => envelope::error_reply(ErrorCode::EvalUnavailable),
418            },
419            None => envelope::error_reply(ErrorCode::EvalUnavailable),
420        };
421        done[*k] = Some(reply);
422    }
423    let i3 = inner.clone();
424    tokio::spawn(async move {
425        while let Some((k, mut reply)) = rx.recv().await {
426            if let Some(id) = &jobs[k] {
427                reply.cache = Cache::NoStore;
428                reply.canonical = None;
429                i3.jobs.complete(id, reply);
430            }
431        }
432        for t in tasks {
433            let _ = t.await;
434        }
435        // Jobs whose task ended without an answer (a panic) are given up.
436        for id in jobs.into_iter().flatten() {
437            if let Poll::Running(..) = i3.jobs.poll_peek(&id) {
438                i3.jobs.complete(&id, envelope::error_reply(ErrorCode::EvalUnavailable));
439            }
440        }
441    });
442    done.into_iter().map(|d| d.unwrap_or_else(|| envelope::error_reply(ErrorCode::EvalUnavailable))).collect()
443}
444
445/// Each batch chunk's answers so far, and whether the budget has passed.
446type Slots = Arc<Vec<Mutex<(bool, Vec<(usize, Reply)>)>>>;
447
448/// Waits for every task until `deadline`: whether all finished.
449async fn futures_join(tasks: &mut [tokio::task::JoinHandle<Option<()>>], deadline: tokio::time::Instant) -> bool {
450    for t in tasks.iter_mut() {
451        if t.is_finished() {
452            continue;
453        }
454        if tokio::time::timeout_at(deadline, t).await.is_err() {
455            return false;
456        }
457    }
458    true
459}
460
461/// The members of a slice.
462#[utoipa::path(get, path = "/slice/{cell}",
463    params(
464        ("cell" = String, Path, description = "A slice: a cell URI's rest or a call with `*` for open arguments, of a stored function; `?asof=`, `limit=` (JSON, default 1,000, at most 10,000), `after=` (the cursor of the previous page)."),
465    ),
466    responses(
467        (status = 200, description = "JSON page of members, or with `Accept: application/vnd.apache.arrow.stream` an Arrow IPC stream", content_type = "application/json"),
468        (status = 400, description = "`bad_request`", content_type = "application/json"),
469        (status = 404, description = "`not_covered`", content_type = "application/json"),
470    ))]
471async fn slice(State(mesh): State<Mesh>, uri: Uri, headers: HeaderMap, req: Request) -> Response {
472    let inner = mesh.inner.clone();
473    let (client, _) = match who(&inner, &headers, peer_of(&req)) {
474        Ok(x) => x,
475        Err(r) => return to_response(r),
476    };
477    if let Err(r) = admit(&inner, &client, 1) {
478        return to_response(r);
479    }
480    let rest = rest_of(&uri, "/slice/");
481    // A manifest at /slice/<slice id>/<members hash> (SPEC 15.13).
482    if let Some((a, b)) = rest.split_once('/') {
483        if lexical::check_hex(a, 64, "$").is_ok() && lexical::check_hex(b, 64, "$").is_ok() {
484            return manifest_of(inner, format!("slice:{a}:{b}")).await;
485        }
486    }
487    let arrow = accepts(&headers, crate::slice::ARROW);
488    let r = inner
489        .pool
490        .clone()
491        .run(move || {
492            let now = inner.now();
493            inner.slice(&rest, &now, arrow)
494        })
495        .await
496        .unwrap_or_else(|| envelope::error_reply(ErrorCode::EvalUnavailable));
497    to_response(r)
498}
499
500fn accepts(headers: &HeaderMap, media: &str) -> bool {
501    headers
502        .get_all(header::ACCEPT)
503        .iter()
504        .filter_map(|v| v.to_str().ok())
505        .any(|v| v.split(',').any(|p| p.split(';').next().unwrap_or("").trim() == media))
506}
507
508/// Whether the client takes JSON (no `Accept`, `*/*`, `application/*` or
509/// `application/json`); for slices also Arrow.
510fn acceptable(headers: &HeaderMap, also: Option<&str>) -> bool {
511    let vals: Vec<&str> = headers.get_all(header::ACCEPT).iter().filter_map(|v| v.to_str().ok()).collect();
512    if vals.is_empty() {
513        return true;
514    }
515    vals.iter().flat_map(|v| v.split(',')).any(|p| {
516        let mut parts = p.split(';');
517        let m = parts.next().unwrap_or("").trim();
518        let q0 = parts.any(|x| {
519            let x = x.trim();
520            x == "q=0" || x == "q=0.0" || x == "q=0.00" || x == "q=0.000"
521        });
522        !q0 && (m == "*/*" || m == "application/*" || m == JSON || also == Some(m))
523    })
524}
525
526async fn object(
527    inner: Arc<Inner>,
528    what: &'static str,
529    f: impl FnOnce(&Inner) -> Option<Vec<u8>> + Send + 'static,
530) -> Response {
531    let i2 = inner.clone();
532    match inner.pool.run(move || f(&i2)).await.flatten() {
533        Some(bytes) => {
534            let ct = if what == "blob" { "application/octet-stream" } else { JSON };
535            to_response(Reply {
536                status: 200,
537                body: bytes.into(),
538                content_type: ct,
539                cache: Cache::Immutable,
540                canonical: None,
541                retry_after: None,
542            })
543        }
544        None => not_found(what),
545    }
546}
547
548/// The refusal of a request by its client's rate limit (or key), if it is refused.
549fn ip_admit(inner: &Inner, headers: &HeaderMap, req: &Request) -> Option<Response> {
550    let client = match who(inner, headers, peer_of(req)) {
551        Ok((c, _)) => c,
552        Err(r) => return Some(to_response(r)),
553    };
554    admit(inner, &client, 1).err().map(to_response)
555}
556
557/// A receipt by its hash (canonical bytes; gzip when asked).
558#[utoipa::path(get, path = "/receipt/{hash}", params(("hash" = String, Path, description = "The receipt hash (SPEC 8.3).")),
559    responses((status = 200, description = "The receipt's canonical JSON; immutable", content_type = "application/json"), (status = 404, description = "Not held")))]
560async fn receipt(State(mesh): State<Mesh>, Path(hash): Path<String>, headers: HeaderMap, req: Request) -> Response {
561    if let Some(r) = ip_admit(&mesh.inner, &headers, &req) {
562        return r;
563    }
564    if lexical::check_hex(&hash, 64, "$").is_err() {
565        return not_found("receipt");
566    }
567    object(mesh.inner.clone(), "receipt", move |i| i.store.receipt(&hash).map(|c| c.into_owned())).await
568}
569
570/// A descriptor by its function hash.
571#[utoipa::path(get, path = "/descriptor/{hash}", params(("hash" = String, Path, description = "The function hash.")),
572    responses((status = 200, description = "The descriptor's canonical JSON; immutable", content_type = "application/json"), (status = 404, description = "Not held")))]
573async fn descriptor(State(mesh): State<Mesh>, Path(hash): Path<String>, headers: HeaderMap, req: Request) -> Response {
574    if let Some(r) = ip_admit(&mesh.inner, &headers, &req) {
575        return r;
576    }
577    if lexical::check_hex(&hash, 64, "$").is_err() {
578        return not_found("descriptor");
579    }
580    object(mesh.inner.clone(), "descriptor", move |i| i.store.descriptor(&hash).map(|c| c.into_owned())).await
581}
582
583async fn manifest_of(inner: Arc<Inner>, entry: String) -> Response {
584    object(inner, "manifest", move |i| i.store.manifest(&entry).map(|c| c.into_owned())).await
585}
586
587/// A slice manifest (SPEC 15.13), by its entry's slice id and members hash.
588#[utoipa::path(get, path = "/manifest/{slice_id}/{members_hash}",
589    params(("slice_id" = String, Path, description = "The slice id."), ("members_hash" = String, Path, description = "The members hash.")),
590    responses((status = 200, description = "The manifest's canonical JSON; immutable", content_type = "application/json"), (status = 404, description = "Not held")))]
591async fn manifest(
592    State(mesh): State<Mesh>,
593    Path((slice_id, members)): Path<(String, String)>,
594    headers: HeaderMap,
595    req: Request,
596) -> Response {
597    if let Some(r) = ip_admit(&mesh.inner, &headers, &req) {
598        return r;
599    }
600    if lexical::check_hex(&slice_id, 64, "$").is_err() || lexical::check_hex(&members, 64, "$").is_err() {
601        return not_found("manifest");
602    }
603    manifest_of(mesh.inner.clone(), format!("slice:{slice_id}:{members}")).await
604}
605
606/// Blob content by SHA-256.
607#[utoipa::path(get, path = "/blob/{sha256}", params(("sha256" = String, Path, description = "The content's SHA-256.")),
608    responses((status = 200, description = "The bytes; immutable", content_type = "application/octet-stream"), (status = 404, description = "Not held")))]
609async fn blob(State(mesh): State<Mesh>, Path(sha): Path<String>, headers: HeaderMap, req: Request) -> Response {
610    if let Some(r) = ip_admit(&mesh.inner, &headers, &req) {
611        return r;
612    }
613    if lexical::check_hex(&sha, 64, "$").is_err() {
614        return not_found("blob");
615    }
616    object(mesh.inner.clone(), "blob", move |i| i.store.blob(&sha).ok().flatten().map(|c| c.into_owned())).await
617}
618
619/// A cell's lineage as nested envelopes.
620#[utoipa::path(get, path = "/explain/{cell}",
621    params(("cell" = String, Path, description = "As for /cell; `depth=` (default 4, at most 16) bounds the walk.")),
622    responses(
623        (status = 200, description = "`{\"envelope\", \"inputs\": [{\"entry\", \"envelope\", \"inputs\"} | {\"entry\", \"slice\", \"members\"}], \"truncated\"}`", content_type = "application/json"),
624        (status = 202, description = "`pending`: the cell is being computed; ask again", content_type = "application/json"),
625        (status = 400, description = "`bad_request`", content_type = "application/json"),
626        (status = 404, description = "`not_covered`", content_type = "application/json"),
627    ))]
628async fn explain(State(mesh): State<Mesh>, uri: Uri, headers: HeaderMap, req: Request) -> Response {
629    let inner = mesh.inner.clone();
630    let (client, principal) = match who(&inner, &headers, peer_of(&req)) {
631        Ok(x) => x,
632        Err(r) => return to_response(r),
633    };
634    if let Err(r) = admit(&inner, &client, 1) {
635        return to_response(r);
636    }
637    let mut rest = rest_of(&uri, "/explain/");
638    let mut depth = inner.config.explain_depth;
639    // `depth=` is the explain's own parameter: taken off the query.
640    if let Some((p, q)) = rest.clone().split_once('?') {
641        let kept: Vec<&str> = q.split('&').filter(|x| !x.starts_with("depth=")).collect();
642        if let Some(d) = q.split('&').find_map(|x| x.strip_prefix("depth=")) {
643            depth = d.parse::<usize>().unwrap_or(depth);
644        }
645        rest = if kept.is_empty() { p.to_string() } else { format!("{p}?{}", kept.join("&")) };
646    }
647    let depth = depth.min(16);
648    let r = within_budget(inner, move |i, on| {
649        let now = i.now();
650        let r = i.answer(&rest, principal.as_ref(), i, &now, on);
651        if r.status != 200 {
652            return r;
653        }
654        let env = json::parse(&r.body).unwrap_or(Value::Null);
655        let mut budget = 2_000usize;
656        let v = i.explain_node(env, depth, &mut budget);
657        Reply::json(200, v.to_jcs(), Cache::NoStore)
658    })
659    .await;
660    to_response(r)
661}
662
663impl Inner {
664    /// An envelope with its inputs' envelopes, `depth` levels down, at most `budget` nodes.
665    fn explain_node(&self, envelope: Value, depth: usize, budget: &mut usize) -> Value {
666        let inputs: Vec<String> = envelope
667            .get("receipt")
668            .and_then(|r| r.get("inputs"))
669            .and_then(|i| {
670                if let Value::Array(a) = i {
671                    Some(a.iter().filter_map(Value::as_str).map(str::to_string).collect())
672                } else {
673                    None
674                }
675            })
676            .unwrap_or_default();
677        let mut truncated = false;
678        let mut out = Vec::new();
679        for e in inputs {
680            if depth == 0 || *budget == 0 {
681                truncated = true;
682                break;
683            }
684            *budget -= 1;
685            if e.starts_with("slice:") {
686                let m = self.store.manifest(&e).and_then(|m| datumesh_core::Manifest::from_json(&m).ok());
687                let mut node = vec![("entry".to_string(), Value::String(e.clone()))];
688                if let Some(m) = m {
689                    node.push(("members".to_string(), envelope::num(m.members().len() as u64)));
690                    node.push(("slice".to_string(), m.slice().to_value()));
691                }
692                out.push(Value::Object(node));
693                continue;
694            }
695            let child = match self.store.receipt(&e) {
696                Some(bytes) => {
697                    let trust = Receipt::from_json(&bytes).map(|r| envelope::trust_of(&r, &self.config.host)).ok();
698                    match trust {
699                        Some(t) => self.explain_node(
700                            json::parse(&envelope::ok(&bytes, t)).unwrap_or(Value::Null),
701                            depth - 1,
702                            budget,
703                        ),
704                        None => Value::Null,
705                    }
706                }
707                None => json::parse(&envelope::not_covered(None)).unwrap_or(Value::Null),
708            };
709            let mut node = vec![("entry".to_string(), Value::String(e.clone()))];
710            if let Value::Object(m) = child {
711                node.extend(m);
712            }
713            out.push(Value::Object(node));
714        }
715        let mut node = vec![("envelope".to_string(), envelope.clone()), ("inputs".to_string(), Value::Array(out))];
716        node.push(("truncated".to_string(), Value::Bool(truncated)));
717        Value::Object(node)
718    }
719}
720
721#[derive(serde::Deserialize)]
722struct FindQuery {
723    q: Option<String>,
724    limit: Option<usize>,
725}
726
727/// Functions by keyword (a stub until 2.2).
728#[utoipa::path(get, path = "/find",
729    params(("q" = String, Query, description = "Keywords, all of which must appear (case-insensitive) in the function's name or catalog description."), ("limit" = Option<usize>, Query, description = "At most this many results (default 20, at most 100).")),
730    responses((status = 200, description = "`{\"query\", \"results\": [{\"name\", \"fn_hash\", \"since\", \"description\", \"descriptor\"}]}`", content_type = "application/json")))]
731async fn find(State(mesh): State<Mesh>, Query(q): Query<FindQuery>, headers: HeaderMap, req: Request) -> Response {
732    if let Some(r) = ip_admit(&mesh.inner, &headers, &req) {
733        return r;
734    }
735    let inner = mesh.inner.clone();
736    let query = q.q.unwrap_or_default();
737    let limit = q.limit.unwrap_or(20).clamp(1, 100);
738    let r = inner
739        .pool
740        .clone()
741        .run(move || {
742            let words: Vec<String> = query.split_whitespace().map(str::to_lowercase).collect();
743            let now = inner.now();
744            let mut results = Vec::new();
745            for (name, desc) in &inner.config.catalog {
746                let hay = format!("{} {}", name.to_lowercase(), desc.to_lowercase());
747                if !words.iter().all(|w| hay.contains(w.as_str())) {
748                    continue;
749                }
750                let Ok(Some(b)) = inner.store.resolve_binding(name, &now) else { continue };
751                let d = inner.store.descriptor(&b.fn_hash).and_then(|d| json::parse(&d).ok()).unwrap_or(Value::Null);
752                results.push(Value::Object(vec![
753                    ("description".into(), Value::String(desc.clone())),
754                    ("descriptor".into(), d),
755                    ("fn_hash".into(), Value::String(b.fn_hash)),
756                    ("name".into(), Value::String(name.clone())),
757                    ("since".into(), Value::String(b.tx)),
758                ]));
759                if results.len() >= limit {
760                    break;
761                }
762            }
763            Value::Object(vec![("query".into(), Value::String(query)), ("results".into(), Value::Array(results))])
764        })
765        .await
766        .unwrap_or(Value::Null);
767    plain_json(200, r, Cache::Public(60))
768}
769
770#[derive(serde::Deserialize)]
771struct VerifyQuery {
772    level: Option<String>,
773    blob: Option<String>,
774    form: Option<String>,
775}
776
777/// Verification as a service: the core verifier on a submitted receipt, with this mesh's
778/// keys; free, unauthenticated, rate-limited per IP.
779#[utoipa::path(post, path = "/verify",
780    params(("level" = Option<String>, Query, description = "`recomputed` (default: recompute with the engine as the resolver) or `signed` (steps 1-6)."), ("blob" = Option<String>, Query, description = "Blob content for step 7.7, as hex (at most 1 MiB)."), ("form" = Option<String>, Query, description = "`offered`: the body is `{\"receipt\": <its JSON text>, \"recompute\"?: <SPEC 11's recompute object>, \"blob_hex\"?}` and step 7 uses what is offered instead of this mesh's engine.")),
781    request_body(content = String, description = "The receipt's JSON text, exactly as received (or the offered form).", content_type = "application/json"),
782    responses(
783        (status = 200, description = "`{\"ok\", \"level\", \"code\", \"hash\", \"computed_by\", \"path\", \"detail\", \"blob_checked\"}`", content_type = "application/json"),
784        (status = 429, description = "`rate_limited`", content_type = "application/json"),
785    ))]
786async fn verify_receipt(
787    State(mesh): State<Mesh>,
788    Query(q): Query<VerifyQuery>,
789    headers: HeaderMap,
790    req: Request,
791) -> Response {
792    let inner = mesh.inner.clone();
793    let ip = client_ip(&inner, &headers, peer_of(&req));
794    if let Err(s) = inner.limiter.admit_verify(ip) {
795        return to_response(envelope::rate_limited_reply(s));
796    }
797    let body: Bytes = match axum::body::to_bytes(req.into_body(), 4 << 20).await {
798        Ok(b) => b,
799        Err(_) => return batch_error("body too large"),
800    };
801    if q.form.as_deref() == Some("offered") {
802        let Ok(v) = json::parse(&body) else { return batch_error("form=offered: not strict JSON") };
803        let Some(receipt) = v.get("receipt").and_then(Value::as_str).map(|s| s.as_bytes().to_vec()) else {
804            return batch_error("form=offered: no receipt text");
805        };
806        let blob = match v.get("blob_hex").and_then(Value::as_str).map(hex) {
807            None => None,
808            Some(Some(b)) if b.len() <= 1 << 20 => Some(b),
809            Some(_) => return batch_error("blob_hex: hex of at most 1 MiB"),
810        };
811        let m = mesh.clone();
812        let rc = v.get("recompute").cloned();
813        let r = inner.pool.run(move || m.verify_offered(&receipt, rc.as_ref(), blob.as_deref())).await;
814        return match r {
815            Some(v) => to_response(Reply::json(200, v.body, Cache::NoStore)),
816            None => to_response(envelope::error_reply(ErrorCode::EvalUnavailable)),
817        };
818    }
819    let signed = q.level.as_deref() == Some("signed");
820    let blob = match q.blob.as_deref().map(hex) {
821        None => None,
822        Some(Some(b)) if b.len() <= 1 << 20 => Some(b),
823        Some(_) => return batch_error("blob: hex of at most 1 MiB"),
824    };
825    let m = mesh.clone();
826    let r = inner.pool.run(move || m.verify(&body, signed, blob.as_deref())).await;
827    match r {
828        Some(v) => to_response(Reply::json(200, v.body, Cache::NoStore)),
829        None => to_response(envelope::error_reply(ErrorCode::EvalUnavailable)),
830    }
831}
832
833fn hex(s: &str) -> Option<Vec<u8>> {
834    if !s.len().is_multiple_of(2) {
835        return None;
836    }
837    (0..s.len()).step_by(2).map(|i| u8::from_str_radix(s.get(i..i + 2)?, 16).ok()).collect()
838}
839
840/// A job: `pending` again while it runs, then the envelope of the original request.
841#[utoipa::path(get, path = "/jobs/{job}", params(("job" = String, Path, description = "The job id of a `pending` envelope.")),
842    responses(
843        (status = 202, description = "`pending`, the same job and request", content_type = "application/json"),
844        (status = 400, description = "`bad_request` with `E_JOB_UNKNOWN`: unknown or expired", content_type = "application/json"),
845        (status = 200, description = "The completed envelope (any outcome but pending, with its status), bound to the original request", content_type = "application/json"),
846    ))]
847async fn job(State(mesh): State<Mesh>, Path(id): Path<String>, headers: HeaderMap, req: Request) -> Response {
848    let inner = mesh.inner.clone();
849    let (client, _) = match who(&inner, &headers, peer_of(&req)) {
850        Ok(x) => x,
851        Err(r) => return to_response(r),
852    };
853    if let Err(r) = admit(&inner, &client, 1) {
854        return to_response(r);
855    }
856    let unknown = || to_response(Reply::json(400, envelope::bad_request(ErrorCode::JobUnknown), Cache::NoStore));
857    let valid = (32..=64).contains(&id.len()) && id.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b));
858    if !valid {
859        return unknown();
860    }
861    match inner.jobs.poll(&id) {
862        Poll::Unknown => unknown(),
863        Poll::Done(r) => to_response(r),
864        Poll::Running(request, elapsed) => {
865            let poll = format!("{}/jobs/{id}", inner.config.origin());
866            match Pending::new(&id, inner.jobs.estimate(elapsed), &poll, request) {
867                Ok(p) => to_response(Reply::json(202, envelope::pending(&p), Cache::NoStore)),
868                Err(_) => to_response(envelope::error_reply(ErrorCode::EvalUnavailable)),
869            }
870        }
871    }
872}
873
874/// The key document (SPEC 9.2): the store's namespace record for this mesh's host.
875#[utoipa::path(get, path = "/.well-known/datumesh.json", responses((status = 200, description = "The key document", content_type = "application/json"), (status = 404, description = "None stored")))]
876async fn key_document(State(mesh): State<Mesh>) -> Response {
877    let m = mesh.clone();
878    match mesh.inner.pool.run(move || m.key_document()).await.flatten() {
879        Some(doc) => to_response(Reply::json(200, doc, Cache::Public(300))),
880        None => not_found("key document"),
881    }
882}
883
884/// The mesh document (SPEC 9.6).
885#[utoipa::path(get, path = "/.well-known/datumesh-mesh.json", responses((status = 200, description = "The mesh document", content_type = "application/json")))]
886async fn mesh_document(State(mesh): State<Mesh>) -> Response {
887    to_response(Reply::json(200, mesh.mesh_document(), Cache::Public(300)))
888}
889
890/// Which function each name denotes as of an instant (SPEC 9.7).
891#[utoipa::path(get, path = "/bindings",
892    params(("name" = String, Query, description = "A function name, percent-encoded as in SPEC 5.1; repeat for several."), ("asof" = Option<String>, Query, description = "An instant (default: now).")),
893    responses((status = 200, description = "`{\"asof\", \"bindings\": {name: {\"fn_hash\", \"since\"}}}`", content_type = "application/json"), (status = 400, description = "A malformed name or asof", content_type = "application/json")))]
894async fn bindings(State(mesh): State<Mesh>, uri: Uri, headers: HeaderMap, req: Request) -> Response {
895    if let Some(r) = ip_admit(&mesh.inner, &headers, &req) {
896        return r;
897    }
898    let inner = mesh.inner.clone();
899    let mut names = Vec::new();
900    let mut asof = None;
901    for p in uri.query().unwrap_or("").split('&').filter(|p| !p.is_empty()) {
902        let (k, v) = p.split_once('=').unwrap_or((p, ""));
903        let Some(v) = pct_decode(v) else { return batch_error("percent-encoding (SPEC 5.1)") };
904        match k {
905            "name" if lexical::check_function_name(&v, "$").is_ok() => names.push(v),
906            "asof" if lexical::check_instant(&v, "$").is_ok() && asof.is_none() => asof = Some(v),
907            _ => return batch_error(&format!("parameter {k}")),
908        }
909    }
910    let r = inner
911        .pool
912        .clone()
913        .run(move || {
914            let at = asof.unwrap_or_else(|| inner.now());
915            let mut out = Vec::new();
916            for n in names {
917                if !inner.config.owns(crate::foreign::host_of(&n)) {
918                    continue;
919                }
920                match inner.store.resolve_binding(&n, &at) {
921                    Ok(Some(b)) => out.push((
922                        n,
923                        Value::Object(vec![
924                            ("fn_hash".into(), Value::String(b.fn_hash)),
925                            ("since".into(), Value::String(b.tx)),
926                        ]),
927                    )),
928                    Ok(None) => {}
929                    Err(_) => return None,
930                }
931            }
932            Some(Value::Object(vec![("asof".into(), Value::String(at)), ("bindings".into(), Value::Object(out))]))
933        })
934        .await
935        .flatten();
936    match r {
937        Some(v) => plain_json(200, v, Cache::Public(inner_latest(&mesh))),
938        None => to_response(envelope::error_reply(ErrorCode::InputUnavailable)),
939    }
940}
941
942fn inner_latest(mesh: &Mesh) -> u64 {
943    mesh.inner.config.ttl.latest_max_age.as_secs()
944}
945
946/// Strict percent-decoding (SPEC 5.1): literal characters as themselves, `%` and two
947/// uppercase hex digits for other bytes, UTF-8.
948fn pct_decode(s: &str) -> Option<String> {
949    let b = s.as_bytes();
950    let mut out = Vec::new();
951    let mut i = 0;
952    while i < b.len() {
953        let c = b[i];
954        if c == b'%' {
955            let h = b.get(i + 1..i + 3)?;
956            if !h.iter().all(|x| x.is_ascii_digit() || (b'A'..=b'F').contains(x)) {
957                return None;
958            }
959            let v = u8::from_str_radix(std::str::from_utf8(h).ok()?, 16).ok()?;
960            if v.is_ascii_alphanumeric() || matches!(v, b'-' | b'.' | b'_' | b'~' | b':') {
961                return None;
962            }
963            out.push(v);
964            i += 3;
965        } else if c.is_ascii_alphanumeric() || matches!(c, b'-' | b'.' | b'_' | b'~' | b':') {
966            out.push(c);
967            i += 1;
968        } else {
969            return None;
970        }
971    }
972    String::from_utf8(out).ok()
973}
974
975/// Liveness and the store's health.
976#[utoipa::path(get, path = "/health", responses((status = 200, description = "`{\"status\": \"ok\"}`", content_type = "application/json"), (status = 503, description = "The store does not answer", content_type = "application/json")))]
977async fn health(State(mesh): State<Mesh>) -> Response {
978    let m = mesh.clone();
979    let ok = tokio::time::timeout(
980        Duration::from_secs(5),
981        mesh.inner.pool.run(move || m.inner.store.namespace(&m.inner.config.host).is_ok()),
982    )
983    .await
984    .ok()
985    .flatten()
986    .unwrap_or(false);
987    let v = Value::Object(vec![("status".into(), Value::String(if ok { "ok" } else { "store unavailable" }.into()))]);
988    plain_json(if ok { 200 } else { 503 }, v, Cache::NoStore)
989}
990
991/// Prometheus metrics.
992#[utoipa::path(get, path = "/metrics", responses((status = 200, description = "Prometheus text", content_type = "text/plain")))]
993async fn metrics(State(mesh): State<Mesh>) -> Response {
994    let mut r = Reply::json(200, metrics_text(&mesh.inner).into_bytes(), Cache::NoStore);
995    r.content_type = "text/plain; version=0.0.4";
996    to_response(r)
997}
998
999/// This document.
1000#[utoipa::path(get, path = "/openapi.json", responses((status = 200, description = "The OpenAPI document", content_type = "application/json")))]
1001async fn openapi_json() -> Response {
1002    let mut r = Reply::json(200, openapi().to_json().unwrap_or_default().into_bytes(), Cache::Public(300));
1003    r.content_type = JSON;
1004    to_response(r)
1005}
1006
1007pub(crate) fn metrics_text(inner: &Inner) -> String {
1008    use std::fmt::Write;
1009    use std::sync::atomic::Ordering::Relaxed;
1010    let mut out = String::new();
1011    inner.metrics.write(&mut out);
1012    let p = &inner.pool;
1013    let g = |out: &mut String, name: &str, help: &str, kind: &str, v: String| {
1014        let _ = writeln!(out, "# HELP {name} {help}\n# TYPE {name} {kind}\n{name} {v}");
1015    };
1016    g(&mut out, "datumesh_pool_size", "The blocking pool's size: the concurrency limit.", "gauge", p.size.to_string());
1017    g(&mut out, "datumesh_pool_in_use", "Pool threads busy now.", "gauge", p.in_use.load(Relaxed).to_string());
1018    g(&mut out, "datumesh_pool_peak", "The most pool threads busy at once.", "gauge", p.peak.load(Relaxed).to_string());
1019    g(&mut out, "datumesh_pool_runs_total", "Tasks run on the pool.", "counter", p.runs.load(Relaxed).to_string());
1020    g(
1021        &mut out,
1022        "datumesh_pool_waited_total",
1023        "Tasks that found the pool saturated and waited for a thread.",
1024        "counter",
1025        p.waited.load(Relaxed).to_string(),
1026    );
1027    let (held, running) = inner.jobs.counts();
1028    g(&mut out, "datumesh_jobs_held", "Jobs held (running or done).", "gauge", held.to_string());
1029    g(&mut out, "datumesh_jobs_running", "Jobs running.", "gauge", running.to_string());
1030    g(
1031        &mut out,
1032        "datumesh_jobs_created_total",
1033        "Requests answered pending.",
1034        "counter",
1035        inner.jobs.created.load(Relaxed).to_string(),
1036    );
1037    g(
1038        &mut out,
1039        "datumesh_rate_limited_total",
1040        "Requests refused by a rate limit.",
1041        "counter",
1042        inner.limiter.refused.load(Relaxed).to_string(),
1043    );
1044    g(
1045        &mut out,
1046        "datumesh_peer_fetches_total",
1047        "Requests to other meshes.",
1048        "counter",
1049        inner.foreign.fetches.load(Relaxed).to_string(),
1050    );
1051    g(
1052        &mut out,
1053        "datumesh_peer_failures_total",
1054        "Requests to other meshes that failed.",
1055        "counter",
1056        inner.foreign.failures.load(Relaxed).to_string(),
1057    );
1058    g(
1059        &mut out,
1060        "datumesh_peer_imported_total",
1061        "Foreign versions imported into the store.",
1062        "counter",
1063        inner.foreign.imported.load(Relaxed).to_string(),
1064    );
1065    let m = inner.engine.memo_stats();
1066    g(&mut out, "datumesh_memo_entries", "Engine memo entries.", "gauge", m.entries.to_string());
1067    g(&mut out, "datumesh_memo_bytes", "Engine memo size, approximately.", "gauge", m.approx_bytes.to_string());
1068    g(
1069        &mut out,
1070        "datumesh_uptime_seconds",
1071        "Seconds since the mesh started.",
1072        "gauge",
1073        inner.started.elapsed().as_secs().to_string(),
1074    );
1075    out
1076}
1077
1078/// Request ids, the access log, metrics and `Accept`.
1079async fn around(State(mesh): State<Mesh>, req: Request, next: Next) -> Response {
1080    let start = Instant::now();
1081    let route = req.extensions().get::<MatchedPath>().map(|m| m.as_str().to_string()).unwrap_or_else(|| "other".into());
1082    let route = match route.as_str() {
1083        "/cell/{*rest}" => "/cell/{cell}".to_string(),
1084        "/slice/{*rest}" => "/slice/{cell}".to_string(),
1085        "/explain/{*rest}" => "/explain/{cell}".to_string(),
1086        _ => route,
1087    };
1088    let id = req
1089        .headers()
1090        .get("x-request-id")
1091        .and_then(|v| v.to_str().ok())
1092        .filter(|s| !s.is_empty() && s.len() <= 64 && s.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-'))
1093        .map(str::to_string)
1094        .unwrap_or_else(|| {
1095            let mut b = [0u8; 8];
1096            let _ = getrandom::fill(&mut b);
1097            lexical::to_hex(&b)
1098        });
1099    let ip = client_ip(&mesh.inner, req.headers(), peer_of(&req));
1100    let method = req.method().clone();
1101    let also = if route == "/slice/{cell}" { Some(crate::slice::ARROW) } else { None };
1102    let mut resp = if route != "/blob/{sha256}" && route != "/metrics" && !acceptable(req.headers(), also) {
1103        let mut r = Reply::json(
1104            406,
1105            Value::Object(vec![("error".into(), Value::String("not acceptable: application/json only".into()))])
1106                .to_jcs(),
1107            Cache::NoStore,
1108        );
1109        r.content_type = JSON;
1110        to_response(r)
1111    } else {
1112        next.run(req).instrument(tracing::info_span!("request", request_id = %id)).await
1113    };
1114    let elapsed = start.elapsed();
1115    let status = resp.status().as_u16();
1116    mesh.inner.metrics.record(&route, status, elapsed);
1117    let principal = resp.extensions().get::<Who>().and_then(|w| w.0.clone());
1118    tracing::info!(target: "datumesh_http::access", request_id = %id, %method, route = %route, status, ms = elapsed.as_secs_f64() * 1000.0, ip = %ip, principal = principal.as_deref().unwrap_or("-"), "request");
1119    if let Ok(v) = HeaderValue::from_str(&id) {
1120        resp.headers_mut().insert("x-request-id", v);
1121    }
1122    resp.headers_mut().insert(header::VARY, HeaderValue::from_static("Accept, Accept-Encoding, Authorization"));
1123    if let Ok(v) = HeaderValue::from_str(&format!("app;dur={:.3}", elapsed.as_secs_f64() * 1000.0)) {
1124        resp.headers_mut().insert("server-timing", v);
1125    }
1126    resp
1127}
1128
1129async fn fallback() -> Response {
1130    not_found("endpoint")
1131}
1132
1133impl Mesh {
1134    /// The router of every endpoint. Serve it with
1135    /// `into_make_service_with_connect_info::<SocketAddr>()` (the client IP comes from it).
1136    pub fn router(&self) -> Router {
1137        Router::new()
1138            .route("/cell/{*rest}", get(cell))
1139            .route("/cells", post(cells))
1140            .route("/slice/{*rest}", get(slice))
1141            .route("/receipt/{hash}", get(receipt))
1142            .route("/descriptor/{hash}", get(descriptor))
1143            .route("/manifest/{slice_id}/{members_hash}", get(manifest))
1144            .route("/blob/{sha256}", get(blob))
1145            .route("/explain/{*rest}", get(explain))
1146            .route("/find", get(find))
1147            .route("/verify", post(verify_receipt))
1148            .route("/jobs/{job}", get(job))
1149            .route("/.well-known/datumesh.json", get(key_document))
1150            .route("/.well-known/datumesh-mesh.json", get(mesh_document))
1151            .route("/bindings", get(bindings))
1152            .route("/health", get(health))
1153            .route("/metrics", get(metrics))
1154            .route("/openapi.json", get(openapi_json))
1155            .fallback(fallback)
1156            .layer(middleware::from_fn_with_state(self.clone(), around))
1157            .layer(tower_http::compression::CompressionLayer::new().gzip(true))
1158            .with_state(self.clone())
1159    }
1160
1161    /// Serves the router on `listener` until `shutdown` resolves.
1162    pub async fn serve(
1163        &self,
1164        listener: tokio::net::TcpListener,
1165        shutdown: impl std::future::Future<Output = ()> + Send + 'static,
1166    ) -> std::io::Result<()> {
1167        axum::serve(listener, self.router().into_make_service_with_connect_info::<SocketAddr>())
1168            .with_graceful_shutdown(shutdown)
1169            .await
1170    }
1171}
1172
1173impl IntoResponse for Reply {
1174    fn into_response(self) -> Response {
1175        to_response(self)
1176    }
1177}