1use std::net::{IpAddr, Ipv4Addr, SocketAddr};
4use std::sync::{Arc, Mutex};
5use std::time::{Duration, Instant};
6
7use axum::body::{Body, Bytes};
8use axum::extract::{ConnectInfo, MatchedPath, Path, Query, Request, State};
9use axum::http::{header, HeaderMap, HeaderValue, StatusCode, Uri};
10use axum::middleware::{self, Next};
11use axum::response::{IntoResponse, Response};
12use axum::routing::{get, post};
13use axum::Router;
14use datumesh_core::json::{self, Value};
15use datumesh_core::{lexical, ErrorCode, Pending, Request as CoreRequest};
16use tracing::Instrument;
17use utoipa::OpenApi;
18
19use crate::apikeys::Principal;
20use crate::envelope::{self, Cache, Reply, JSON};
21use crate::jobs::Poll;
22use crate::limit::Client;
23use crate::mesh::{Inner, Memo, Mesh};
24
25#[derive(Clone)]
27struct Who(Option<String>);
28
29fn to_response(r: Reply) -> Response {
30 let mut resp = Response::new(Body::from(r.body));
31 *resp.status_mut() = StatusCode::from_u16(r.status).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
32 let h = resp.headers_mut();
33 h.insert(header::CONTENT_TYPE, HeaderValue::from_static(r.content_type));
34 if let Ok(v) = HeaderValue::from_str(&r.cache.header()) {
35 h.insert(header::CACHE_CONTROL, v);
36 }
37 if let Some(c) = r.canonical {
38 if let Ok(v) = HeaderValue::from_str(&format!("<{c}>; rel=\"canonical\"")) {
39 h.insert(header::LINK, v);
40 }
41 }
42 if let Some(s) = r.retry_after {
43 h.insert(header::RETRY_AFTER, HeaderValue::from(s));
44 }
45 resp
46}
47
48fn plain_json(status: u16, v: Value, cache: Cache) -> Response {
49 to_response(Reply::json(status, v.to_jcs(), cache))
50}
51
52fn not_found(what: &str) -> Response {
53 plain_json(404, Value::Object(vec![("error".into(), Value::String(format!("no such {what}")))]), Cache::Public(60))
54}
55
56fn who(inner: &Inner, headers: &HeaderMap, peer: IpAddr) -> Result<(Client, Option<Principal>), Reply> {
59 let ip = client_ip(inner, headers, peer);
60 match headers.get(header::AUTHORIZATION) {
61 None => Ok((Client::Ip(ip), None)),
62 Some(v) => {
63 let p = v
64 .to_str()
65 .ok()
66 .and_then(|s| s.strip_prefix("Bearer "))
67 .and_then(|k| inner.config.api_keys.principal(k.trim()));
68 match p {
69 Some(p) => Ok((Client::Key(p.id.clone(), p.per_minute), Some(p.clone()))),
70 None => Err(Reply::json(403, envelope::unlicensed(), Cache::Private)),
71 }
72 }
73 }
74}
75
76fn client_ip(inner: &Inner, headers: &HeaderMap, peer: IpAddr) -> IpAddr {
77 if inner.config.trust_proxy {
78 if let Some(ip) = headers
79 .get("x-forwarded-for")
80 .and_then(|v| v.to_str().ok())
81 .and_then(|s| s.rsplit(',').next())
82 .and_then(|s| s.trim().parse::<IpAddr>().ok())
83 {
84 return ip;
85 }
86 }
87 peer
88}
89
90fn peer_of(req: &Request) -> IpAddr {
91 req.extensions().get::<ConnectInfo<SocketAddr>>().map(|c| c.0.ip()).unwrap_or(IpAddr::V4(Ipv4Addr::UNSPECIFIED))
92}
93
94fn rest_of(uri: &Uri, prefix: &str) -> String {
96 let path = uri.path().strip_prefix(prefix).unwrap_or("");
97 match uri.query() {
98 Some(q) => format!("{path}?{q}"),
99 None => path.to_string(),
100 }
101}
102
103async fn within_budget(
107 inner: Arc<Inner>,
108 f: impl FnOnce(&Inner, &mut dyn FnMut(&crate::call::Target)) -> Reply + Send + 'static,
109) -> Reply {
110 let deadline = tokio::time::Instant::now() + inner.config.budget;
111 let (tx, mut rx) = tokio::sync::watch::channel::<Option<CoreRequest>>(None);
112 let pool = inner.pool.clone();
113 let i2 = inner.clone();
114 let mut task = tokio::spawn(async move {
115 pool.run(move || {
116 let mut on = |t: &crate::call::Target| {
117 if let Some(r) = request_of(t) {
118 let _ = tx.send(Some(r));
119 }
120 };
121 f(&i2, &mut on)
122 })
123 .await
124 });
125 match tokio::time::timeout_at(deadline, &mut task).await {
126 Ok(Ok(Some(r))) => r,
127 Ok(_) => envelope::error_reply(ErrorCode::EvalUnavailable),
128 Err(_) => {
129 let request = rx.borrow_and_update().clone();
130 let Some(request) = request else {
131 return envelope::error_reply(ErrorCode::EvalUnavailable);
134 };
135 pend(inner, request, task)
136 }
137 }
138}
139
140fn request_of(t: &crate::call::Target) -> Option<CoreRequest> {
142 let r = CoreRequest::new(t.cell_id(), t.asof()).ok()?;
143 match &t.pin {
144 Some(p) => r.with_pin(p).ok(),
145 None => Some(r),
146 }
147}
148
149fn pend(inner: Arc<Inner>, request: CoreRequest, task: tokio::task::JoinHandle<Option<Reply>>) -> Reply {
151 let Some(id) = inner.jobs.create(request.clone()) else {
152 return envelope::error_reply(ErrorCode::EvalUnavailable);
153 };
154 let poll = format!("{}/jobs/{id}", inner.config.origin());
155 let est = inner.jobs.estimate(inner.config.budget.as_secs());
156 let i2 = inner.clone();
157 let id2 = id.clone();
158 tokio::spawn(async move {
159 let r = match task.await {
160 Ok(Some(r)) if r.status != 202 => r,
161 _ => envelope::error_reply(ErrorCode::EvalUnavailable),
162 };
163 let mut r = r;
164 r.cache = Cache::NoStore;
165 r.canonical = None;
166 i2.jobs.complete(&id2, r);
167 });
168 match Pending::new(&id, est, &poll, request) {
169 Ok(p) => Reply::json(202, envelope::pending(&p), Cache::NoStore),
170 Err(_) => envelope::error_reply(ErrorCode::EvalUnavailable),
171 }
172}
173
174fn admit(inner: &Inner, client: &Client, cost: u32) -> Result<(), Reply> {
175 inner.limiter.admit(client, cost).map_err(envelope::rate_limited_reply)
176}
177
178#[derive(OpenApi)]
179#[openapi(
180 info(
181 title = "datumesh-mesh",
182 description = "The Datumesh engine behind HTTPS (spec-0.4.1): cell envelopes (SPEC 5.5), batches, slices, receipts, descriptors, manifests, blobs, lineage, search, verification, jobs, key and mesh documents (SPEC 9.2, 9.6), bindings (SPEC 9.7).",
183 license(name = "Apache-2.0")
184 ),
185 paths(
186 cell,
187 cells,
188 slice,
189 versions,
190 receipt,
191 descriptor,
192 manifest,
193 blob,
194 explain,
195 find,
196 verify_receipt,
197 job,
198 key_document,
199 mesh_document,
200 bindings,
201 describe,
202 health,
203 metrics,
204 openapi_json
205 )
206)]
207struct ApiDoc;
208
209pub fn openapi() -> utoipa::openapi::OpenApi {
211 ApiDoc::openapi()
212}
213
214#[utoipa::path(get, path = "/cell/{cell}",
216 params(("cell" = String, Path, description = "A cell URI's rest (SPEC 5.3), byte for byte, `asof` included; or a call (SPEC 15.3's read syntax, URL-encoded) with an optional `?asof=`.")),
217 responses(
218 (status = 200, description = "`ok` or `stale`", content_type = "application/json"),
219 (status = 202, description = "`pending`: poll the job (SPEC 5.5)", content_type = "application/json"),
220 (status = 400, description = "`bad_request`", content_type = "application/json"),
221 (status = 402, description = "`payment_required`", content_type = "application/json"),
222 (status = 403, description = "`unlicensed`", content_type = "application/json"),
223 (status = 404, description = "`not_covered` (with `fault` when the evaluation faulted)", content_type = "application/json"),
224 (status = 406, description = "the Accept header excludes JSON"),
225 (status = 429, description = "`rate_limited`, with `Retry-After`", content_type = "application/json"),
226 (status = 500, description = "`error`", content_type = "application/json"),
227 (status = 503, description = "`error`: the store or another mesh is unavailable", content_type = "application/json"),
228 ))]
229async fn cell(State(mesh): State<Mesh>, uri: Uri, headers: HeaderMap, req: Request) -> Response {
230 let inner = mesh.inner.clone();
231 let (client, principal) = match who(&inner, &headers, peer_of(&req)) {
232 Ok(x) => x,
233 Err(r) => return to_response(r),
234 };
235 let mut resp = match admit(&inner, &client, 1) {
236 Err(r) => to_response(r),
237 Ok(()) => {
238 let rest = rest_of(&uri, "/cell/");
239 let p = principal.clone();
240 let r = within_budget(inner.clone(), move |i, on| {
241 let now = i.now();
242 i.answer(&rest, p.as_ref(), i, &now, on)
243 })
244 .await;
245 if let Some(o) = r.outcome() {
246 inner.metrics.outcome("/cell/{cell}", &o);
247 }
248 to_response(r)
249 }
250 };
251 resp.extensions_mut().insert(Who(principal.map(|p| p.id)));
252 resp
253}
254
255#[utoipa::path(post, path = "/cells",
257 request_body(content = String, description = "A JSON array of requests, each a call or a cell URI (`cell://...`) as a string, or `{\"cell\": <that>}`; at most the configured bound (1,000).", content_type = "application/json"),
258 responses(
259 (status = 200, description = "A JSON array of envelopes, in order; each has the HTTP status its outcome has (`error`: 500)", content_type = "application/json"),
260 (status = 400, description = "Not such an array, or too long", content_type = "application/json"),
261 (status = 429, description = "`rate_limited` (a batch of n cells counts as n / 50 requests)", content_type = "application/json"),
262 ))]
263async fn cells(State(mesh): State<Mesh>, headers: HeaderMap, req: Request) -> Response {
264 let inner = mesh.inner.clone();
265 let peer = peer_of(&req);
266 let (client, principal) = match who(&inner, &headers, peer) {
267 Ok(x) => x,
268 Err(r) => return to_response(r),
269 };
270 let body = match axum::body::to_bytes(req.into_body(), 64 << 20).await {
271 Ok(b) => b,
272 Err(_) => return to_response(envelope::bad_request_reply(ErrorCode::CanonLimit)),
273 };
274 let items = match parse_batch(&body, inner.config.batch_max) {
275 Ok(i) => i,
276 Err((code, detail)) => {
278 tracing::info!(target: "datumesh_http", detail = %detail, "batch refused");
279 return to_response(envelope::bad_request_reply(code));
280 }
281 };
282 if let Err(r) = admit(&inner, &client, inner.limiter.batch_cost(items.len())) {
283 return to_response(r);
284 }
285 let pieces = batch(inner.clone(), items, principal.clone()).await;
286 let mut counts: std::collections::BTreeMap<&str, u64> = std::collections::BTreeMap::new();
287 for w in &pieces {
288 for (o, k) in &w.outcomes {
289 *counts.entry(o).or_default() += k;
290 }
291 }
292 for (o, k) in counts {
293 inner.metrics.outcomes("/cells", o, k);
294 }
295 let mut out = Vec::with_capacity(pieces.iter().map(|w| w.bytes.len()).sum::<usize>() + 2);
298 for w in &pieces {
299 out.extend_from_slice(&w.bytes);
300 }
301 out.extend_from_slice(if pieces.is_empty() { b"[]" } else { b"]" });
302 let mut resp =
303 to_response(Reply::json(200, out, if principal.is_some() { Cache::Private } else { Cache::NoStore }));
304 resp.extensions_mut().insert(Who(principal.map(|p| p.id)));
305 resp
306}
307
308fn batch_error(e: &str) -> Response {
309 plain_json(400, Value::Object(vec![("error".into(), Value::String(e.to_string()))]), Cache::NoStore)
310}
311
312fn parse_batch(body: &[u8], max: usize) -> Result<Vec<String>, (ErrorCode, String)> {
315 let pu = |m: String| (ErrorCode::ParseUri, m);
316 let v: serde_json::Value = serde_json::from_slice(body).map_err(|e| pu(format!("not JSON: {e}")))?;
318 let serde_json::Value::Array(items) = v else { return Err(pu("not a JSON array".into())) };
319 if items.len() > max {
320 return Err((ErrorCode::CanonLimit, format!("{} requests: at most {max}", items.len())));
321 }
322 items
323 .into_iter()
324 .map(|i| {
325 let s = match i {
326 serde_json::Value::String(s) => s,
327 serde_json::Value::Object(mut o) => match o.remove("cell") {
328 Some(serde_json::Value::String(s)) => s,
329 _ => return Err(pu("an object without a string cell".to_string())),
330 },
331 _ => return Err(pu("an item that is neither a string nor an object".to_string())),
332 };
333 Ok(match s.strip_prefix("cell://") {
334 Some(r) => r.to_string(),
335 None => s,
336 })
337 })
338 .collect()
339}
340
341#[derive(Default)]
347struct Written {
348 bytes: Vec<u8>,
349 outcomes: Vec<(String, u64)>,
350}
351
352impl Written {
353 fn of<'a>(first: bool, replies: impl Iterator<Item = &'a Reply> + Clone) -> Written {
355 let mut w = Written {
356 bytes: Vec::with_capacity(replies.clone().map(|r| r.body.len() + 32).sum::<usize>() + 1),
357 outcomes: vec![],
358 };
359 w.bytes.push(if first { b'[' } else { b',' });
360 for (i, r) in replies.enumerate() {
361 if i > 0 {
362 w.bytes.push(b',');
363 }
364 w.push(r);
365 }
366 w
367 }
368
369 fn push(&mut self, r: &Reply) {
370 self.bytes.extend_from_slice(b"{\"envelope\":");
371 self.bytes.extend_from_slice(&r.body);
372 self.bytes.extend_from_slice(b",\"status\":");
373 push_u16(&mut self.bytes, r.status);
374 self.bytes.push(b'}');
375 if let Some(o) = r.outcome_str() {
376 match self.outcomes.iter_mut().find(|(x, _)| x == o) {
377 Some((_, k)) => *k += 1,
378 None => self.outcomes.push((o.to_string(), 1)),
379 }
380 }
381 }
382}
383
384fn push_u16(out: &mut Vec<u8>, n: u16) {
386 let mut buf = [0u8; 5];
387 let (mut i, mut n) = (buf.len(), n);
388 loop {
389 i -= 1;
390 buf[i] = b'0' + (n % 10) as u8;
391 n /= 10;
392 if n == 0 {
393 break;
394 }
395 }
396 out.extend_from_slice(&buf[i..]);
397}
398
399fn cores() -> usize {
402 static CORES: std::sync::OnceLock<usize> = std::sync::OnceLock::new();
403 *CORES.get_or_init(|| std::thread::available_parallelism().map(|c| c.get()).unwrap_or(4))
404}
405
406async fn batch(inner: Arc<Inner>, items: Vec<String>, principal: Option<Principal>) -> Vec<Written> {
412 let n = items.len();
413 let deadline = tokio::time::Instant::now() + inner.config.budget;
414 let chunks = n.div_ceil(64).clamp(1, inner.config.pool_size.min(cores()));
415 let per = n.div_ceil(chunks).max(1);
416 let ranges: Vec<(usize, usize)> =
417 (0..chunks).map(|c| (c * per, ((c + 1) * per).min(n))).take_while(|(lo, hi)| lo < hi).collect();
418 let now = inner.now();
419 let items = Arc::new(items);
420 let shared: Slots = Arc::new(ranges.iter().map(|_| Mutex::new(Slot::default())).collect());
424 let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<(usize, Reply)>();
425 let seen = Arc::new(crate::mesh::Seen::default());
426 let mut tasks = Vec::new();
427 for (c, &(lo, hi)) in ranges.iter().enumerate() {
428 let (i2, items, tx, p, now, shared) =
429 (inner.clone(), items.clone(), tx.clone(), principal.clone(), now.clone(), shared.clone());
430 let seen = seen.clone();
431 let pool = inner.pool.clone();
432 tasks.push(tokio::spawn(async move {
433 pool.run(move || {
434 let memo = Memo::shared(&i2, &seen);
435 for k in lo..hi {
436 let reply = i2.answer_with(&items[k], p.as_ref(), &memo, &now, &mut |_| {}, false);
437 let mut slot = shared[c].lock().unwrap_or_else(|p| p.into_inner());
438 if slot.late {
439 let _ = tx.send((k, reply));
440 } else {
441 slot.answers.push((k, reply));
442 }
443 }
444 let mut slot = shared[c].lock().unwrap_or_else(|p| p.into_inner());
445 if !slot.late {
446 let answers = std::mem::take(&mut slot.answers);
447 slot.written = Some(Written::of(c == 0, answers.iter().map(|(_, r)| r)));
448 }
449 })
450 .await
451 }));
452 }
453 drop(tx);
454 let all = futures_join(&mut tasks, deadline).await;
455 let mut done: Vec<Option<Reply>> = vec![None; n];
456 let mut written: Vec<Option<Written>> = Vec::with_capacity(ranges.len());
457 for slot in shared.iter() {
458 let mut s = slot.lock().unwrap_or_else(|p| p.into_inner());
459 s.late = true;
460 for (k, r) in s.answers.drain(..) {
461 done[k] = Some(r);
462 }
463 written.push(s.written.take());
464 }
465 let runs = |done: Vec<Option<Reply>>, written: Vec<Option<Written>>| -> Vec<Written> {
468 let error = envelope::error_reply(ErrorCode::EvalUnavailable);
469 ranges
470 .iter()
471 .zip(written)
472 .enumerate()
473 .map(|(c, (&(lo, hi), w))| {
474 w.unwrap_or_else(|| Written::of(c == 0, done[lo..hi].iter().map(|d| d.as_ref().unwrap_or(&error))))
475 })
476 .collect()
477 };
478 if all {
479 return runs(done, written);
480 }
481 let open: Vec<usize> = ranges
484 .iter()
485 .zip(&written)
486 .filter(|(_, w)| w.is_none())
487 .flat_map(|(&(lo, hi), _)| lo..hi)
488 .filter(|k| done[*k].is_none())
489 .collect();
490 let (i2, items2, open2, now2) = (inner.clone(), items.clone(), open.clone(), now.clone());
491 let requests: Vec<Option<CoreRequest>> = inner
492 .pool
493 .run(move || {
494 let memo = Memo::new(&i2);
495 open2
496 .iter()
497 .map(|k| {
498 crate::call::resolve(&items2[*k], &now2, &i2.config.aliases, &memo, false)
499 .ok()
500 .and_then(|t| request_of(&t))
501 })
502 .collect()
503 })
504 .await
505 .unwrap_or_default();
506 let poll_base = format!("{}/jobs/", inner.config.origin());
507 let est = inner.jobs.estimate(inner.config.budget.as_secs());
508 let mut jobs: Vec<Option<String>> = vec![None; n];
509 for (i, k) in open.iter().enumerate() {
510 let reply = match requests.get(i).cloned().flatten() {
511 Some(req) => match inner.jobs.create(req.clone()) {
512 Some(id) => {
513 let p = Pending::new(&id, est, &format!("{poll_base}{id}"), req);
514 jobs[*k] = Some(id);
515 match p {
516 Ok(p) => Reply::json(202, envelope::pending(&p), Cache::NoStore),
517 Err(_) => envelope::error_reply(ErrorCode::EvalUnavailable),
518 }
519 }
520 None => envelope::error_reply(ErrorCode::EvalUnavailable),
521 },
522 None => envelope::error_reply(ErrorCode::EvalUnavailable),
523 };
524 done[*k] = Some(reply);
525 }
526 let i3 = inner.clone();
527 tokio::spawn(async move {
528 while let Some((k, mut reply)) = rx.recv().await {
529 if let Some(id) = &jobs[k] {
530 reply.cache = Cache::NoStore;
531 reply.canonical = None;
532 i3.jobs.complete(id, reply);
533 }
534 }
535 for t in tasks {
536 let _ = t.await;
537 }
538 for id in jobs.into_iter().flatten() {
540 if let Poll::Running(..) = i3.jobs.poll_peek(&id) {
541 i3.jobs.complete(&id, envelope::error_reply(ErrorCode::EvalUnavailable));
542 }
543 }
544 });
545 runs(done, written)
546}
547
548#[derive(Default)]
551struct Slot {
552 late: bool,
553 answers: Vec<(usize, Reply)>,
554 written: Option<Written>,
555}
556
557type Slots = Arc<Vec<Mutex<Slot>>>;
559
560async fn futures_join(tasks: &mut [tokio::task::JoinHandle<Option<()>>], deadline: tokio::time::Instant) -> bool {
562 for t in tasks.iter_mut() {
563 if t.is_finished() {
564 continue;
565 }
566 if tokio::time::timeout_at(deadline, t).await.is_err() {
567 return false;
568 }
569 }
570 true
571}
572
573#[utoipa::path(get, path = "/versions/{cell}",
575 params(("cell" = String, Path, description = "A cell URI's rest without asof (a pin allowed), then optionally `from=` and `to=` (instants, inclusive bounds on tx) as its last parameters.")),
576 responses(
577 (status = 200, description = "`{\"cell\", \"versions\": [{\"asof\", \"receipt\", \"tx\"}], \"next\"?}` ascending by tx", content_type = "application/json"),
578 (status = 400, description = "`bad_request`", content_type = "application/json"),
579 (status = 404, description = "`not_covered` (with `E_NAME_UNBOUND` for an unbound name)", content_type = "application/json"),
580 ))]
581async fn versions(State(mesh): State<Mesh>, uri: Uri, headers: HeaderMap, req: Request) -> Response {
582 if let Some(r) = ip_admit(&mesh.inner, &headers, &req) {
583 return r;
584 }
585 let inner = mesh.inner.clone();
586 let rest = rest_of(&uri, "/versions/");
587 let r = inner
588 .pool
589 .clone()
590 .run(move || inner.versions(&rest))
591 .await
592 .unwrap_or_else(|| envelope::error_reply(ErrorCode::EvalUnavailable));
593 to_response(r)
594}
595
596const VERSIONS_PAGE: usize = 1000;
598
599impl Inner {
600 fn versions(&self, rest: &str) -> Reply {
601 let (path, query) = match rest.split_once('?') {
603 Some((p, q)) => (p, q),
604 None => (rest, ""),
605 };
606 let mut ps: Vec<&str> = if query.is_empty() { vec![] } else { query.split('&').collect() };
607 let (mut from, mut to) = (None, None);
608 while let Some(last) = ps.last() {
609 if let Some(v) = last.strip_prefix("from=") {
610 from = Some(v.to_string());
611 } else if let Some(v) = last.strip_prefix("to=") {
612 to = Some(v.to_string());
613 } else {
614 break;
615 }
616 ps.pop();
617 }
618 for t in [&from, &to].into_iter().flatten() {
619 if lexical::check_instant(t, "$").is_err() {
620 return envelope::bad_request_reply(ErrorCode::CanonInstant);
621 }
622 }
623 let cell_rest = if ps.is_empty() { path.to_string() } else { format!("{path}?{}", ps.join("&")) };
624 let now = self.now();
625 let target = match crate::call::resolve(&cell_rest, &now, &self.config.aliases, self, false) {
626 Ok(t) => t,
627 Err(r) => return self.rejection(r, None),
628 };
629 if target.asof().is_some() {
630 return envelope::bad_request_reply(ErrorCode::ParseUri);
631 }
632 let lo = from.unwrap_or_else(|| "0001-01-01T00:00:00.000000Z".into());
633 let hi = to.clone().unwrap_or_else(|| "9999-12-31T23:59:59.999999Z".into());
634 let mut vs = match self.store.versions(target.cell_id(), &lo, &hi) {
635 Ok(v) => v,
636 Err(_) => return envelope::error_answer_reply(ErrorCode::InputUnavailable, Some(&target.bound.fn_hash)),
637 };
638 vs.sort_by(|a, b| (&a.tx, &a.receipt_hash).cmp(&(&b.tx, &b.receipt_hash)));
639 let more = vs.len() > VERSIONS_PAGE;
640 vs.truncate(VERSIONS_PAGE);
641 let list: Vec<Value> = vs
642 .iter()
643 .map(|v| {
644 Value::Object(vec![
645 ("asof".into(), Value::String(v.asof.clone())),
646 ("receipt".into(), Value::String(v.receipt_hash.clone())),
647 ("tx".into(), Value::String(v.tx.clone())),
648 ])
649 })
650 .collect();
651 let mut body = vec![("cell".to_string(), target.cell.to_value()), ("versions".to_string(), Value::Array(list))];
652 if more {
653 if let Some(last) = vs.last() {
654 let us = datumesh_formula::calendar::parse_instant(&last.tx).unwrap_or(0) + 1;
655 let next_from = datumesh_formula::calendar::instant_text(us).unwrap_or_default();
656 let mut next = format!("{}/versions/{cell_rest}", self.config.origin());
657 next.push(if cell_rest.contains('?') { '&' } else { '?' });
658 next.push_str(&format!("from={next_from}"));
659 if let Some(t) = &to {
660 next.push_str(&format!("&to={t}"));
661 }
662 body.push(("next".to_string(), Value::String(next)));
663 }
664 }
665 Reply::json(200, Value::Object(body).to_jcs(), Cache::Public(self.config.ttl.latest_max_age.as_secs()))
666 }
667}
668
669#[utoipa::path(get, path = "/slice/{cell}",
671 params(
672 ("cell" = String, Path, description = "A slice: a cell URI's rest or a call with `*` for open arguments, of a stored function; `?asof=`, `limit=` (JSON, default 1,000, at most 10,000), `after=` (the cursor of the previous page)."),
673 ),
674 responses(
675 (status = 200, description = "JSON page of members, or with `Accept: application/vnd.apache.arrow.stream` an Arrow IPC stream", content_type = "application/json"),
676 (status = 400, description = "`bad_request`", content_type = "application/json"),
677 (status = 404, description = "`not_covered`", content_type = "application/json"),
678 ))]
679async fn slice(State(mesh): State<Mesh>, uri: Uri, headers: HeaderMap, req: Request) -> Response {
680 let inner = mesh.inner.clone();
681 let (client, _) = match who(&inner, &headers, peer_of(&req)) {
682 Ok(x) => x,
683 Err(r) => return to_response(r),
684 };
685 if let Err(r) = admit(&inner, &client, 1) {
686 return to_response(r);
687 }
688 let rest = rest_of(&uri, "/slice/");
690 let arrow = accepts(&headers, crate::slice::ARROW);
691 let r = inner
692 .pool
693 .clone()
694 .run(move || {
695 let now = inner.now();
696 inner.slice(&rest, &now, arrow)
697 })
698 .await
699 .unwrap_or_else(|| envelope::error_reply(ErrorCode::EvalUnavailable));
700 to_response(r)
701}
702
703fn accepts(headers: &HeaderMap, media: &str) -> bool {
704 headers
705 .get_all(header::ACCEPT)
706 .iter()
707 .filter_map(|v| v.to_str().ok())
708 .any(|v| v.split(',').any(|p| p.split(';').next().unwrap_or("").trim() == media))
709}
710
711fn acceptable(headers: &HeaderMap, also: Option<&str>) -> bool {
714 let vals: Vec<&str> = headers.get_all(header::ACCEPT).iter().filter_map(|v| v.to_str().ok()).collect();
715 if vals.is_empty() {
716 return true;
717 }
718 vals.iter().flat_map(|v| v.split(',')).any(|p| {
719 let mut parts = p.split(';');
720 let m = parts.next().unwrap_or("").trim();
721 let q0 = parts.any(|x| {
722 let x = x.trim();
723 x == "q=0" || x == "q=0.0" || x == "q=0.00" || x == "q=0.000"
724 });
725 !q0 && (m == "*/*" || m == "application/*" || m == JSON || also == Some(m))
726 })
727}
728
729async fn object(
730 inner: Arc<Inner>,
731 what: &'static str,
732 f: impl FnOnce(&Inner) -> Option<Vec<u8>> + Send + 'static,
733) -> Response {
734 let i2 = inner.clone();
735 match inner.pool.run(move || f(&i2)).await.flatten() {
736 Some(bytes) => {
737 let ct = if what == "blob" { "application/octet-stream" } else { JSON };
738 to_response(Reply {
739 status: 200,
740 body: bytes.into(),
741 content_type: ct,
742 cache: Cache::Immutable,
743 canonical: None,
744 retry_after: None,
745 })
746 }
747 None => not_found(what),
748 }
749}
750
751fn ip_admit(inner: &Inner, headers: &HeaderMap, req: &Request) -> Option<Response> {
753 let client = match who(inner, headers, peer_of(req)) {
754 Ok((c, _)) => c,
755 Err(r) => return Some(to_response(r)),
756 };
757 admit(inner, &client, 1).err().map(to_response)
758}
759
760#[utoipa::path(get, path = "/receipt/{hash}", params(("hash" = String, Path, description = "The receipt hash (SPEC 8.3).")),
762 responses((status = 200, description = "The receipt's canonical JSON; immutable", content_type = "application/json"), (status = 404, description = "Not held")))]
763async fn receipt(State(mesh): State<Mesh>, Path(hash): Path<String>, headers: HeaderMap, req: Request) -> Response {
764 if let Some(r) = ip_admit(&mesh.inner, &headers, &req) {
765 return r;
766 }
767 if lexical::check_hex(&hash, 64, "$").is_err() {
768 return not_found("receipt");
769 }
770 object(mesh.inner.clone(), "receipt", move |i| i.store.receipt(&hash).map(|c| c.into_owned())).await
771}
772
773#[utoipa::path(get, path = "/descriptor/{hash}", params(("hash" = String, Path, description = "The function hash.")),
775 responses((status = 200, description = "The descriptor's canonical JSON; immutable", content_type = "application/json"), (status = 404, description = "Not held")))]
776async fn descriptor(State(mesh): State<Mesh>, Path(hash): Path<String>, headers: HeaderMap, req: Request) -> Response {
777 if let Some(r) = ip_admit(&mesh.inner, &headers, &req) {
778 return r;
779 }
780 if lexical::check_hex(&hash, 64, "$").is_err() {
781 return not_found("descriptor");
782 }
783 object(mesh.inner.clone(), "descriptor", move |i| i.store.descriptor(&hash).map(|c| c.into_owned())).await
784}
785
786async fn manifest_of(inner: Arc<Inner>, entry: String) -> Response {
787 object(inner, "manifest", move |i| i.store.manifest(&entry).map(|c| c.into_owned())).await
788}
789
790#[utoipa::path(get, path = "/manifest/{slice_id}/{members_hash}",
792 params(("slice_id" = String, Path, description = "The slice id."), ("members_hash" = String, Path, description = "The members hash.")),
793 responses((status = 200, description = "The manifest's canonical JSON; immutable", content_type = "application/json"), (status = 404, description = "Not held")))]
794async fn manifest(
795 State(mesh): State<Mesh>,
796 Path((slice_id, members)): Path<(String, String)>,
797 headers: HeaderMap,
798 req: Request,
799) -> Response {
800 if let Some(r) = ip_admit(&mesh.inner, &headers, &req) {
801 return r;
802 }
803 if lexical::check_hex(&slice_id, 64, "$").is_err() || lexical::check_hex(&members, 64, "$").is_err() {
804 return not_found("manifest");
805 }
806 manifest_of(mesh.inner.clone(), format!("slice:{slice_id}:{members}")).await
807}
808
809#[utoipa::path(get, path = "/blob/{sha256}", params(("sha256" = String, Path, description = "The content's SHA-256.")),
811 responses((status = 200, description = "The bytes; immutable", content_type = "application/octet-stream"), (status = 404, description = "Not held")))]
812async fn blob(State(mesh): State<Mesh>, Path(sha): Path<String>, headers: HeaderMap, req: Request) -> Response {
813 if let Some(r) = ip_admit(&mesh.inner, &headers, &req) {
814 return r;
815 }
816 if lexical::check_hex(&sha, 64, "$").is_err() {
817 return not_found("blob");
818 }
819 object(mesh.inner.clone(), "blob", move |i| i.store.blob(&sha).ok().flatten().map(|c| c.into_owned())).await
820}
821
822#[utoipa::path(get, path = "/explain/{cell}",
824 params(("cell" = String, Path, description = "As for /cell; `depth=` (default 4, at most 16) bounds the walk.")),
825 responses(
826 (status = 200, description = "`{\"envelope\", \"inputs\": [{\"entry\", \"envelope\", \"inputs\"} | {\"entry\", \"slice\", \"members\"}], \"truncated\"}`", content_type = "application/json"),
827 (status = 202, description = "`pending`: the cell is being computed; ask again", content_type = "application/json"),
828 (status = 400, description = "`bad_request`", content_type = "application/json"),
829 (status = 404, description = "`not_covered`", content_type = "application/json"),
830 ))]
831async fn explain(State(mesh): State<Mesh>, uri: Uri, headers: HeaderMap, req: Request) -> Response {
832 let inner = mesh.inner.clone();
833 let (client, principal) = match who(&inner, &headers, peer_of(&req)) {
834 Ok(x) => x,
835 Err(r) => return to_response(r),
836 };
837 if let Err(r) = admit(&inner, &client, 1) {
838 return to_response(r);
839 }
840 let mut rest = rest_of(&uri, "/explain/");
841 let mut depth = inner.config.explain_depth;
842 if let Some((p, q)) = rest.clone().split_once('?') {
844 let kept: Vec<&str> = q.split('&').filter(|x| !x.starts_with("depth=")).collect();
845 if let Some(d) = q.split('&').find_map(|x| x.strip_prefix("depth=")) {
846 depth = d.parse::<usize>().unwrap_or(depth);
847 }
848 rest = if kept.is_empty() { p.to_string() } else { format!("{p}?{}", kept.join("&")) };
849 }
850 let depth = depth.min(16);
851 let r = within_budget(inner, move |i, on| {
852 let now = i.now();
853 let r = i.answer(&rest, principal.as_ref(), i, &now, on);
854 if r.status != 200 {
855 return r;
856 }
857 let env = json::parse(&r.body).unwrap_or(Value::Null);
858 let mut budget = 2_000usize;
859 let v = i.explain_node(env, depth, &mut budget);
860 Reply::json(200, v.to_jcs(), Cache::NoStore)
861 })
862 .await;
863 to_response(r)
864}
865
866impl Inner {
867 fn explain_node(&self, envelope: Value, depth: usize, budget: &mut usize) -> Value {
869 let inputs: Vec<String> = envelope
870 .get("receipt")
871 .and_then(|r| r.get("inputs"))
872 .and_then(|i| {
873 if let Value::Array(a) = i {
874 Some(a.iter().filter_map(Value::as_str).map(str::to_string).collect())
875 } else {
876 None
877 }
878 })
879 .unwrap_or_default();
880 let mut truncated = false;
881 let mut out = Vec::new();
882 for e in inputs {
883 if depth == 0 || *budget == 0 {
884 truncated = true;
885 break;
886 }
887 *budget -= 1;
888 if e.starts_with("slice:") {
889 let m = self.store.manifest(&e).and_then(|m| datumesh_core::Manifest::from_json(&m).ok());
890 let mut node = vec![("entry".to_string(), Value::String(e.clone()))];
891 if let Some(m) = m {
892 node.push(("members".to_string(), envelope::num(m.members().len() as u64)));
893 node.push(("slice".to_string(), m.slice().to_value()));
894 }
895 out.push(Value::Object(node));
896 continue;
897 }
898 let child = match self.receipt_info_by_hash(&e) {
899 Some(info) => self.explain_node(json::parse(&info.ok).unwrap_or(Value::Null), depth - 1, budget),
900 None => json::parse(&envelope::not_covered(None)).unwrap_or(Value::Null),
901 };
902 let mut node = vec![("entry".to_string(), Value::String(e.clone()))];
903 if let Value::Object(m) = child {
904 node.extend(m);
905 }
906 out.push(Value::Object(node));
907 }
908 let mut node = vec![("envelope".to_string(), envelope.clone()), ("inputs".to_string(), Value::Array(out))];
909 node.push(("truncated".to_string(), Value::Bool(truncated)));
910 Value::Object(node)
911 }
912}
913
914#[derive(serde::Deserialize)]
915struct FindQuery {
916 q: Option<String>,
917 limit: Option<usize>,
918}
919
920#[utoipa::path(get, path = "/find",
922 params(("q" = String, Query, description = "Keywords, all of which must appear (case-insensitive) in the function's name or catalog description."), ("limit" = Option<usize>, Query, description = "At most this many results (default 20, at most 100).")),
923 responses((status = 200, description = "`{\"query\", \"results\": [{\"name\", \"fn_hash\", \"since\", \"description\", \"descriptor\"}]}`", content_type = "application/json")))]
924async fn find(State(mesh): State<Mesh>, Query(q): Query<FindQuery>, headers: HeaderMap, req: Request) -> Response {
925 if let Some(r) = ip_admit(&mesh.inner, &headers, &req) {
926 return r;
927 }
928 let inner = mesh.inner.clone();
929 let query = q.q.unwrap_or_default();
930 let limit = q.limit.unwrap_or(20).clamp(1, 100);
931 let r = inner
932 .pool
933 .clone()
934 .run(move || {
935 let words: Vec<String> = query.split_whitespace().map(str::to_lowercase).collect();
936 let now = inner.now();
937 let mut results = Vec::new();
938 for (name, desc) in &inner.config.catalog {
939 let hay = format!("{} {}", name.to_lowercase(), desc.to_lowercase());
940 if !words.iter().all(|w| hay.contains(w.as_str())) {
941 continue;
942 }
943 let Ok(Some(b)) = inner.store.resolve_binding(name, &now) else { continue };
944 let d = inner.store.descriptor(&b.fn_hash).and_then(|d| json::parse(&d).ok()).unwrap_or(Value::Null);
945 results.push(Value::Object(vec![
946 ("description".into(), Value::String(desc.clone())),
947 ("descriptor".into(), d),
948 ("fn_hash".into(), Value::String(b.fn_hash)),
949 ("name".into(), Value::String(name.clone())),
950 ("since".into(), Value::String(b.tx)),
951 ]));
952 if results.len() >= limit {
953 break;
954 }
955 }
956 Value::Object(vec![("query".into(), Value::String(query)), ("results".into(), Value::Array(results))])
957 })
958 .await
959 .unwrap_or(Value::Null);
960 plain_json(200, r, Cache::Public(60))
961}
962
963#[derive(serde::Deserialize)]
964struct VerifyQuery {
965 level: Option<String>,
966 blob: Option<String>,
967 form: Option<String>,
968}
969
970#[utoipa::path(post, path = "/verify",
973 params(("level" = Option<String>, Query, description = "`recomputed` (default: recompute with the engine as the resolver) or `signed` (steps 1-6)."), ("blob" = Option<String>, Query, description = "Blob content for step 7.7, as hex (at most 1 MiB)."), ("form" = Option<String>, Query, description = "`offered`: the body is `{\"receipt\": <its JSON text>, \"recompute\"?: <SPEC 11's recompute object>, \"blob_hex\"?}` and step 7 uses what is offered instead of this mesh's engine.")),
974 request_body(content = String, description = "The receipt's JSON text, exactly as received (or the offered form).", content_type = "application/json"),
975 responses(
976 (status = 200, description = "`{\"ok\", \"level\", \"code\", \"hash\", \"computed_by\", \"path\", \"detail\", \"blob_checked\"}`", content_type = "application/json"),
977 (status = 429, description = "`rate_limited`", content_type = "application/json"),
978 ))]
979async fn verify_receipt(
980 State(mesh): State<Mesh>,
981 Query(q): Query<VerifyQuery>,
982 headers: HeaderMap,
983 req: Request,
984) -> Response {
985 let inner = mesh.inner.clone();
986 let ip = client_ip(&inner, &headers, peer_of(&req));
987 if let Err(s) = inner.limiter.admit_verify(ip) {
988 return to_response(envelope::rate_limited_reply(s));
989 }
990 let body: Bytes = match axum::body::to_bytes(req.into_body(), 4 << 20).await {
991 Ok(b) => b,
992 Err(_) => return batch_error("body too large"),
993 };
994 if q.form.as_deref() == Some("offered") {
995 let Ok(v) = json::parse(&body) else { return batch_error("form=offered: not strict JSON") };
996 let Some(receipt) = v.get("receipt").and_then(Value::as_str).map(|s| s.as_bytes().to_vec()) else {
997 return batch_error("form=offered: no receipt text");
998 };
999 let blob = match v.get("blob_hex").and_then(Value::as_str).map(hex) {
1000 None => None,
1001 Some(Some(b)) if b.len() <= 1 << 20 => Some(b),
1002 Some(_) => return batch_error("blob_hex: hex of at most 1 MiB"),
1003 };
1004 let m = mesh.clone();
1005 let rc = v.get("recompute").cloned();
1006 let r = inner.pool.run(move || m.verify_offered(&receipt, rc.as_ref(), blob.as_deref())).await;
1007 return match r {
1008 Some(v) => to_response(Reply::json(200, v.body, Cache::NoStore)),
1009 None => to_response(envelope::error_reply(ErrorCode::EvalUnavailable)),
1010 };
1011 }
1012 let signed = q.level.as_deref() == Some("signed");
1013 let blob = match q.blob.as_deref().map(hex) {
1014 None => None,
1015 Some(Some(b)) if b.len() <= 1 << 20 => Some(b),
1016 Some(_) => return batch_error("blob: hex of at most 1 MiB"),
1017 };
1018 let m = mesh.clone();
1019 let r = inner.pool.run(move || m.verify(&body, signed, blob.as_deref())).await;
1020 match r {
1021 Some(v) => to_response(Reply::json(200, v.body, Cache::NoStore)),
1022 None => to_response(envelope::error_reply(ErrorCode::EvalUnavailable)),
1023 }
1024}
1025
1026fn hex(s: &str) -> Option<Vec<u8>> {
1027 if !s.len().is_multiple_of(2) {
1028 return None;
1029 }
1030 (0..s.len()).step_by(2).map(|i| u8::from_str_radix(s.get(i..i + 2)?, 16).ok()).collect()
1031}
1032
1033#[utoipa::path(get, path = "/jobs/{job}", params(("job" = String, Path, description = "The job id of a `pending` envelope.")),
1035 responses(
1036 (status = 202, description = "`pending`, the same job and request", content_type = "application/json"),
1037 (status = 400, description = "`bad_request` with `E_JOB_UNKNOWN`: unknown or expired", content_type = "application/json"),
1038 (status = 200, description = "The completed envelope (any outcome but pending, with its status), bound to the original request", content_type = "application/json"),
1039 ))]
1040async fn job(State(mesh): State<Mesh>, Path(id): Path<String>, headers: HeaderMap, req: Request) -> Response {
1041 let inner = mesh.inner.clone();
1042 let (client, _) = match who(&inner, &headers, peer_of(&req)) {
1043 Ok(x) => x,
1044 Err(r) => return to_response(r),
1045 };
1046 if let Err(r) = admit(&inner, &client, 1) {
1047 return to_response(r);
1048 }
1049 let unknown = || to_response(Reply::json(400, envelope::bad_request(ErrorCode::JobUnknown), Cache::NoStore));
1050 let valid = (32..=64).contains(&id.len()) && id.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b));
1051 if !valid {
1052 return unknown();
1053 }
1054 match inner.jobs.poll(&id) {
1055 Poll::Unknown => unknown(),
1056 Poll::Done(r) => to_response(r),
1057 Poll::Running(request, elapsed) => {
1058 let poll = format!("{}/jobs/{id}", inner.config.origin());
1059 match Pending::new(&id, inner.jobs.estimate(elapsed), &poll, request) {
1060 Ok(p) => to_response(Reply::json(202, envelope::pending(&p), Cache::NoStore)),
1061 Err(_) => to_response(envelope::error_reply(ErrorCode::EvalUnavailable)),
1062 }
1063 }
1064 }
1065}
1066
1067#[utoipa::path(get, path = "/.well-known/datumesh.json", responses((status = 200, description = "The key document", content_type = "application/json"), (status = 404, description = "None stored")))]
1069async fn key_document(State(mesh): State<Mesh>) -> Response {
1070 let m = mesh.clone();
1071 match mesh.inner.pool.run(move || m.key_document()).await.flatten() {
1072 Some(doc) => to_response(Reply::json(200, doc, Cache::Public(300))),
1073 None => not_found("key document"),
1074 }
1075}
1076
1077#[utoipa::path(get, path = "/.well-known/datumesh-mesh.json", responses((status = 200, description = "The mesh document", content_type = "application/json")))]
1079async fn mesh_document(State(mesh): State<Mesh>) -> Response {
1080 to_response(Reply::json(200, mesh.mesh_document(), Cache::Public(300)))
1081}
1082
1083#[utoipa::path(get, path = "/bindings",
1085 params(("name" = String, Query, description = "A function name, percent-encoded as in SPEC 5.1; repeat for several."), ("asof" = Option<String>, Query, description = "An instant (default: now).")),
1086 responses((status = 200, description = "`{\"asof\", \"bindings\": {name: {\"fn_hash\", \"since\"}}}`", content_type = "application/json"), (status = 400, description = "A malformed name or asof", content_type = "application/json")))]
1087async fn bindings(State(mesh): State<Mesh>, uri: Uri, headers: HeaderMap, req: Request) -> Response {
1088 if let Some(r) = ip_admit(&mesh.inner, &headers, &req) {
1089 return r;
1090 }
1091 let inner = mesh.inner.clone();
1092 let mut names = Vec::new();
1093 let mut asof = None;
1094 for p in uri.query().unwrap_or("").split('&').filter(|p| !p.is_empty()) {
1095 let (k, v) = p.split_once('=').unwrap_or((p, ""));
1096 let Some(v) = pct_decode(v) else { return batch_error("percent-encoding (SPEC 5.1)") };
1097 match k {
1098 "name" if lexical::check_function_name(&v, "$").is_ok() => names.push(v),
1099 "asof" if lexical::check_instant(&v, "$").is_ok() && asof.is_none() => asof = Some(v),
1100 _ => return batch_error(&format!("parameter {k}")),
1101 }
1102 }
1103 let r = inner
1104 .pool
1105 .clone()
1106 .run(move || {
1107 let at = asof.unwrap_or_else(|| inner.now());
1108 let mut out = Vec::new();
1109 for n in names {
1110 if !inner.config.owns(crate::foreign::host_of(&n)) {
1111 continue;
1112 }
1113 let found = match inner.store.resolve_class_binding(&n, &at) {
1116 Ok(Some(c)) => ("class_hash", c.class_hash, c.tx),
1117 Ok(None) => match inner.store.resolve_binding(&n, &at) {
1118 Ok(Some(b)) => ("fn_hash", b.fn_hash, b.tx),
1119 Ok(None) => continue,
1120 Err(_) => return None,
1121 },
1122 Err(_) => return None,
1123 };
1124 out.push((
1125 n,
1126 Value::Object(vec![
1127 (found.0.into(), Value::String(found.1)),
1128 ("since".into(), Value::String(found.2)),
1129 ]),
1130 ));
1131 }
1132 Some(Value::Object(vec![("asof".into(), Value::String(at)), ("bindings".into(), Value::Object(out))]))
1133 })
1134 .await
1135 .flatten();
1136 match r {
1137 Some(v) => plain_json(200, v, Cache::Public(inner_latest(&mesh))),
1138 None => to_response(envelope::error_reply(ErrorCode::InputUnavailable)),
1139 }
1140}
1141
1142fn inner_latest(mesh: &Mesh) -> u64 {
1143 mesh.inner.config.ttl.latest_max_age.as_secs()
1144}
1145
1146fn pct_decode(s: &str) -> Option<String> {
1149 let b = s.as_bytes();
1150 let mut out = Vec::new();
1151 let mut i = 0;
1152 while i < b.len() {
1153 let c = b[i];
1154 if c == b'%' {
1155 let h = b.get(i + 1..i + 3)?;
1156 if !h.iter().all(|x| x.is_ascii_digit() || (b'A'..=b'F').contains(x)) {
1157 return None;
1158 }
1159 let v = u8::from_str_radix(std::str::from_utf8(h).ok()?, 16).ok()?;
1160 if v.is_ascii_alphanumeric() || matches!(v, b'-' | b'.' | b'_' | b'~' | b':') {
1161 return None;
1162 }
1163 out.push(v);
1164 i += 3;
1165 } else if c.is_ascii_alphanumeric() || matches!(c, b'-' | b'.' | b'_' | b'~' | b':') {
1166 out.push(c);
1167 i += 1;
1168 } else {
1169 return None;
1170 }
1171 }
1172 String::from_utf8(out).ok()
1173}
1174
1175#[utoipa::path(get, path = "/health", responses((status = 200, description = "`{\"status\": \"ok\"}`", content_type = "application/json"), (status = 503, description = "The store does not answer", content_type = "application/json")))]
1177async fn health(State(mesh): State<Mesh>) -> Response {
1178 let m = mesh.clone();
1179 let ok = tokio::time::timeout(
1180 Duration::from_secs(5),
1181 mesh.inner.pool.run(move || m.inner.store.namespace(&m.inner.config.host).is_ok()),
1182 )
1183 .await
1184 .ok()
1185 .flatten()
1186 .unwrap_or(false);
1187 let v = Value::Object(vec![("status".into(), Value::String(if ok { "ok" } else { "store unavailable" }.into()))]);
1188 plain_json(if ok { 200 } else { 503 }, v, Cache::NoStore)
1189}
1190
1191#[utoipa::path(get, path = "/metrics", responses((status = 200, description = "Prometheus text", content_type = "text/plain")))]
1193async fn metrics(State(mesh): State<Mesh>) -> Response {
1194 let mut r = Reply::json(200, metrics_text(&mesh.inner).into_bytes(), Cache::NoStore);
1195 r.content_type = "text/plain; version=0.0.4";
1196 to_response(r)
1197}
1198
1199#[utoipa::path(get, path = "/openapi.json", responses((status = 200, description = "The OpenAPI document", content_type = "application/json")))]
1201async fn openapi_json() -> Response {
1202 let mut r = Reply::json(200, openapi().to_json().unwrap_or_default().into_bytes(), Cache::Public(300));
1203 r.content_type = JSON;
1204 to_response(r)
1205}
1206
1207pub(crate) fn metrics_text(inner: &Inner) -> String {
1208 use std::fmt::Write;
1209 use std::sync::atomic::Ordering::Relaxed;
1210 let mut out = String::new();
1211 inner.metrics.write(&mut out);
1212 let p = &inner.pool;
1213 let g = |out: &mut String, name: &str, help: &str, kind: &str, v: String| {
1214 let _ = writeln!(out, "# HELP {name} {help}\n# TYPE {name} {kind}\n{name} {v}");
1215 };
1216 g(&mut out, "datumesh_pool_size", "The blocking pool's size: the concurrency limit.", "gauge", p.size.to_string());
1217 g(&mut out, "datumesh_pool_in_use", "Pool threads busy now.", "gauge", p.in_use.load(Relaxed).to_string());
1218 g(&mut out, "datumesh_pool_peak", "The most pool threads busy at once.", "gauge", p.peak.load(Relaxed).to_string());
1219 g(&mut out, "datumesh_pool_runs_total", "Tasks run on the pool.", "counter", p.runs.load(Relaxed).to_string());
1220 g(
1221 &mut out,
1222 "datumesh_pool_waited_total",
1223 "Tasks that found the pool saturated and waited for a thread.",
1224 "counter",
1225 p.waited.load(Relaxed).to_string(),
1226 );
1227 let (held, running) = inner.jobs.counts();
1228 g(&mut out, "datumesh_jobs_held", "Jobs held (running or done).", "gauge", held.to_string());
1229 g(&mut out, "datumesh_jobs_running", "Jobs running.", "gauge", running.to_string());
1230 g(
1231 &mut out,
1232 "datumesh_jobs_created_total",
1233 "Requests answered pending.",
1234 "counter",
1235 inner.jobs.created.load(Relaxed).to_string(),
1236 );
1237 g(
1238 &mut out,
1239 "datumesh_rate_limited_total",
1240 "Requests refused by a rate limit.",
1241 "counter",
1242 inner.limiter.refused.load(Relaxed).to_string(),
1243 );
1244 g(
1245 &mut out,
1246 "datumesh_peer_fetches_total",
1247 "Requests to other meshes.",
1248 "counter",
1249 inner.foreign.fetches.load(Relaxed).to_string(),
1250 );
1251 g(
1252 &mut out,
1253 "datumesh_peer_failures_total",
1254 "Requests to other meshes that failed.",
1255 "counter",
1256 inner.foreign.failures.load(Relaxed).to_string(),
1257 );
1258 g(
1259 &mut out,
1260 "datumesh_peer_imported_total",
1261 "Foreign versions imported into the store.",
1262 "counter",
1263 inner.foreign.imported.load(Relaxed).to_string(),
1264 );
1265 let m = inner.engine.memo_stats();
1266 g(&mut out, "datumesh_memo_entries", "Engine memo entries.", "gauge", m.entries.to_string());
1267 g(&mut out, "datumesh_memo_bytes", "Engine memo size, approximately.", "gauge", m.approx_bytes.to_string());
1268 g(
1269 &mut out,
1270 "datumesh_uptime_seconds",
1271 "Seconds since the mesh started.",
1272 "gauge",
1273 inner.started.elapsed().as_secs().to_string(),
1274 );
1275 out
1276}
1277
1278async fn around(State(mesh): State<Mesh>, req: Request, next: Next) -> Response {
1280 let start = Instant::now();
1281 let route = req.extensions().get::<MatchedPath>().map(|m| m.as_str().to_string()).unwrap_or_else(|| "other".into());
1282 let route = match route.as_str() {
1283 "/cell/{*rest}" => "/cell/{cell}".to_string(),
1284 "/slice/{*rest}" => "/slice/{cell}".to_string(),
1285 "/explain/{*rest}" => "/explain/{cell}".to_string(),
1286 "/versions/{*rest}" => "/versions/{cell}".to_string(),
1287 _ => route,
1288 };
1289 let id = req
1290 .headers()
1291 .get("x-request-id")
1292 .and_then(|v| v.to_str().ok())
1293 .filter(|s| !s.is_empty() && s.len() <= 64 && s.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-'))
1294 .map(str::to_string)
1295 .unwrap_or_else(|| {
1296 let mut b = [0u8; 8];
1297 let _ = getrandom::fill(&mut b);
1298 lexical::to_hex(&b)
1299 });
1300 let ip = client_ip(&mesh.inner, req.headers(), peer_of(&req));
1301 let method = req.method().clone();
1302 let also = if route == "/slice/{cell}" { Some(crate::slice::ARROW) } else { None };
1303 let mut resp = if route != "/blob/{sha256}" && route != "/metrics" && !acceptable(req.headers(), also) {
1304 let mut r = Reply::json(
1305 406,
1306 Value::Object(vec![("error".into(), Value::String("not acceptable: application/json only".into()))])
1307 .to_jcs(),
1308 Cache::NoStore,
1309 );
1310 r.content_type = JSON;
1311 to_response(r)
1312 } else {
1313 next.run(req).instrument(tracing::info_span!("request", request_id = %id)).await
1314 };
1315 let elapsed = start.elapsed();
1316 let status = resp.status().as_u16();
1317 mesh.inner.metrics.record(&route, status, elapsed);
1318 let principal = resp.extensions().get::<Who>().and_then(|w| w.0.clone());
1319 tracing::info!(target: "datumesh_http::access", request_id = %id, %method, route = %route, status, ms = elapsed.as_secs_f64() * 1000.0, ip = %ip, principal = principal.as_deref().unwrap_or("-"), "request");
1320 if let Ok(v) = HeaderValue::from_str(&id) {
1321 resp.headers_mut().insert("x-request-id", v);
1322 }
1323 resp.headers_mut().insert(header::VARY, HeaderValue::from_static("Accept, Accept-Encoding, Authorization"));
1324 if let Ok(v) = HeaderValue::from_str(&format!("app;dur={:.3}", elapsed.as_secs_f64() * 1000.0)) {
1325 resp.headers_mut().insert("server-timing", v);
1326 }
1327 resp
1328}
1329
1330#[utoipa::path(get, path = "/describe/{id}",
1333 params(("id" = String, Path, description = "A class name, an object's key encoding, an object id, a cell id or a short id (12 hex digits).")),
1334 responses((status = 200, description = "A describe-class or describe-object document (SPEC 16.8)", content_type = "application/json"), (status = 400, description = "`E_KEY_ENCODING`, `E_KEY_NORMALISATION` or `E_SHORT_ID_AMBIGUOUS`", content_type = "application/json"), (status = 404, description = "An id the mesh does not know", content_type = "application/json")))]
1335async fn describe(State(mesh): State<Mesh>, uri: Uri, headers: HeaderMap, req: Request) -> Response {
1336 if let Some(r) = ip_admit(&mesh.inner, &headers, &req) {
1337 return r;
1338 }
1339 let Some(id) = crate::call::url_decode(uri.path().strip_prefix("/describe/").unwrap_or("")) else {
1340 return to_response(envelope::bad_request_reply(ErrorCode::CanonPercent));
1341 };
1342 let inner = mesh.inner.clone();
1343 match inner.pool.clone().run(move || mesh.describe(&id)).await {
1344 Some(r) => to_response(r),
1345 None => to_response(envelope::error_reply(ErrorCode::EvalUnavailable)),
1346 }
1347}
1348
1349async fn fallback() -> Response {
1350 not_found("endpoint")
1351}
1352
1353impl Mesh {
1354 pub fn router(&self) -> Router {
1357 Router::new()
1358 .route("/cell/{*rest}", get(cell))
1359 .route("/cells", post(cells))
1360 .route("/slice/{*rest}", get(slice))
1361 .route("/receipt/{hash}", get(receipt))
1362 .route("/descriptor/{hash}", get(descriptor))
1363 .route("/manifest/{slice_id}/{members_hash}", get(manifest))
1364 .route("/versions/{*rest}", get(versions))
1365 .route("/blob/{sha256}", get(blob))
1366 .route("/explain/{*rest}", get(explain))
1367 .route("/find", get(find))
1368 .route("/verify", post(verify_receipt))
1369 .route("/jobs/{job}", get(job))
1370 .route("/.well-known/datumesh.json", get(key_document))
1371 .route("/.well-known/datumesh-mesh.json", get(mesh_document))
1372 .route("/bindings", get(bindings))
1373 .route("/describe/{*id}", get(describe))
1374 .route("/health", get(health))
1375 .route("/metrics", get(metrics))
1376 .route("/openapi.json", get(openapi_json))
1377 .fallback(fallback)
1378 .layer(middleware::from_fn_with_state(self.clone(), around))
1379 .layer(tower_http::compression::CompressionLayer::new().gzip(true))
1380 .with_state(self.clone())
1381 }
1382
1383 pub async fn serve(
1385 &self,
1386 listener: tokio::net::TcpListener,
1387 shutdown: impl std::future::Future<Output = ()> + Send + 'static,
1388 ) -> std::io::Result<()> {
1389 axum::serve(listener, self.router().into_make_service_with_connect_info::<SocketAddr>())
1390 .with_graceful_shutdown(shutdown)
1391 .await
1392 }
1393}
1394
1395impl IntoResponse for Reply {
1396 fn into_response(self) -> Response {
1397 to_response(self)
1398 }
1399}
1400
1401#[cfg(test)]
1402mod tests {
1403 use super::*;
1404
1405 fn reply(status: u16, body: &str) -> Reply {
1406 Reply::json(status, body.as_bytes().to_vec(), Cache::NoStore)
1407 }
1408
1409 #[test]
1412 fn runs_make_the_batch() {
1413 let h = "a".repeat(64);
1414 let replies = [
1415 reply(200, &format!(r#"{{"fn_hash":"{h}","outcome":"ok","receipt":{{"value":"1"}},"trust":"primary"}}"#)),
1416 reply(400, r#"{"error":"E_CANON_NUMBER","outcome":"bad_request"}"#),
1417 reply(
1418 200,
1419 &format!(r#"{{"age_seconds":3,"fn_hash":"{h}","outcome":"stale","receipt":{{}},"trust":"computed"}}"#),
1420 ),
1421 reply(404, r#"{"error":"E_NAME_UNBOUND","outcome":"not_covered"}"#),
1422 reply(200, &format!(r#"{{"fn_hash":"{h}","outcome":"ok","receipt":{{"value":"2"}},"trust":"primary"}}"#)),
1423 ];
1424 let mut want = b"[".to_vec();
1425 for (i, r) in replies.iter().enumerate() {
1426 if i > 0 {
1427 want.push(b',');
1428 }
1429 want.extend_from_slice(b"{\"envelope\":");
1430 want.extend_from_slice(&r.body);
1431 want.extend_from_slice(format!(",\"status\":{}}}", r.status).as_bytes());
1432 }
1433 want.push(b']');
1434 for split in 1..replies.len() {
1435 let runs = [Written::of(true, replies[..split].iter()), Written::of(false, replies[split..].iter())];
1436 let mut got: Vec<u8> = runs.iter().flat_map(|w| w.bytes.iter().copied()).collect();
1437 got.push(b']');
1438 assert_eq!(String::from_utf8(got).unwrap(), String::from_utf8(want.clone()).unwrap());
1439 let mut counts = std::collections::BTreeMap::new();
1440 for (o, k) in runs.iter().flat_map(|w| w.outcomes.iter()) {
1441 *counts.entry(o.as_str()).or_insert(0) += k;
1442 }
1443 let want_counts: std::collections::BTreeMap<&str, u64> =
1444 [("bad_request", 1), ("not_covered", 1), ("ok", 2), ("stale", 1)].into_iter().collect();
1445 assert_eq!(counts, want_counts);
1446 }
1447 for (n, s) in [(0u16, "0"), (7, "7"), (200, "200"), (404, "404"), (65535, "65535")] {
1448 let mut out = Vec::new();
1449 push_u16(&mut out, n);
1450 assert_eq!(out, s.as_bytes());
1451 }
1452 }
1453
1454 #[test]
1456 fn outcomes() {
1457 let h = "0".repeat(64);
1458 for (body, want) in [
1459 (format!(r#"{{"fn_hash":"{h}","outcome":"ok","receipt":{{"outcome":"x"}}}}"#), Some("ok")),
1460 (r#"{"age_seconds":12,"outcome":"stale"}"#.to_string(), Some("stale")),
1461 (r#"{"error":"E_PARSE_URI","outcome":"bad_request"}"#.to_string(), Some("bad_request")),
1462 (r#"{"outcome":"pending","poll":"https://a.example/jobs/1"}"#.to_string(), Some("pending")),
1463 (r#"{"error":"no such endpoint"}"#.to_string(), None),
1464 (r#"{"outcome""#.to_string(), None),
1465 ] {
1466 assert_eq!(reply(200, &body).outcome_str(), want, "{body}");
1467 assert_eq!(reply(200, &body).outcome(), want.map(str::to_string), "{body}");
1468 }
1469 let mut arrow = reply(200, r#"{"outcome":"ok"}"#);
1470 arrow.content_type = "application/vnd.apache.arrow.stream";
1471 assert_eq!(arrow.outcome(), None);
1472 }
1473}