Skip to main content

server/
http.rs

1//! Thin HTTP wrapper over [`SharedDb`]. Every endpoint is a lock, a public
2//! core-api call, then a response — no business logic.
3//!
4//! # Single-sink design
5//!
6//! The HTTP router is the designated broadcast producer. [`router`] installs
7//! one `broadcast::Sender` as the [`mushroomdb::GraphDb`] event sink. MCP and
8//! CLI mutations on the same [`SharedDb`] fire into that same sink (one
9//! producer, many `/watch` subscribers). A second [`router`] call replaces
10//! the sink and terminates every existing subscriber with
11//! [`tokio::sync::broadcast::error::RecvError::Closed`].
12
13use crate::json::{
14    edge_history_result_json, namespace_arg, node_edges_json, node_history_json, node_info_json,
15    params_from_json, parse_ingest_edges, result_set_json, rule_def_from_json, stamp_namespace,
16    stamp_namespace_row,
17};
18use crate::{AppState, AuthIdentity, ReadOnlyKey};
19use arrow_bridge::to_ipc_bytes;
20use axum::extract::{Extension, Path, Query, Request, State};
21use axum::http::{header, HeaderMap, HeaderValue, Method, StatusCode};
22use axum::middleware::{self, Next};
23use axum::response::{IntoResponse, Response};
24use axum::routing::{get, post};
25use axum::{Json, Router};
26use mushroomdb::{
27    is_write_query, json_to_rows, json_to_value, AccessKeys, AsOfScope, AutoFk, BackupReport,
28    BatchOp, DegreeConfig, Dir, GraphError, IngestOptions, KeyTier, MaskMode, NodeMask,
29    PageRankConfig, PropPredicate, ReaderSnapshot, ResultSet, SchemaError, SchemaErrorKind,
30    SharedDb, SuggestConfig, Value, WccConfig, SUGGEST_DEFAULT_SEED,
31};
32use serde_json::{json, Value as Js};
33use std::collections::{BTreeMap, HashMap};
34use std::fmt::{Display, Write as _};
35use std::net::SocketAddr;
36use std::path::PathBuf;
37use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
38use std::sync::{Arc, PoisonError};
39use tower_http::services::ServeDir;
40
41// The `/tx/*` routes and the open-transaction store. Declared here, not in
42// `lib.rs`, so the module reaches this file's private helpers (`graph_err`,
43// `forbidden`, ...) instead of widening each of them to `pub(crate)`.
44#[path = "tx.rs"]
45mod tx;
46pub(crate) use tx::TxStore;
47pub use tx::{TxLimits, TX_MAX_ENV, TX_MAX_LIFETIME_ENV, TX_TIMEOUT_ENV};
48
49/// Build the HTTP router over `db`. The graph read endpoints (`/query`,
50/// `/node/{key}`, its `/edges` and `/neighborhood`), `/health`, `/metrics` and
51/// `/algo/*` run on a lock-free [`ReaderSnapshot`], whose capture never waits
52/// behind a writer (see [`SharedDb::reader`]). `/stats`, `/explain`,
53/// `/suggest`, `/find_similar` and the history routes read the rule engine or
54/// the WAL and take the read lock; `/ingest` takes the write lock. Guards are
55/// dropped before any `.await`.
56/// `GET /watch` upgrades to a WebSocket fed by the post-commit sink.
57///
58/// **Call at most once per [`SharedDb`].** A second call replaces the sink
59/// and terminates all existing `/watch` subscribers with
60/// [`tokio::sync::broadcast::error::RecvError::Closed`].
61///
62/// Installing the watch sink replaces any previously installed
63/// [`mushroomdb::GraphDb::set_event_sink`]. The sink only
64/// `broadcast::Sender::send`s (non-blocking) and never re-enters `db`.
65///
66/// # Blocking
67///
68/// [`SharedDb`] uses a std [`std::sync::RwLock`]. Write handlers
69/// (`query_write`, `/ingest`, `create_rule`) run in
70/// `tokio::task::spawn_blocking` and drop the write guard before `.await`.
71/// Reads stay on the worker (neighborhood is µs). `suggest` and `algo`
72/// already use the blocking pool. A live (non-`as_of`) graph read holds no
73/// guard while it executes, so a long read never holds the writer up, and an
74/// indexed equality lookup is served from the snapshot's own index.
75///
76/// Every `router_*` entry point reads `MUSHROOMDB_CORS_ORIGINS` (except
77/// [`router_with_cors`], which takes the layer explicitly). Fail-fast on an
78/// invalid value holds only for the CLI; a library caller gets CORS off and an
79/// error log. Validate with [`crate::cors_layer_from_env`] first to fail early.
80pub fn router(db: SharedDb) -> Router {
81    router_with_auth(db, None)
82}
83
84/// [`router`] with an optional bearer/`?token=` requirement on every route
85/// except unauthenticated `GET /health`.
86///
87/// Role enforcement is not active on this entry point; use
88/// [`router_with_role_tokens`] when role-bound tokens are required.
89pub fn router_with_auth(db: SharedDb, token: Option<String>) -> Router {
90    build_app(
91        db,
92        token,
93        HashMap::new(),
94        UiFallback::None,
95        default_advertise_addr(),
96        false,
97    )
98}
99
100/// [`router_with_auth`] whose `GET /ready` follows `ready`: 503 while it is
101/// false, 200 once it is true. Every other router is ready as soon as it is built.
102pub fn router_with_ready_flag(
103    db: SharedDb,
104    token: Option<String>,
105    ready: Arc<AtomicBool>,
106) -> Router {
107    build_app_with_ready(
108        db,
109        token,
110        HashMap::new(),
111        UiFallback::None,
112        default_advertise_addr(),
113        false,
114        ready,
115        None,
116    )
117}
118
119/// [`router_with_role_tokens`] plus the `keys.json` set (see [`AccessKeys`]).
120///
121/// An `admin` key is full access, a `write` key may mutate but is refused
122/// `/backup`, `/stats`, `/metrics`, `/watch` and `/subscribe`, and a `read` key
123/// is the role it names. The env token and `role_tokens` keep working beside
124/// them. `keys` is shared, so a reload of it (see [`crate::watch_sighup`])
125/// reaches every router built from it.
126pub fn router_with_access_keys(
127    db: SharedDb,
128    token: Option<String>,
129    role_tokens: HashMap<String, String>,
130    keys: AccessKeys,
131) -> Router {
132    build_app_with_ready(
133        db,
134        token,
135        role_tokens,
136        UiFallback::None,
137        default_advertise_addr(),
138        false,
139        Arc::new(AtomicBool::new(true)),
140        Some(keys),
141    )
142}
143
144/// [`router_with_auth`] with an explicit CORS layer (see [`crate::parse_cors_origins`]).
145/// `None` adds no CORS headers. The other routers read `MUSHROOMDB_CORS_ORIGINS`
146/// instead; this one ignores the environment.
147pub fn router_with_cors(
148    db: SharedDb,
149    token: Option<String>,
150    cors: Option<tower_http::cors::CorsLayer>,
151) -> Router {
152    build_app_with_cors(
153        db,
154        token,
155        HashMap::new(),
156        UiFallback::None,
157        default_advertise_addr(),
158        false,
159        Arc::new(AtomicBool::new(true)),
160        cors,
161        None,
162        TxStore::new(TxLimits::from_env()),
163    )
164}
165
166/// [`router_with_access_keys`] with the open-transaction limits given instead
167/// of read from `MUSHROOMDB_TX_MAX` and `MUSHROOMDB_TX_TIMEOUT_SECS`. For an
168/// embedder that serves several routers, and for tests that need a short idle
169/// timeout.
170pub fn router_with_tx_limits(
171    db: SharedDb,
172    token: Option<String>,
173    role_tokens: HashMap<String, String>,
174    keys: Option<AccessKeys>,
175    limits: TxLimits,
176) -> Router {
177    build_app_with_ready_and_txs(
178        db,
179        token,
180        role_tokens,
181        UiFallback::None,
182        default_advertise_addr(),
183        false,
184        Arc::new(AtomicBool::new(true)),
185        keys,
186        TxStore::new(limits),
187    )
188}
189
190/// [`router`] with a full-access token and a map of role-bound tokens.
191///
192/// Role-bound tokens (`role_tokens`: bearer → role name) receive masked reads
193/// on `/query` and `/node/*`, and 403 on all write or subscription endpoints.
194/// Unknown token: 401.  Token bound to a role not in the DB at request time: 401.
195pub fn router_with_role_tokens(
196    db: SharedDb,
197    token: Option<String>,
198    role_tokens: HashMap<String, String>,
199) -> Router {
200    build_app(
201        db,
202        token,
203        role_tokens,
204        UiFallback::None,
205        default_advertise_addr(),
206        false,
207    )
208}
209
210/// Same as [`router_with_auth`], then `ServeDir` as the fallback so API routes win.
211///
212/// Role enforcement is not active on this entry point; use
213/// [`router_with_role_tokens`] when role-bound tokens are required.
214pub fn router_with_ui(
215    db: SharedDb,
216    ui_dir: impl AsRef<std::path::Path>,
217    token: Option<String>,
218) -> Router {
219    build_app(
220        db,
221        token,
222        HashMap::new(),
223        UiFallback::Dir(ui_dir.as_ref().to_path_buf()),
224        default_advertise_addr(),
225        false,
226    )
227}
228
229/// Like [`router_with_ui`] but marks the connection as TLS-active so the auth
230/// cookie carries the `Secure` attribute. Use when the binary is serving
231/// directly over HTTPS (i.e. via `serve_tls`) rather than behind a proxy.
232pub fn router_with_ui_tls(
233    db: SharedDb,
234    ui_dir: impl AsRef<std::path::Path>,
235    token: Option<String>,
236) -> Router {
237    build_app(
238        db,
239        token,
240        HashMap::new(),
241        UiFallback::Dir(ui_dir.as_ref().to_path_buf()),
242        default_advertise_addr(),
243        true,
244    )
245}
246
247#[cfg(feature = "embed-ui")]
248static EMBEDDED_UI: include_dir::Dir<'_> =
249    include_dir::include_dir!("$CARGO_MANIFEST_DIR/../../ui/dist");
250
251/// [`router`] plus the `embed-ui` static tree as fallback.
252#[cfg(feature = "embed-ui")]
253pub fn router_with_embedded_ui(db: SharedDb) -> Router {
254    build_app(
255        db,
256        None,
257        HashMap::new(),
258        UiFallback::Embedded,
259        default_advertise_addr(),
260        false,
261    )
262}
263
264#[cfg(feature = "embed-ui")]
265async fn embedded_fallback(uri: axum::http::Uri) -> Response {
266    let rel = if uri.path() == "/" || uri.path().is_empty() {
267        "index.html"
268    } else {
269        uri.path().trim_start_matches('/')
270    };
271    if rel.split('/').any(|seg| seg == "..") {
272        return StatusCode::NOT_FOUND.into_response();
273    }
274    match EMBEDDED_UI.get_file(rel) {
275        Some(file) => (
276            StatusCode::OK,
277            [(header::CONTENT_TYPE, embedded_ctype(rel))],
278            file.contents(),
279        )
280            .into_response(),
281        None => StatusCode::NOT_FOUND.into_response(),
282    }
283}
284
285#[cfg(feature = "embed-ui")]
286fn embedded_ctype(path: &str) -> &'static str {
287    if path.ends_with(".html") {
288        "text/html; charset=utf-8"
289    } else if path.ends_with(".js") {
290        "application/javascript; charset=utf-8"
291    } else if path.ends_with(".css") {
292        "text/css; charset=utf-8"
293    } else if path.ends_with(".woff2") {
294        "font/woff2"
295    } else if path.ends_with(".svg") {
296        "image/svg+xml"
297    } else if path.ends_with(".ico") {
298        "image/x-icon"
299    } else if path.ends_with(".txt") {
300        "text/plain; charset=utf-8"
301    } else {
302        "application/octet-stream"
303    }
304}
305
306/// Bind `addr` (port 0 is ephemeral) and serve.
307///
308/// Sends the resolved local address on `ready` once the listener is accepting.
309/// Does not hold a database lock.
310///
311/// Role enforcement is not active on this entry point; use
312/// [`serve_with_role_tokens`] when role-bound tokens are required.
313#[deprecated(
314    since = "0.2.0",
315    note = "Use `serve_with_role_tokens` instead; this variant silently ignores role-token configuration."
316)]
317#[doc(hidden)]
318pub async fn serve(
319    db: SharedDb,
320    addr: SocketAddr,
321    ready: tokio::sync::oneshot::Sender<SocketAddr>,
322    token: Option<String>,
323) -> std::io::Result<()> {
324    serve_inner(
325        db,
326        addr,
327        ready,
328        UiFallback::None,
329        token,
330        HashMap::new(),
331        None,
332        Arc::new(AtomicBool::new(true)),
333        None,
334        TxLimits::from_env(),
335    )
336    .await
337}
338
339/// [`serve`] with role-bound tokens in addition to the optional full-access token.
340///
341/// `role_tokens` maps bearer values to role names.  See [`router_with_role_tokens`]
342/// for the enforcement semantics.
343pub async fn serve_with_role_tokens(
344    db: SharedDb,
345    addr: SocketAddr,
346    ready: tokio::sync::oneshot::Sender<SocketAddr>,
347    token: Option<String>,
348    role_tokens: HashMap<String, String>,
349) -> std::io::Result<()> {
350    serve_inner(
351        db,
352        addr,
353        ready,
354        UiFallback::None,
355        token,
356        role_tokens,
357        None,
358        Arc::new(AtomicBool::new(true)),
359        None,
360        TxLimits::from_env(),
361    )
362    .await
363}
364
365/// [`serve`] plus a UI dist directory mounted behind the API routes.
366///
367/// Role enforcement is not active on this entry point; use
368/// [`serve_with_ui_and_role_tokens`] when role-bound tokens are required.
369#[deprecated(
370    since = "0.2.0",
371    note = "Use `serve_with_ui_and_role_tokens` instead; this variant silently ignores role-token configuration."
372)]
373#[doc(hidden)]
374pub async fn serve_with_ui(
375    db: SharedDb,
376    addr: SocketAddr,
377    ready: tokio::sync::oneshot::Sender<SocketAddr>,
378    ui_dir: PathBuf,
379    token: Option<String>,
380) -> std::io::Result<()> {
381    serve_inner(
382        db,
383        addr,
384        ready,
385        UiFallback::Dir(ui_dir),
386        token,
387        HashMap::new(),
388        None,
389        Arc::new(AtomicBool::new(true)),
390        None,
391        TxLimits::from_env(),
392    )
393    .await
394}
395
396/// [`serve_with_ui`] with role-bound tokens.
397pub async fn serve_with_ui_and_role_tokens(
398    db: SharedDb,
399    addr: SocketAddr,
400    ready: tokio::sync::oneshot::Sender<SocketAddr>,
401    ui_dir: PathBuf,
402    token: Option<String>,
403    role_tokens: HashMap<String, String>,
404) -> std::io::Result<()> {
405    serve_inner(
406        db,
407        addr,
408        ready,
409        UiFallback::Dir(ui_dir),
410        token,
411        role_tokens,
412        None,
413        Arc::new(AtomicBool::new(true)),
414        None,
415        TxLimits::from_env(),
416    )
417    .await
418}
419
420/// [`serve`] plus the compiled-in UI (no-op fallback if `embed-ui` is off).
421///
422/// `role_tokens` maps bearer values to role names; see [`router_with_role_tokens`]
423/// for enforcement semantics.  Pass `HashMap::new()` when no role tokens are needed.
424#[cfg(feature = "embed-ui")]
425pub async fn serve_with_embedded_ui(
426    db: SharedDb,
427    addr: SocketAddr,
428    ready: tokio::sync::oneshot::Sender<SocketAddr>,
429    token: Option<String>,
430    role_tokens: HashMap<String, String>,
431) -> std::io::Result<()> {
432    serve_inner(
433        db,
434        addr,
435        ready,
436        UiFallback::Embedded,
437        token,
438        role_tokens,
439        None,
440        Arc::new(AtomicBool::new(true)),
441        None,
442        TxLimits::from_env(),
443    )
444    .await
445}
446
447/// The graceful-shutdown variants, added in 0.6.12.
448///
449/// Each mirrors the entry point above it and takes one extra argument: a
450/// receiver that, when it fires **or is dropped**, stops the server accepting
451/// new connections and lets every request already in flight finish. Without
452/// one, the only way to stop a server was to abort the task running it, which
453/// severs in-flight responses and live `/watch` and `/subscribe` subscribers
454/// mid-stream.
455///
456/// These are additive on purpose. The four originals keep their signatures and
457/// their behaviour, so nothing embedding this crate has to change; they simply
458/// pass no signal. The cost is eight entry points where four would do, and that
459/// was the trade chosen over a breaking parameter.
460pub async fn serve_with_shutdown(
461    db: SharedDb,
462    addr: SocketAddr,
463    ready: tokio::sync::oneshot::Sender<SocketAddr>,
464    token: Option<String>,
465    shutdown: tokio::sync::oneshot::Receiver<()>,
466) -> std::io::Result<()> {
467    serve_inner(
468        db,
469        addr,
470        ready,
471        UiFallback::None,
472        token,
473        HashMap::new(),
474        Some(shutdown),
475        Arc::new(AtomicBool::new(true)),
476        None,
477        TxLimits::from_env(),
478    )
479    .await
480}
481
482/// [`serve_with_role_tokens`] that stops gracefully. See [`serve_with_shutdown`].
483pub async fn serve_with_role_tokens_and_shutdown(
484    db: SharedDb,
485    addr: SocketAddr,
486    ready: tokio::sync::oneshot::Sender<SocketAddr>,
487    token: Option<String>,
488    role_tokens: HashMap<String, String>,
489    shutdown: tokio::sync::oneshot::Receiver<()>,
490) -> std::io::Result<()> {
491    serve_with_role_tokens_and_shutdown_with_ready_flag(
492        db,
493        addr,
494        ready,
495        token,
496        role_tokens,
497        shutdown,
498        Arc::new(AtomicBool::new(true)),
499        None,
500        TxLimits::from_env(),
501    )
502    .await
503}
504
505/// [`serve_with_role_tokens_and_shutdown`] whose `GET /ready` follows `ready_flag`.
506#[allow(clippy::too_many_arguments)]
507pub async fn serve_with_role_tokens_and_shutdown_with_ready_flag(
508    db: SharedDb,
509    addr: SocketAddr,
510    ready: tokio::sync::oneshot::Sender<SocketAddr>,
511    token: Option<String>,
512    role_tokens: HashMap<String, String>,
513    shutdown: tokio::sync::oneshot::Receiver<()>,
514    ready_flag: Arc<AtomicBool>,
515    keys: Option<AccessKeys>,
516    tx_limits: TxLimits,
517) -> std::io::Result<()> {
518    serve_inner(
519        db,
520        addr,
521        ready,
522        UiFallback::None,
523        token,
524        role_tokens,
525        Some(shutdown),
526        ready_flag,
527        keys,
528        tx_limits,
529    )
530    .await
531}
532
533/// [`serve_with_ui_and_role_tokens`] that stops gracefully. See [`serve_with_shutdown`].
534pub async fn serve_with_ui_and_role_tokens_and_shutdown(
535    db: SharedDb,
536    addr: SocketAddr,
537    ready: tokio::sync::oneshot::Sender<SocketAddr>,
538    ui_dir: PathBuf,
539    token: Option<String>,
540    role_tokens: HashMap<String, String>,
541    shutdown: tokio::sync::oneshot::Receiver<()>,
542) -> std::io::Result<()> {
543    serve_with_ui_and_role_tokens_and_shutdown_with_ready_flag(
544        db,
545        addr,
546        ready,
547        ui_dir,
548        token,
549        role_tokens,
550        shutdown,
551        Arc::new(AtomicBool::new(true)),
552        None,
553        TxLimits::from_env(),
554    )
555    .await
556}
557
558/// [`serve_with_ui_and_role_tokens_and_shutdown`] whose `GET /ready` follows `ready_flag`.
559#[allow(clippy::too_many_arguments)]
560pub async fn serve_with_ui_and_role_tokens_and_shutdown_with_ready_flag(
561    db: SharedDb,
562    addr: SocketAddr,
563    ready: tokio::sync::oneshot::Sender<SocketAddr>,
564    ui_dir: PathBuf,
565    token: Option<String>,
566    role_tokens: HashMap<String, String>,
567    shutdown: tokio::sync::oneshot::Receiver<()>,
568    ready_flag: Arc<AtomicBool>,
569    keys: Option<AccessKeys>,
570    tx_limits: TxLimits,
571) -> std::io::Result<()> {
572    serve_inner(
573        db,
574        addr,
575        ready,
576        UiFallback::Dir(ui_dir),
577        token,
578        role_tokens,
579        Some(shutdown),
580        ready_flag,
581        keys,
582        tx_limits,
583    )
584    .await
585}
586
587/// [`serve_with_embedded_ui`] that stops gracefully. See [`serve_with_shutdown`].
588#[cfg(feature = "embed-ui")]
589pub async fn serve_with_embedded_ui_and_shutdown(
590    db: SharedDb,
591    addr: SocketAddr,
592    ready: tokio::sync::oneshot::Sender<SocketAddr>,
593    token: Option<String>,
594    role_tokens: HashMap<String, String>,
595    shutdown: tokio::sync::oneshot::Receiver<()>,
596) -> std::io::Result<()> {
597    serve_with_embedded_ui_and_shutdown_with_ready_flag(
598        db,
599        addr,
600        ready,
601        token,
602        role_tokens,
603        shutdown,
604        Arc::new(AtomicBool::new(true)),
605        None,
606        TxLimits::from_env(),
607    )
608    .await
609}
610
611/// [`serve_with_embedded_ui_and_shutdown`] whose `GET /ready` follows `ready_flag`.
612#[cfg(feature = "embed-ui")]
613#[allow(clippy::too_many_arguments)]
614pub async fn serve_with_embedded_ui_and_shutdown_with_ready_flag(
615    db: SharedDb,
616    addr: SocketAddr,
617    ready: tokio::sync::oneshot::Sender<SocketAddr>,
618    token: Option<String>,
619    role_tokens: HashMap<String, String>,
620    shutdown: tokio::sync::oneshot::Receiver<()>,
621    ready_flag: Arc<AtomicBool>,
622    keys: Option<AccessKeys>,
623    tx_limits: TxLimits,
624) -> std::io::Result<()> {
625    serve_inner(
626        db,
627        addr,
628        ready,
629        UiFallback::Embedded,
630        token,
631        role_tokens,
632        Some(shutdown),
633        ready_flag,
634        keys,
635        tx_limits,
636    )
637    .await
638}
639
640enum UiFallback {
641    None,
642    Dir(PathBuf),
643    #[cfg(feature = "embed-ui")]
644    Embedded,
645}
646
647#[allow(clippy::too_many_arguments)]
648async fn serve_inner(
649    db: SharedDb,
650    addr: SocketAddr,
651    ready: tokio::sync::oneshot::Sender<SocketAddr>,
652    ui: UiFallback,
653    token: Option<String>,
654    role_tokens: HashMap<String, String>,
655    shutdown: Option<tokio::sync::oneshot::Receiver<()>>,
656    ready_flag: Arc<AtomicBool>,
657    keys: Option<AccessKeys>,
658    tx_limits: TxLimits,
659) -> std::io::Result<()> {
660    let listener = tokio::net::TcpListener::bind(addr).await?;
661    let local = listener.local_addr()?;
662    if ready.send(local).is_err() {
663        // Caller dropped the readiness receiver; still serve.
664        eprintln!("serve: readiness receiver dropped before bind notify");
665    }
666    let txs = TxStore::new(tx_limits);
667    let app = build_app_with_ready_and_txs(
668        db,
669        token,
670        role_tokens,
671        ui,
672        local,
673        false,
674        ready_flag,
675        keys,
676        txs.clone(),
677    );
678    match shutdown {
679        None => axum::serve(listener, app).await,
680        // Stop accepting, then let every request already in flight finish.
681        // Dropping the sender counts as the signal, so a caller that goes away
682        // shuts the server down rather than stranding it.
683        Some(rx) => {
684            axum::serve(listener, app)
685                .with_graceful_shutdown(async move {
686                    let _ = rx.await;
687                    // The start of the drain: open transactions roll back and
688                    // a new `begin` is refused.
689                    txs.close();
690                })
691                .await
692        }
693    }
694}
695
696/// Serve over TLS (requires the `tls` cargo feature).
697///
698/// Accepts a path to a PEM certificate and a PEM private key. Uses rustls
699/// via `axum-server`. The `ready` sender fires with the bound local address
700/// once the listener is accepting, mirroring the contract of [`serve_inner`].
701///
702/// Pass `token` / `role_tokens` the same way you would for [`serve_with_role_tokens`].
703#[cfg(feature = "tls")]
704pub async fn serve_tls(
705    db: SharedDb,
706    addr: SocketAddr,
707    ready: tokio::sync::oneshot::Sender<SocketAddr>,
708    cert_path: std::path::PathBuf,
709    key_path: std::path::PathBuf,
710    token: Option<String>,
711    role_tokens: HashMap<String, String>,
712) -> std::io::Result<()> {
713    let config = crate::tls::http_rustls_config(&cert_path, &key_path)?;
714    let listener = std::net::TcpListener::bind(addr)?;
715    listener.set_nonblocking(true)?;
716    let local = listener.local_addr()?;
717    if ready.send(local).is_err() {
718        eprintln!("serve_tls: readiness receiver dropped before bind notify");
719    }
720    let app = build_app(db, token, role_tokens, UiFallback::None, local, true);
721    axum_server::from_tcp_rustls(listener, config)?
722        .serve(app.into_make_service())
723        .await
724}
725
726/// [`serve_tls`] that stops gracefully. See [`serve_with_shutdown`].
727///
728/// `axum_server` has its own shutdown mechanism — a [`axum_server::Handle`] —
729/// rather than axum's `with_graceful_shutdown`, so the signal is bridged onto
730/// one here. In-flight requests get [`TLS_GRACE`] to finish before the
731/// listener is closed regardless; an unbounded grace period would let one
732/// stuck request hold a restart open forever.
733#[cfg(feature = "tls")]
734#[allow(clippy::too_many_arguments)]
735pub async fn serve_tls_with_shutdown(
736    db: SharedDb,
737    addr: SocketAddr,
738    ready: tokio::sync::oneshot::Sender<SocketAddr>,
739    cert_path: std::path::PathBuf,
740    key_path: std::path::PathBuf,
741    token: Option<String>,
742    role_tokens: HashMap<String, String>,
743    shutdown: tokio::sync::oneshot::Receiver<()>,
744) -> std::io::Result<()> {
745    serve_tls_with_shutdown_with_ready_flag(
746        db,
747        addr,
748        ready,
749        cert_path,
750        key_path,
751        token,
752        role_tokens,
753        shutdown,
754        Arc::new(AtomicBool::new(true)),
755        None,
756        TxLimits::from_env(),
757    )
758    .await
759}
760
761/// [`serve_tls_with_shutdown`] whose `GET /ready` follows `ready_flag`.
762#[cfg(feature = "tls")]
763#[allow(clippy::too_many_arguments)]
764pub async fn serve_tls_with_shutdown_with_ready_flag(
765    db: SharedDb,
766    addr: SocketAddr,
767    ready: tokio::sync::oneshot::Sender<SocketAddr>,
768    cert_path: std::path::PathBuf,
769    key_path: std::path::PathBuf,
770    token: Option<String>,
771    role_tokens: HashMap<String, String>,
772    shutdown: tokio::sync::oneshot::Receiver<()>,
773    ready_flag: Arc<AtomicBool>,
774    keys: Option<AccessKeys>,
775    tx_limits: TxLimits,
776) -> std::io::Result<()> {
777    let config = crate::tls::http_rustls_config(&cert_path, &key_path)?;
778    let listener = std::net::TcpListener::bind(addr)?;
779    listener.set_nonblocking(true)?;
780    let local = listener.local_addr()?;
781    if ready.send(local).is_err() {
782        eprintln!("serve_tls: readiness receiver dropped before bind notify");
783    }
784    let txs = TxStore::new(tx_limits);
785    let app = build_app_with_ready_and_txs(
786        db,
787        token,
788        role_tokens,
789        UiFallback::None,
790        local,
791        true,
792        ready_flag,
793        keys,
794        txs.clone(),
795    );
796    let handle = axum_server::Handle::new();
797    tokio::spawn({
798        let handle = handle.clone();
799        async move {
800            let _ = shutdown.await;
801            txs.close();
802            handle.graceful_shutdown(Some(TLS_GRACE));
803        }
804    });
805    axum_server::from_tcp_rustls(listener, config)?
806        .handle(handle)
807        .serve(app.into_make_service())
808        .await
809}
810
811/// How long a TLS server lets in-flight requests finish after a shutdown
812/// signal before closing anyway.
813#[cfg(feature = "tls")]
814const TLS_GRACE: std::time::Duration = std::time::Duration::from_secs(30);
815
816fn default_advertise_addr() -> SocketAddr {
817    SocketAddr::from(([127, 0, 0, 1], 8080))
818}
819
820fn build_app(
821    db: SharedDb,
822    token: Option<String>,
823    role_tokens: HashMap<String, String>,
824    ui: UiFallback,
825    addr: SocketAddr,
826    tls_active: bool,
827) -> Router {
828    build_app_with_ready(
829        db,
830        token,
831        role_tokens,
832        ui,
833        addr,
834        tls_active,
835        Arc::new(AtomicBool::new(true)),
836        None,
837    )
838}
839
840/// Reads `MUSHROOMDB_CORS_ORIGINS`. An invalid value does not fail here: the CLI
841/// validates before opening the store; other callers get CORS off plus an error log.
842#[allow(clippy::too_many_arguments)]
843fn build_app_with_ready(
844    db: SharedDb,
845    token: Option<String>,
846    role_tokens: HashMap<String, String>,
847    ui: UiFallback,
848    addr: SocketAddr,
849    tls_active: bool,
850    ready: Arc<AtomicBool>,
851    keys: Option<AccessKeys>,
852) -> Router {
853    build_app_with_ready_and_txs(
854        db,
855        token,
856        role_tokens,
857        ui,
858        addr,
859        tls_active,
860        ready,
861        keys,
862        TxStore::new(TxLimits::from_env()),
863    )
864}
865
866/// [`build_app_with_ready`] over a transaction store the caller holds, so the
867/// serve loop can close it when the shutdown signal arrives.
868#[allow(clippy::too_many_arguments)]
869fn build_app_with_ready_and_txs(
870    db: SharedDb,
871    token: Option<String>,
872    role_tokens: HashMap<String, String>,
873    ui: UiFallback,
874    addr: SocketAddr,
875    tls_active: bool,
876    ready: Arc<AtomicBool>,
877    keys: Option<AccessKeys>,
878    txs: TxStore,
879) -> Router {
880    // The CLI validates the variable before it opens the store; an invalid value
881    // reaching here (library use) fails closed: no CORS, and an error logged.
882    let cors = crate::cors::cors_layer_from_env().unwrap_or_else(|e| {
883        tracing::error!("{e}; CORS disabled");
884        None
885    });
886    build_app_with_cors(
887        db,
888        token,
889        role_tokens,
890        ui,
891        addr,
892        tls_active,
893        ready,
894        cors,
895        keys,
896        txs,
897    )
898}
899
900#[allow(clippy::too_many_arguments)]
901fn build_app_with_cors(
902    db: SharedDb,
903    token: Option<String>,
904    role_tokens: HashMap<String, String>,
905    ui: UiFallback,
906    addr: SocketAddr,
907    tls_active: bool,
908    ready: Arc<AtomicBool>,
909    cors: Option<tower_http::cors::CorsLayer>,
910    keys: Option<AccessKeys>,
911    txs: TxStore,
912) -> Router {
913    debug_assert!(
914        !db.read().has_event_sink(),
915        "router() must be called at most once per SharedDb; a second call \
916         replaces the sink and terminates all existing /watch subscribers \
917         with RecvError::Closed"
918    );
919    let (tx, _) = tokio::sync::broadcast::channel(1024);
920    {
921        let tx = tx.clone();
922        db.write().set_event_sink(Box::new(move |ev| {
923            let _ = tx.send(ev);
924        }));
925    }
926    let state = AppState {
927        db,
928        watch: tx,
929        token,
930        role_tokens,
931        addr,
932        tls_active,
933        started_at: std::time::Instant::now(),
934        started_unix_s: unix_seconds_now(),
935        ready,
936        access_keys: keys,
937        txs,
938    };
939    let app = Router::new()
940        .route("/health", get(health))
941        .route("/ready", get(ready_handler))
942        .route("/query", post(query))
943        .route("/schema/indexes", get(schema_indexes_list))
944        .route("/schema/indexes", post(schema_index_create))
945        .route(
946            "/schema/indexes/{name}",
947            axum::routing::delete(schema_index_drop),
948        )
949        .route("/schema/constraints", get(schema_constraints_list))
950        .route("/schema/constraints", post(schema_constraint_create))
951        .route(
952            "/schema/constraints/{name}",
953            axum::routing::delete(schema_constraint_drop),
954        )
955        .route("/stats", get(stats))
956        .route("/metrics", get(metrics))
957        .route("/ingest", post(ingest))
958        .route("/rules", post(create_rule))
959        .route("/suggest", get(suggest))
960        .route("/explain", get(explain))
961        .route("/find_similar", post(find_similar))
962        .route("/node/{key}", get(node_info))
963        .route("/node/{key}", axum::routing::delete(delete_node))
964        .route("/node/{key}/edges", get(node_edges))
965        .route("/node/{key}/neighborhood", get(neighborhood))
966        .route("/node/{key}/history", get(node_history_handler))
967        .route("/history/edge", get(edge_history_handler))
968        .route("/history/was_linked", get(was_linked_handler))
969        .route(
970            "/node/{key}/prop/{field}",
971            axum::routing::put(set_node_prop),
972        )
973        .route(
974            "/node/{key}/prop/{field}",
975            axum::routing::delete(remove_node_prop),
976        )
977        // Simple BatchOp-mapped endpoints — routed through the group-commit
978        // queue (submit_batch) so concurrent node/edge CRUD does not hold
979        // the write lock during fsync.
980        .route("/nodes", post(create_node))
981        .route("/nodes/{key}/rename", post(rename_node))
982        .route("/edges", post(create_edge))
983        .route("/edges/upsert", post(upsert_edge))
984        .route(
985            "/edges/{etype}/{src}/{dst}",
986            axum::routing::delete(delete_edge),
987        )
988        .route("/algo/pagerank", post(algo_pagerank))
989        .route("/algo/wcc", post(algo_wcc))
990        .route("/algo/degree", post(algo_degree))
991        .route("/backup", post(backup))
992        .merge(tx::routes())
993        .with_state(state.clone());
994
995    // The two WebSocket routes are built separately and merged unlayered: a
996    // request timeout applied to them would sever a healthy subscription at
997    // the deadline, and a subscription is meant to outlive any request.
998    let streaming = Router::new()
999        .route("/watch", get(crate::ws::watch))
1000        .route("/subscribe", get(crate::subscribe::subscribe))
1001        .with_state(state.clone());
1002
1003    // Everything else gets a deadline and a ceiling.
1004    //
1005    // `RequestBodyTimeoutLayer` is deliberately not what is wanted here: the
1006    // risk is a handler that never finishes, not only a client that never
1007    // finishes sending. `TimeoutLayer` bounds the whole exchange and answers
1008    // 408 when it expires.
1009    //
1010    // The concurrency limit sheds rather than queues. An unbounded queue in
1011    // front of a single-writer engine converts overload into latency that
1012    // never recovers; 503 is the honest answer, and the one a load balancer
1013    // can act on.
1014    let app = app
1015        .layer(tower_http::timeout::TimeoutLayer::with_status_code(
1016            StatusCode::REQUEST_TIMEOUT,
1017            REQUEST_TIMEOUT,
1018        ))
1019        .layer(
1020            tower::ServiceBuilder::new()
1021                // Load shedding surfaces as a `BoxError`, and a Router's stack
1022                // must be infallible, so the overload is mapped to a response
1023                // here rather than escaping as an error.
1024                .layer(axum::error_handling::HandleErrorLayer::new(
1025                    |_: tower::BoxError| async {
1026                        (
1027                            StatusCode::SERVICE_UNAVAILABLE,
1028                            Json(json!({
1029                                "error": "server at its concurrency limit; retry"
1030                            })),
1031                        )
1032                    },
1033                ))
1034                .load_shed()
1035                .concurrency_limit(MAX_CONCURRENT_REQUESTS),
1036        );
1037    let app = app.merge(streaming);
1038    let app = match ui {
1039        UiFallback::None => app,
1040        UiFallback::Dir(dir) => app.fallback_service(ServeDir::new(dir)),
1041        #[cfg(feature = "embed-ui")]
1042        UiFallback::Embedded => app.fallback(embedded_fallback),
1043    };
1044    let app = app
1045        .layer(middleware::from_fn_with_state(state, auth_middleware))
1046        // Outside auth, so a 401 is logged too — an unauthenticated flood is
1047        // exactly the thing an operator needs to see.
1048        .layer(middleware::from_fn(log_middleware))
1049        // Cap request bodies (default axum limit is only 2 MiB; we allow larger
1050        // ingest batches but reject multi-GB bodies that would OOM the collector
1051        // before any handler runs).
1052        .layer(axum::extract::DefaultBodyLimit::max(MAX_REQUEST_BODY_BYTES));
1053    // Outermost: a preflight is answered here, before auth (it carries no
1054    // credentials), and a 401 still gets CORS headers so a page can read it.
1055    match cors {
1056        Some(cors) => app
1057            .layer(cors)
1058            .layer(middleware::from_fn(crate::cors::strip_unless_allowed)),
1059        None => app,
1060    }
1061}
1062
1063/// Maximum accepted HTTP request body size (64 MiB). Large enough for batched
1064/// `/ingest` payloads, small enough to prevent a single request from
1065/// exhausting memory. Clients with larger imports should chunk.
1066/// Per-request logging, with a policy chosen so the log stays readable.
1067///
1068/// A line per request at one level is what makes a server log unreadable: the
1069/// 200s bury the 500s, and nobody reads it after the first week. So the level
1070/// follows what happened:
1071///
1072/// | Outcome | Level | Visible by default |
1073/// |---|---|---|
1074/// | 5xx | `error` | yes — always worth a look |
1075/// | slower than [`SLOW_REQUEST`] | `warn` | yes — the request that will time out next |
1076/// | 4xx | `info` | yes — one line, the caller's mistake or a probe |
1077/// | everything else | `debug` | no |
1078///
1079/// A healthy server is therefore quiet, and turning `debug` on for one module
1080/// gives the full trace without recompiling.
1081///
1082/// Every response carries `x-request-id`, echoed from the caller's header when
1083/// it sent one and generated otherwise, so an error someone reports can be
1084/// found in the log. Before 0.6.12 there was no `tracing`, no access log and no
1085/// request id anywhere in the tree: a 500 in production could not be tied to
1086/// the request that caused it.
1087async fn log_middleware(req: Request, next: Next) -> Response {
1088    let request_id = req
1089        .headers()
1090        .get("x-request-id")
1091        .and_then(|v| v.to_str().ok())
1092        .filter(|v| !v.is_empty() && v.len() <= 128 && v.is_ascii())
1093        .map(str::to_owned)
1094        .unwrap_or_else(new_request_id);
1095
1096    let method = req.method().clone();
1097    // The path only. A query string can carry `?token=`, and a log is exactly
1098    // the wrong place for a credential.
1099    let path = req.uri().path().to_owned();
1100    let started = std::time::Instant::now();
1101
1102    let mut res = next.run(req).await;
1103    let status = res.status();
1104    let ms = started.elapsed().as_millis();
1105
1106    if status.is_server_error() {
1107        tracing::error!(%request_id, %method, %path, status = status.as_u16(), ms, "request failed");
1108    } else if started.elapsed() >= SLOW_REQUEST {
1109        tracing::warn!(%request_id, %method, %path, status = status.as_u16(), ms, "slow request");
1110    } else if status.is_client_error() {
1111        tracing::info!(%request_id, %method, %path, status = status.as_u16(), ms, "request refused");
1112    } else {
1113        tracing::debug!(%request_id, %method, %path, status = status.as_u16(), ms, "request");
1114    }
1115
1116    if let Ok(v) = axum::http::HeaderValue::from_str(&request_id) {
1117        res.headers_mut().insert("x-request-id", v);
1118    }
1119    res
1120}
1121
1122/// A request id with no dependency on a uuid crate: the process id, a
1123/// monotonic counter, and the low bits of a nanosecond clock. Unique enough to
1124/// correlate a report with a log line, which is all it is for.
1125fn new_request_id() -> String {
1126    use std::sync::atomic::{AtomicU64, Ordering};
1127    static N: AtomicU64 = AtomicU64::new(0);
1128    let n = N.fetch_add(1, Ordering::Relaxed);
1129    let nanos = std::time::SystemTime::now()
1130        .duration_since(std::time::UNIX_EPOCH)
1131        .map(|d| d.subsec_nanos() as u64)
1132        .unwrap_or(0);
1133    format!("{:x}-{:x}-{:x}", std::process::id(), n, nanos)
1134}
1135
1136/// A request slower than this is logged at `warn` whatever its status. Well
1137/// under [`REQUEST_TIMEOUT`], so the log names the requests heading for a
1138/// timeout before one happens.
1139const SLOW_REQUEST: std::time::Duration = std::time::Duration::from_secs(5);
1140
1141const MAX_REQUEST_BODY_BYTES: usize = 64 * 1024 * 1024;
1142
1143/// How long any one non-streaming request may take before the server answers
1144/// `408 Request Timeout` and drops it.
1145///
1146/// Sized for the slowest legitimate handler rather than the typical one: a
1147/// `/query` against a large store, a `/backup`, or a `/suggest` that profiles
1148/// the graph can all take tens of seconds. The point is not to make slow
1149/// requests fast — it is that a handler which will never finish stops holding
1150/// a connection, a permit and a read guard forever.
1151///
1152/// The WebSocket routes are exempt; see `build_app`.
1153const REQUEST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(120);
1154
1155/// How many non-streaming requests may be in flight at once before the server
1156/// sheds load with `503 Service Unavailable`.
1157///
1158/// The engine is single-writer and many-reader, so the ceiling exists to bound
1159/// memory and the blocking pool, not to serialise work. Chosen well above any
1160/// plausible operator workload and well below what would exhaust the process.
1161const MAX_CONCURRENT_REQUESTS: usize = 256;
1162
1163/// Readiness for a container platform. Needs no token and says nothing about
1164/// the graph.
1165async fn ready_handler(State(state): State<AppState>) -> Response {
1166    if state.ready.load(Ordering::Acquire) {
1167        json_ok(json!({ "ok": true }))
1168    } else {
1169        let mut response = json_ok(json!({ "ok": false, "reason": "not ready" }));
1170        *response.status_mut() = StatusCode::SERVICE_UNAVAILABLE;
1171        response
1172    }
1173}
1174
1175async fn health(
1176    State(state): State<AppState>,
1177    Extension(identity): Extension<AuthIdentity>,
1178) -> Response {
1179    // Liveness is for everyone; the graph's shape is not. `/stats` and
1180    // `/metrics` already refuse a role-bound token because counters leak graph
1181    // size, and an endpoint that hands the same counters to a caller with no
1182    // token at all makes that refusal decorative.
1183    if !matches!(identity, AuthIdentity::Full) {
1184        return json_ok(json!({ "ok": true }));
1185    }
1186    let (nodes, _, edges) = state.db.reader().counts();
1187    json_ok(json!({
1188        "ok": true,
1189        "nodes": nodes,
1190        "edges": edges,
1191        "addr": state.addr.to_string(),
1192    }))
1193}
1194
1195/// Run a GraphDb write on the blocking pool. The write guard lives only
1196/// inside `f` and is dropped before this future awaits.
1197async fn blocking_write<T, F>(f: F) -> std::result::Result<T, Response>
1198where
1199    T: Send + 'static,
1200    F: FnOnce() -> mushroomdb::Result<T> + Send + 'static,
1201{
1202    blocking_write_with(f, graph_err).await
1203}
1204
1205/// [`blocking_write`] with the engine error answered by `on_err`.
1206async fn blocking_write_with<T, F>(
1207    f: F,
1208    on_err: fn(GraphError) -> Response,
1209) -> std::result::Result<T, Response>
1210where
1211    T: Send + 'static,
1212    F: FnOnce() -> mushroomdb::Result<T> + Send + 'static,
1213{
1214    match tokio::task::spawn_blocking(f).await {
1215        Ok(Ok(v)) => Ok(v),
1216        Ok(Err(e)) => Err(on_err(e)),
1217        Err(_) => Err(err_response("write task panicked")),
1218    }
1219}
1220
1221const TOKEN_COOKIE: &str = "mushroomdb_token";
1222
1223async fn auth_middleware(State(state): State<AppState>, mut req: Request, next: Next) -> Response {
1224    // No auth configured at all: every request is Full, no restriction.
1225    let keys_enforce = state
1226        .access_keys
1227        .as_ref()
1228        .is_some_and(AccessKeys::enforcing);
1229    if state.token.is_none() && state.role_tokens.is_empty() && !keys_enforce {
1230        req.extensions_mut().insert(AuthIdentity::Full);
1231        return next.run(req).await;
1232    }
1233
1234    let presented = request_token(&req);
1235
1236    // Health stays reachable without a credential — a load balancer has none —
1237    // but reaching it is not the same as being trusted by it. A caller whose
1238    // token does not resolve to full access is `Anonymous` here, and the
1239    // handler discloses liveness only.
1240    //
1241    // Before 0.6.12 this inserted `Full` unconditionally, so an unauthenticated
1242    // caller read `nodes_live` and `edges` from `/health` while a role-bound
1243    // token was refused the same counters by `/metrics` and `/stats` on the
1244    // stated grounds that counters leak graph size. Two endpoints, one
1245    // disclosure, opposite policies.
1246    if req.method() == Method::GET && req.uri().path() == "/ready" {
1247        req.extensions_mut().insert(AuthIdentity::Anonymous);
1248        return next.run(req).await;
1249    }
1250    if req.method() == Method::GET && req.uri().path() == "/health" {
1251        let full = state
1252            .token
1253            .clone()
1254            .filter(|s| !s.is_empty())
1255            .zip(presented.as_deref())
1256            .is_some_and(|(tok, p)| constant_time_eq(p.as_bytes(), tok.as_bytes()))
1257            || presented
1258                .as_deref()
1259                .zip(state.access_keys.as_ref())
1260                .is_some_and(|(p, keys)| keys.resolve(p) == Some(KeyTier::Admin));
1261        req.extensions_mut().insert(if full {
1262            AuthIdentity::Full
1263        } else {
1264            AuthIdentity::Anonymous
1265        });
1266        return next.run(req).await;
1267    }
1268
1269    // Check full-access token first.
1270    if let Some(ref full_tok) = state.token.clone().filter(|s| !s.is_empty()) {
1271        if presented
1272            .as_deref()
1273            .is_some_and(|p| constant_time_eq(p.as_bytes(), full_tok.as_bytes()))
1274        {
1275            let set_cookie = presented_bearer_or_query(&req).as_deref() == Some(full_tok.as_str());
1276            req.extensions_mut().insert(AuthIdentity::Full);
1277            let mut res = next.run(req).await;
1278            if set_cookie && is_html_response(&res) {
1279                attach_token_cookie(&mut res, full_tok, state.tls_active);
1280            }
1281            return res;
1282        }
1283    }
1284
1285    // Check role-bound tokens.
1286    if let Some(tok) = presented.as_deref() {
1287        if let Some(role_name) = role_for_token(&state.role_tokens, tok) {
1288            // Early-deny paths whose handlers use WebSocket upgrade: the WS
1289            // extraction consumes the request before the handler body runs, so
1290            // the handler-level identity check is unreachable in those cases.
1291            // Returning 403 here also removes the need for the handler to
1292            // hold a read lock just to enforce this.
1293            let path = req.uri().path();
1294            if path == "/subscribe" || path == "/watch" {
1295                return forbidden("role-bound token: this endpoint is not permitted");
1296            }
1297            req.extensions_mut()
1298                .insert(AuthIdentity::Role(role_name.clone()));
1299            return next.run(req).await;
1300        }
1301    }
1302
1303    // Check the keys.json set. `resolve` compares every key in constant time.
1304    if let Some(tier) = presented
1305        .as_deref()
1306        .zip(state.access_keys.as_ref())
1307        .and_then(|(tok, keys)| keys.resolve(tok))
1308    {
1309        let path = req.uri().path();
1310        let ws_path = path == "/subscribe" || path == "/watch";
1311        match tier {
1312            KeyTier::Admin => {
1313                let set_cookie = presented_bearer_or_query(&req);
1314                req.extensions_mut().insert(AuthIdentity::Full);
1315                let mut res = next.run(req).await;
1316                if let Some(token) = set_cookie.filter(|_| is_html_response(&res)) {
1317                    attach_token_cookie(&mut res, &token, state.tls_active);
1318                }
1319                return res;
1320            }
1321            KeyTier::Write => {
1322                // The WS extractor runs before the handler body, so these
1323                // are refused here as well as in the handlers.
1324                if ws_path {
1325                    return forbidden(WRITE_TIER_REFUSAL);
1326                }
1327                req.extensions_mut().insert(AuthIdentity::Write);
1328                return next.run(req).await;
1329            }
1330            KeyTier::Read(role_name) => {
1331                if ws_path {
1332                    return forbidden("role-bound token: this endpoint is not permitted");
1333                }
1334                // A role may carry a write scope; a read key never writes.
1335                if !read_key_may(req.method(), path) {
1336                    return forbidden(READ_TIER_REFUSAL);
1337                }
1338                req.extensions_mut().insert(AuthIdentity::Role(role_name));
1339                req.extensions_mut().insert(ReadOnlyKey);
1340                return next.run(req).await;
1341            }
1342        }
1343    }
1344
1345    // Nothing matched → 401.
1346    unauthorized()
1347}
1348
1349/// The role bound to `presented`, comparing it with every role token in
1350/// constant time (a hash-map lookup would stop at the first difference).
1351fn role_for_token<'a>(
1352    role_tokens: &'a HashMap<String, String>,
1353    presented: &str,
1354) -> Option<&'a String> {
1355    let mut found = None;
1356    for (token, role) in role_tokens {
1357        if constant_time_eq(presented.as_bytes(), token.as_bytes()) {
1358            found = Some(role);
1359        }
1360    }
1361    found
1362}
1363
1364/// Whether a read-tier key may make this request at all. An allowlist, so a
1365/// route added later is refused to read keys until someone decides otherwise:
1366/// reads (`GET`), the two `POST` routes that carry a read (`/query`, whose
1367/// handler refuses a write statement, and `/find_similar`), and the `/tx/*`
1368/// routes: a read key may hold a transaction, whose handler refuses a write.
1369fn read_key_may(method: &Method, path: &str) -> bool {
1370    match *method {
1371        Method::GET | Method::HEAD => true,
1372        Method::POST => path == "/query" || path == "/find_similar" || path.starts_with("/tx/"),
1373        Method::DELETE => path.starts_with("/tx/"),
1374        _ => false,
1375    }
1376}
1377
1378/// The 403 detail a read-tier key gets on a write.
1379pub(crate) const READ_TIER_REFUSAL: &str = "read-tier key: writes are not permitted";
1380
1381fn request_token(req: &Request) -> Option<String> {
1382    presented_bearer_or_query(req).or_else(|| presented_cookie(req))
1383}
1384
1385fn presented_bearer_or_query(req: &Request) -> Option<String> {
1386    if let Some(header) = req
1387        .headers()
1388        .get(header::AUTHORIZATION)
1389        .and_then(|v| v.to_str().ok())
1390    {
1391        if let Some(value) = bearer_token(header) {
1392            return Some(value.to_string());
1393        }
1394    }
1395    query_param(req.uri().query().unwrap_or(""), "token")
1396}
1397
1398fn presented_cookie(req: &Request) -> Option<String> {
1399    let header = req.headers().get(header::COOKIE)?.to_str().ok()?;
1400    cookie_named(header, TOKEN_COOKIE).map(str::to_string)
1401}
1402
1403fn cookie_named<'a>(header: &'a str, name: &str) -> Option<&'a str> {
1404    for part in header.split(';') {
1405        let part = part.trim();
1406        let Some((k, v)) = part.split_once('=') else {
1407            continue;
1408        };
1409        if k.trim() == name {
1410            return Some(v.trim());
1411        }
1412    }
1413    None
1414}
1415
1416fn is_html_response(res: &Response) -> bool {
1417    res.headers()
1418        .get(header::CONTENT_TYPE)
1419        .and_then(|v| v.to_str().ok())
1420        .is_some_and(|ct| {
1421            ct.split(';')
1422                .next()
1423                .unwrap_or("")
1424                .trim()
1425                .eq_ignore_ascii_case("text/html")
1426        })
1427}
1428
1429fn attach_token_cookie(res: &mut Response, token: &str, secure: bool) {
1430    let secure_attr = if secure { "; Secure" } else { "" };
1431    let value = format!("{TOKEN_COOKIE}={token}; Path=/; SameSite=Lax; HttpOnly{secure_attr}");
1432    if let Ok(hv) = HeaderValue::from_str(&value) {
1433        res.headers_mut().insert(header::SET_COOKIE, hv);
1434    }
1435}
1436
1437fn bearer_token(header: &str) -> Option<&str> {
1438    let (scheme, value) = header.split_once(' ')?;
1439    if scheme.eq_ignore_ascii_case("Bearer") {
1440        Some(value.trim())
1441    } else {
1442        None
1443    }
1444}
1445
1446fn query_param(query: &str, key: &str) -> Option<String> {
1447    for pair in query.split('&') {
1448        if pair.is_empty() {
1449            continue;
1450        }
1451        match pair.split_once('=') {
1452            Some((k, v)) if k == key => return percent_decode_plus(v),
1453            None if pair == key => return Some(String::new()),
1454            _ => {}
1455        }
1456    }
1457    None
1458}
1459
1460/// `application/x-www-form-urlencoded`: `+` is space, `%HH` is a byte.
1461fn percent_decode_plus(s: &str) -> Option<String> {
1462    let bytes = s.as_bytes();
1463    let mut out = Vec::with_capacity(bytes.len());
1464    let mut i = 0;
1465    while i < bytes.len() {
1466        match bytes[i] {
1467            b'+' => {
1468                out.push(b' ');
1469                i += 1;
1470            }
1471            b'%' => {
1472                if i + 2 >= bytes.len() {
1473                    return None;
1474                }
1475                let hi = from_hex(bytes[i + 1])?;
1476                let lo = from_hex(bytes[i + 2])?;
1477                out.push((hi << 4) | lo);
1478                i += 3;
1479            }
1480            c => {
1481                out.push(c);
1482                i += 1;
1483            }
1484        }
1485    }
1486    String::from_utf8(out).ok()
1487}
1488
1489fn from_hex(b: u8) -> Option<u8> {
1490    match b {
1491        b'0'..=b'9' => Some(b - b'0'),
1492        b'a'..=b'f' => Some(b - b'a' + 10),
1493        b'A'..=b'F' => Some(b - b'A' + 10),
1494        _ => None,
1495    }
1496}
1497
1498fn unauthorized() -> Response {
1499    (
1500        StatusCode::UNAUTHORIZED,
1501        Json(json!({"error": "unauthorized"})),
1502    )
1503        .into_response()
1504}
1505
1506/// The 403 detail a write-tier key gets on an operator route.
1507pub(crate) const WRITE_TIER_REFUSAL: &str = "write-tier token: route not allowed";
1508
1509/// 403 for a write-tier key on a route that is for operators.
1510pub(crate) fn refuse_write_tier(identity: &AuthIdentity) -> Option<Response> {
1511    matches!(identity, AuthIdentity::Write).then(|| forbidden(WRITE_TIER_REFUSAL))
1512}
1513
1514/// 403 response for role-bound token operations that are denied in v1.
1515fn forbidden(detail: &str) -> Response {
1516    (StatusCode::FORBIDDEN, Json(json!({"error": detail}))).into_response()
1517}
1518
1519/// Convert a `mask_for_role` error into an HTTP response.
1520///
1521/// - `GraphError::Corrupt` → 500: the roles sidecar is poisoned; the server
1522///   refuses all role-token requests until the file is fixed and the DB is
1523///   re-opened.  Full-access tokens are unaffected.
1524/// - `GraphError::KeyNotFound` with a `role:` prefix → 401: the token is bound
1525///   to a role that does not exist in the DB at this request time.
1526fn role_mask_err(e: GraphError) -> Response {
1527    match e {
1528        GraphError::Corrupt { detail } => (
1529            StatusCode::INTERNAL_SERVER_ERROR,
1530            Json(json!({"error": format!("roles misconfigured: {detail}")})),
1531        )
1532            .into_response(),
1533        GraphError::KeyNotFound { key } if key.starts_with("role:") => unauthorized(),
1534        other => graph_err(other),
1535    }
1536}
1537
1538/// Convert a role-scoped write's error into an HTTP response: a role the
1539/// store no longer defines is 401, as it is for a read (`role_mask_err`) and
1540/// over Bolt; anything else is [`graph_err`]. Unlike `role_mask_err`, a
1541/// `Corrupt` keeps `graph_err`'s text, since on the write path it need not be
1542/// about roles.
1543fn role_write_err(e: GraphError) -> Response {
1544    match e {
1545        GraphError::KeyNotFound { key } if key.starts_with("role:") => unauthorized(),
1546        other => graph_err(other),
1547    }
1548}
1549
1550fn err_response(detail: impl Into<String>) -> Response {
1551    (
1552        StatusCode::BAD_REQUEST,
1553        Json(json!({"error": detail.into()})),
1554    )
1555        .into_response()
1556}
1557
1558/// 503 response for a store held by another process's write lock.
1559///
1560/// `Busy` is transient and nothing was written, so retrying is always safe.
1561/// 503 plus `Retry-After` is what standard client retry middleware keys on;
1562/// a 4xx would tell the client the request itself was malformed.
1563fn busy_response(detail: String) -> Response {
1564    (
1565        StatusCode::SERVICE_UNAVAILABLE,
1566        [(header::RETRY_AFTER, HeaderValue::from_static("1"))],
1567        Json(json!({ "error": detail })),
1568    )
1569        .into_response()
1570}
1571
1572fn graph_err(e: GraphError) -> Response {
1573    match e {
1574        // §4.3: role-scoped write denials map to 403 with the verbatim reason
1575        // string (Display delegates to reason, so .to_string() == reason).
1576        GraphError::RoleWriteDenied { reason } => forbidden(&reason),
1577        // Ledger row 12. `role_mask_err` already answered 500 for `Corrupt`
1578        // while this catch-all answered 400, so one damaged store answered
1579        // differently depending on whether the role-mask memo was warm. A
1580        // corrupt store is a server-side condition: 400 tells a caller to change
1581        // an input, and no input they can send will help. The message is
1582        // deliberately `to_string()` — identical to what the catch-all produced
1583        // — so only the status moves.
1584        corrupt @ GraphError::Corrupt { .. } => (
1585            StatusCode::INTERNAL_SERVER_ERROR,
1586            Json(json!({ "error": corrupt.to_string() })),
1587        )
1588            .into_response(),
1589        GraphError::QueryError { detail } | GraphError::IngestError { detail } => {
1590            err_response(detail)
1591        }
1592        // Transient: another process holds the cross-process write lock.
1593        busy @ GraphError::Busy { .. } => busy_response(busy.to_string()),
1594        // The data directory is not writable by the server's user. That is the
1595        // operator's to fix and no request can change it, so it is a 500, not
1596        // the 400 the catch-all would give. The full message (paths, uids) goes
1597        // to the log; the client gets a fixed one.
1598        GraphError::Unwritable { path, detail, .. } => {
1599            tracing::error!(path = %path, "{detail}");
1600            (
1601                StatusCode::INTERNAL_SERVER_ERROR,
1602                Json(json!({ "error": GraphError::UNWRITABLE_CLIENT_MESSAGE })),
1603            )
1604                .into_response()
1605        }
1606        // A unique-constraint clash is a conflict with stored state, like a
1607        // duplicate key. The text names the constraint and field only.
1608        clash @ GraphError::ConstraintViolation { .. } => (
1609            StatusCode::CONFLICT,
1610            Json(json!({ "error": clash.to_string(), "code": "constraint" })),
1611        )
1612            .into_response(),
1613        // A value over the store's size limit: the request is too large for
1614        // what it stores, not malformed.
1615        big @ GraphError::ValueTooLarge { .. } => (
1616            StatusCode::PAYLOAD_TOO_LARGE,
1617            Json(json!({ "error": big.to_string() })),
1618        )
1619            .into_response(),
1620        other => err_response(other.to_string()),
1621    }
1622}
1623
1624fn key_not_found(key: String) -> Response {
1625    (
1626        StatusCode::NOT_FOUND,
1627        Json(json!({"error": GraphError::KeyNotFound { key }.to_string()})),
1628    )
1629        .into_response()
1630}
1631
1632fn conflict_response(key: String) -> Response {
1633    (
1634        StatusCode::CONFLICT,
1635        Json(json!({"error": GraphError::DuplicateKey { key }.to_string()})),
1636    )
1637        .into_response()
1638}
1639
1640fn json_ok(value: Js) -> Response {
1641    (StatusCode::OK, Json(value)).into_response()
1642}
1643
1644fn ingest_options(v: Option<&Js>) -> Result<IngestOptions, String> {
1645    let Some(v) = v else {
1646        return Ok(IngestOptions::default());
1647    };
1648    if v.is_null() {
1649        return Ok(IngestOptions::default());
1650    }
1651    let obj = v
1652        .as_object()
1653        .ok_or_else(|| "options must be an object".to_string())?;
1654    let mut opts = IngestOptions::default();
1655    if let Some(kf) = obj.get("key_field") {
1656        opts.key_field = kf
1657            .as_str()
1658            .ok_or_else(|| "options.key_field must be a string".to_string())?
1659            .to_string();
1660    }
1661    if let Some(fk) = obj.get("auto_fk") {
1662        if fk == &Js::Bool(false) || fk.as_str() == Some("off") {
1663            opts.auto_fk = AutoFk::Off;
1664        } else if let Some(m) = fk.as_object() {
1665            let suf = m
1666                .get("suffix")
1667                .and_then(Js::as_str)
1668                .ok_or_else(|| "options.auto_fk.suffix must be a string".to_string())?;
1669            opts.auto_fk = AutoFk::Auto {
1670                suffix: suf.to_string(),
1671            };
1672        } else {
1673            return Err("options.auto_fk must be false, \"off\", or {suffix}".into());
1674        }
1675    }
1676    Ok(opts)
1677}
1678
1679/// Format a `ResultSet` as Arrow IPC or JSON depending on `format`.
1680fn format_query_result(rs: ResultSet, format: &str) -> Response {
1681    match format {
1682        "" => match to_ipc_bytes(&rs) {
1683            Ok(bytes) => (
1684                StatusCode::OK,
1685                [(header::CONTENT_TYPE, "application/vnd.apache.arrow.stream")],
1686                bytes,
1687            )
1688                .into_response(),
1689            Err(e) => err_response(e),
1690        },
1691        "json" => json_ok(result_set_json(&rs)),
1692        other => err_response(format!("unknown format: {other}")),
1693    }
1694}
1695
1696async fn query(
1697    State(state): State<AppState>,
1698    Extension(identity): Extension<AuthIdentity>,
1699    read_only_key: Option<Extension<ReadOnlyKey>>,
1700    Query(qs): Query<BTreeMap<String, String>>,
1701    Json(body): Json<Js>,
1702) -> Response {
1703    let cypher = match body.get("cypher").and_then(Js::as_str) {
1704        Some(s) => s.to_string(),
1705        None => return err_response("missing cypher"),
1706    };
1707    let params = match params_from_json(body.get("params")) {
1708        Ok(p) => p,
1709        Err(e) => return err_response(e),
1710    };
1711    let format = qs.get("format").map(String::as_str).unwrap_or("");
1712
1713    // Optional time-travel: `as_of` is a 0-based WAL commit index. The query
1714    // runs against the graph as it existed at that commit (read-only).
1715    let as_of = match body.get("as_of") {
1716        None | Some(Js::Null) => None,
1717        Some(v) => match v.as_u64() {
1718            Some(n) => Some(n),
1719            None => return err_response("as_of must be a non-negative integer commit index"),
1720        },
1721    };
1722
1723    // Parse client-supplied mask (optional array of node keys).
1724    let mask_keys: Option<Vec<String>> = match body.get("mask") {
1725        None | Some(Js::Null) => None,
1726        Some(Js::Array(arr)) => {
1727            let mut keys = Vec::with_capacity(arr.len());
1728            for v in arr {
1729                match v.as_str() {
1730                    Some(s) => keys.push(s.to_string()),
1731                    None => return err_response("mask must be an array of strings"),
1732                }
1733            }
1734            Some(keys)
1735        }
1736        Some(_) => return err_response("mask must be an array of strings"),
1737    };
1738
1739    // The second visibility axis. `namespace` is a leg that **intersects**
1740    // whatever restriction the identity and the body already impose — a role
1741    // token's binding, a client mask, both — so it can only narrow. A role bound
1742    // to namespaces honours them with no `namespace` in the body; one outside
1743    // the binding is the empty intersection, never the union.
1744    let namespace = match namespace_arg(body.get("namespace")) {
1745        Ok(n) => n,
1746        Err(e) => return err_response(e),
1747    };
1748
1749    // Stub mode discloses node existence, which is exactly the question an
1750    // as-of read is asking. The two do not compose.
1751    if as_of.is_some()
1752        && body
1753            .get("stub_hidden")
1754            .and_then(|v| v.as_bool())
1755            .unwrap_or(false)
1756    {
1757        return err_response("as_of (time-travel) does not compose with stub_hidden");
1758    }
1759
1760    // Role token: write Cypher routes to query_write_authz (scope + mask
1761    // resolved under the write lock, §5 discipline).  Read Cypher uses the
1762    // lock-free epoch snapshot path unchanged.
1763    if let AuthIdentity::Role(ref role_name) = identity {
1764        let is_write = match is_write_query(&cypher) {
1765            Ok(b) => b,
1766            Err(e) => return err_response(e),
1767        };
1768        if is_write && read_only_key.is_some() {
1769            return forbidden(READ_TIER_REFUSAL);
1770        }
1771        if as_of.is_some() && is_write {
1772            return err_response("as_of (time-travel) queries are read-only");
1773        }
1774        // `mask` and `namespace` are read guards: both documented as making the
1775        // request a read, and the full-token branch below enforces that by
1776        // routing anything carrying either to `query_masked`, which refuses a
1777        // write. The role branch used to check `is_write` first and never look
1778        // at them, so a client sending a write *with* a namespace as its guard
1779        // got the write executed instead of the 400 the docs promise. Not a
1780        // widening — `check_single_op_authz` still gates the write — but the
1781        // guard the caller asked for was not applied.
1782        if is_write && (mask_keys.is_some() || namespace.is_some()) {
1783            return (
1784                StatusCode::BAD_REQUEST,
1785                Json(json!({"error": "masked queries are read-only"})),
1786            )
1787                .into_response();
1788        }
1789        if is_write {
1790            let role = role_name.clone();
1791            let cypher_c = cypher.clone();
1792            let params_c = params.clone();
1793            let db = state.db.clone();
1794            return match blocking_write_with(
1795                move || db.write().query_write_authz(&role, &cypher_c, &params_c),
1796                role_write_err,
1797            )
1798            .await
1799            {
1800                Ok(rs) => format_query_result(rs, format),
1801                Err(resp) => resp,
1802            };
1803        }
1804        // Time travel: the role's keys and labels are resolved against the
1805        // graph as it was at `commit`, and a client mask intersects them
1806        // there.  The role *definition* is the current one — `roles.json` is a
1807        // sidecar and is never a WAL record, so it has no past version.
1808        if let Some(commit) = as_of {
1809            let scope = match mask_keys {
1810                Some(ref keys) => AsOfScope::RoleAndKeys(role_name, keys),
1811                None => AsOfScope::Role(role_name),
1812            };
1813            let g = state.db.read();
1814            let out = match namespace.as_deref() {
1815                Some(ns) => g.query_at_scoped_in_namespace(commit, &cypher, &params, scope, ns),
1816                None => g.query_at_scoped(commit, &cypher, &params, scope),
1817            };
1818            return match out {
1819                Ok(rs) => format_query_result(rs, format),
1820                Err(e) => role_mask_err(e),
1821            };
1822        }
1823        let snap = state.db.reader();
1824        let role_mask = match snap.mask_for_role(role_name) {
1825            Ok(m) => m,
1826            Err(e) => return role_mask_err(e),
1827        };
1828        let effective_mask = if let Some(ref keys) = mask_keys {
1829            // Client mask intersects role mask — never widens visibility.
1830            let client_mask = NodeMask::from_ids(keys.iter().filter_map(|k| snap.resolve_key(k)));
1831            role_mask.intersect(&client_mask)
1832        } else {
1833            role_mask
1834        };
1835        // And the namespace leg intersects whatever that came to, on the same
1836        // snapshot the query runs against.
1837        let effective_mask = match namespace.as_deref() {
1838            Some(ns) => match snap.mask_for_namespace(ns) {
1839                Ok(ns_mask) => effective_mask.intersect(&ns_mask),
1840                Err(e) => return graph_err(e),
1841            },
1842            None => effective_mask,
1843        };
1844        return match snap.query_masked(&cypher, &params, &effective_mask) {
1845            Ok(rs) => format_query_result(rs, format),
1846            Err(GraphError::MaskedReadOnly) => (
1847                StatusCode::BAD_REQUEST,
1848                Json(json!({"error": "masked queries are read-only"})),
1849            )
1850                .into_response(),
1851            Err(e) => graph_err(e),
1852        };
1853    }
1854
1855    // Full token: existing paths below.
1856
1857    // When a client-supplied mask is present, route to query_masked (read-only).
1858    // The mask and query_masked resolve against one snapshot so the mask and
1859    // the query see the same graph.
1860    //
1861    // `stub_hidden: true` opts into MaskMode::Stub for the mask; Cypher query
1862    // behaviour is identical in both modes (hidden nodes are excluded from
1863    // query results regardless of mode).
1864    if mask_keys.is_some() || namespace.is_some() {
1865        // Time travel: the allow-list resolves against the as-of graph, so a
1866        // key that did not exist at `commit` resolves to nothing.
1867        if let Some(commit) = as_of {
1868            let g = state.db.read();
1869            let out = match (&mask_keys, namespace.as_deref()) {
1870                (Some(keys), Some(ns)) => g.query_at_scoped_in_namespace(
1871                    commit,
1872                    &cypher,
1873                    &params,
1874                    AsOfScope::Keys(keys),
1875                    ns,
1876                ),
1877                (Some(keys), None) => {
1878                    g.query_at_scoped(commit, &cypher, &params, AsOfScope::Keys(keys))
1879                }
1880                (None, Some(ns)) => {
1881                    g.query_at_scoped(commit, &cypher, &params, AsOfScope::Namespace(ns))
1882                }
1883                (None, None) => unreachable!("one of the two is Some in this branch"),
1884            };
1885            return match out {
1886                Ok(rs) => format_query_result(rs, format),
1887                Err(e) => graph_err(e),
1888            };
1889        }
1890        let stub_hidden = body
1891            .get("stub_hidden")
1892            .and_then(|v| v.as_bool())
1893            .unwrap_or(false);
1894        let with_mode = |m: NodeMask| {
1895            if stub_hidden {
1896                m.with_mode(MaskMode::Stub)
1897            } else {
1898                m
1899            }
1900        };
1901        // The mask and the query resolve against one lock-free snapshot, so
1902        // they see the same graph without holding the read lock for the query.
1903        let snap = state.db.reader();
1904        let key_mask = mask_keys
1905            .as_ref()
1906            .map(|keys| snapshot_mask_from_keys(&snap, keys.iter().map(String::as_str)));
1907        let mask = match (key_mask, namespace.as_deref()) {
1908            (Some(keys), Some(ns)) => match snap.mask_for_namespace(ns) {
1909                Ok(ns_mask) => keys.intersect(&ns_mask),
1910                Err(e) => return graph_err(e),
1911            },
1912            (Some(keys), None) => keys,
1913            (None, Some(ns)) => match snap.mask_for_namespace(ns) {
1914                Ok(ns_mask) => ns_mask,
1915                Err(e) => return graph_err(e),
1916            },
1917            (None, None) => unreachable!("one of the two is Some in this branch"),
1918        };
1919        return masked_query_response(
1920            snap.query_masked(&cypher, &params, &with_mode(mask)),
1921            format,
1922        );
1923    }
1924
1925    // Detect write statements to dispatch to the correct lock.
1926    // Write statements (CREATE / MATCH…SET / MATCH…DELETE / MERGE) need the
1927    // write lock so mutations flow through WAL + rule engine with fsync before
1928    // the response is sent.  Read queries (MATCH … RETURN …) use the
1929    // lock-free reader (see `read_query`).
1930    let is_write = match is_write_query(&cypher) {
1931        Ok(b) => b,
1932        Err(e) => return err_response(e),
1933    };
1934
1935    if as_of.is_some() && is_write {
1936        return err_response("as_of (time-travel) queries are read-only");
1937    }
1938
1939    let rs = if is_write {
1940        let db = state.db.clone();
1941        match blocking_write(move || db.write().query_write(&cypher, &params)).await {
1942            Ok(rs) => rs,
1943            Err(resp) => return resp,
1944        }
1945    } else if let Some(commit) = as_of {
1946        match state.db.read().query_at(commit, &cypher, &params) {
1947            Ok(rs) => rs,
1948            Err(e) => return graph_err(e),
1949        }
1950    } else {
1951        match read_query(&state.db, &cypher, &params) {
1952            Ok(rs) => rs,
1953            Err(e) => return graph_err(e),
1954        }
1955    };
1956
1957    format_query_result(rs, format)
1958}
1959
1960// ── /schema/indexes and /schema/constraints (TY-11) ─────────────────────────
1961//
1962// Each route runs what the matching Cypher DDL statement runs
1963// (`GraphDb::schema_*`, `SHOW …`), so HTTP, Cypher and the CLI cannot
1964// disagree: a row has the fields and values `SHOW` gives it, a role-bound
1965// caller sees what its `SHOW` shows, and a write is refused to the callers a
1966// DDL write is refused to.
1967
1968/// The rows of `SHOW INDEXES` / `SHOW CONSTRAINTS` as objects, narrowed to
1969/// what `identity` can see exactly as the same statement through `/query` is.
1970fn schema_listing_response(state: &AppState, identity: &AuthIdentity, statement: &str) -> Response {
1971    let params = BTreeMap::new();
1972    let rs = match identity {
1973        AuthIdentity::Role(role_name) => {
1974            let snap = state.db.reader();
1975            let role_mask = match snap.mask_for_role(role_name) {
1976                Ok(m) => m,
1977                Err(e) => return role_mask_err(e),
1978            };
1979            snap.query_masked(statement, &params, &role_mask)
1980        }
1981        AuthIdentity::Full | AuthIdentity::Write => state.db.reader().query(statement, &params),
1982        AuthIdentity::Anonymous => return forbidden("this endpoint requires a token"),
1983    };
1984    let rs = match rs {
1985        Ok(rs) => rs,
1986        Err(e) => return graph_err(e),
1987    };
1988    let columns: Vec<&String> = rs.columns().iter().collect();
1989    let rows: Vec<Js> = (0..rs.len())
1990        .map(|i| {
1991            let obj: serde_json::Map<String, Js> = columns
1992                .iter()
1993                .zip(rs.row(i))
1994                .map(|(col, cell)| {
1995                    let v = cell.as_ref().map_or(Js::Null, crate::json::value_to_json);
1996                    ((*col).clone(), v)
1997                })
1998                .collect();
1999            Js::Object(obj)
2000        })
2001        .collect();
2002    json_ok(Js::Array(rows))
2003}
2004
2005async fn schema_indexes_list(
2006    State(state): State<AppState>,
2007    Extension(identity): Extension<AuthIdentity>,
2008) -> Response {
2009    schema_listing_response(&state, &identity, "SHOW INDEXES")
2010}
2011
2012async fn schema_constraints_list(
2013    State(state): State<AppState>,
2014    Extension(identity): Extension<AuthIdentity>,
2015) -> Response {
2016    schema_listing_response(&state, &identity, "SHOW CONSTRAINTS")
2017}
2018
2019/// 403 for a role-bound token on a schema write, as a DDL statement gives it.
2020const SCHEMA_ROLE_REFUSAL: &str =
2021    "role-bound token: schema statements (CREATE/DROP INDEX, CREATE/DROP \
2022     CONSTRAINT) are not permitted";
2023
2024/// The refusal a schema write gets for `identity`, if any. A read-tier key never
2025/// reaches here (the auth middleware refuses it first), but carries a role.
2026fn refuse_schema_write(identity: &AuthIdentity) -> Option<Response> {
2027    match identity {
2028        AuthIdentity::Role(_) => Some(forbidden(SCHEMA_ROLE_REFUSAL)),
2029        AuthIdentity::Anonymous => Some(forbidden("this endpoint requires a token")),
2030        AuthIdentity::Full | AuthIdentity::Write => None,
2031    }
2032}
2033
2034/// A schema write's error as HTTP, by the kind the engine decided from the
2035/// state it found: a name nothing answers to is 404, a collision with what
2036/// exists (a taken name or pair, an index a constraint backs, existing
2037/// duplicates, a view-owned property) is 409, and a malformed request is 400.
2038fn schema_err(e: SchemaError) -> Response {
2039    let status = match e.kind {
2040        SchemaErrorKind::NotFound => StatusCode::NOT_FOUND,
2041        SchemaErrorKind::Conflict => StatusCode::CONFLICT,
2042        SchemaErrorKind::Invalid => StatusCode::BAD_REQUEST,
2043        SchemaErrorKind::Other => return graph_err(e.error),
2044    };
2045    let detail = match e.error {
2046        GraphError::QueryError { detail } | GraphError::RuleInvalid { detail } => detail,
2047        other => other.to_string(),
2048    };
2049    (status, Json(json!({ "error": detail }))).into_response()
2050}
2051
2052/// The refusal for a schema body that is not an object, or carries a field
2053/// outside `allowed`, naming it.
2054fn schema_body_check(body: &Js, allowed: &[&str]) -> std::result::Result<(), String> {
2055    let obj = body
2056        .as_object()
2057        .ok_or_else(|| "the body must be a JSON object".to_string())?;
2058    match obj.keys().find(|k| !allowed.contains(&k.as_str())) {
2059        Some(unknown) => Err(format!(
2060            "unknown field '{unknown}'; this route takes: {}",
2061            allowed.join(", ")
2062        )),
2063        None => Ok(()),
2064    }
2065}
2066
2067/// A required, non-empty string field of a schema request body.
2068fn schema_text<'a>(body: &'a Js, field: &str) -> std::result::Result<&'a str, String> {
2069    match body.get(field).and_then(Js::as_str) {
2070        Some(s) if !s.is_empty() => Ok(s),
2071        _ => Err(format!(
2072            "{field} is required and must be a non-empty string"
2073        )),
2074    }
2075}
2076
2077/// An optional `name`: absent or null is `None`; otherwise it must be an
2078/// identifier, so every surface (HTTP path, Cypher `DROP INDEX name`) can
2079/// address it.
2080fn schema_name(body: &Js) -> std::result::Result<Option<&str>, String> {
2081    match body.get("name") {
2082        None | Some(Js::Null) => Ok(None),
2083        Some(v) => match v.as_str() {
2084            Some(s) if is_schema_identifier(s) => Ok(Some(s)),
2085            _ => Err(
2086                "name must be a string of letters, digits and underscores, not starting with a digit"
2087                    .to_string(),
2088            ),
2089        },
2090    }
2091}
2092
2093fn is_schema_identifier(s: &str) -> bool {
2094    let mut chars = s.chars();
2095    chars
2096        .next()
2097        .is_some_and(|c| c.is_ascii_alphabetic() || c == '_')
2098        && chars.all(|c| c.is_ascii_alphanumeric() || c == '_')
2099}
2100
2101async fn schema_index_create(
2102    State(state): State<AppState>,
2103    Extension(identity): Extension<AuthIdentity>,
2104    Json(body): Json<Js>,
2105) -> Response {
2106    if let Some(refusal) = refuse_schema_write(&identity) {
2107        return refusal;
2108    }
2109    if let Err(e) = schema_body_check(&body, &["name", "label", "property", "kind"]) {
2110        return err_response(e);
2111    }
2112    let (label, property) = match (schema_text(&body, "label"), schema_text(&body, "property")) {
2113        (Ok(l), Ok(p)) => (l.to_string(), p.to_string()),
2114        (Err(e), _) | (_, Err(e)) => return err_response(e),
2115    };
2116    let name = match schema_name(&body) {
2117        Ok(n) => n.map(str::to_string),
2118        Err(e) => return err_response(e),
2119    };
2120    let kind =
2121        match body.get("kind") {
2122            None | Some(Js::Null) => "range",
2123            Some(v) => match v.as_str() {
2124                Some(k @ ("range" | "equality")) => k,
2125                _ => return err_response(
2126                    "kind must be \"range\" or \"equality\"; full-text indexes are not created \
2127                     over this route",
2128                ),
2129            },
2130        };
2131    let kind = kind.to_string();
2132    let db = state.db.clone();
2133    let created = tokio::task::spawn_blocking(move || {
2134        let mut g = db.write();
2135        match kind.as_str() {
2136            "equality" => g.schema_create_equality_index(name.as_deref(), &label, &property),
2137            _ => g.schema_create_range_index(name.as_deref(), &label, &property),
2138        }
2139        .map(|listed| (listed, kind, label, property))
2140    })
2141    .await;
2142    match created {
2143        Ok(Ok((listed, kind, label, property))) => (
2144            StatusCode::CREATED,
2145            Json(json!({
2146                "name": listed, "type": kind, "label": label,
2147                "property": property, "state": "online"
2148            })),
2149        )
2150            .into_response(),
2151        Ok(Err(e)) => schema_err(e),
2152        Err(_) => err_response("write task panicked"),
2153    }
2154}
2155
2156async fn schema_index_drop(
2157    State(state): State<AppState>,
2158    Extension(identity): Extension<AuthIdentity>,
2159    Path(name): Path<String>,
2160) -> Response {
2161    if let Some(refusal) = refuse_schema_write(&identity) {
2162        return refusal;
2163    }
2164    let db = state.db.clone();
2165    match tokio::task::spawn_blocking(move || db.write().schema_drop_index(&name)).await {
2166        Ok(Ok(())) => StatusCode::NO_CONTENT.into_response(),
2167        Ok(Err(e)) => schema_err(e),
2168        Err(_) => err_response("write task panicked"),
2169    }
2170}
2171
2172async fn schema_constraint_create(
2173    State(state): State<AppState>,
2174    Extension(identity): Extension<AuthIdentity>,
2175    Json(body): Json<Js>,
2176) -> Response {
2177    if let Some(refusal) = refuse_schema_write(&identity) {
2178        return refusal;
2179    }
2180    if let Err(e) = schema_body_check(&body, &["name", "label", "property", "type"]) {
2181        return err_response(e);
2182    }
2183    let (label, property) = match (schema_text(&body, "label"), schema_text(&body, "property")) {
2184        (Ok(l), Ok(p)) => (l.to_string(), p.to_string()),
2185        (Err(e), _) | (_, Err(e)) => return err_response(e),
2186    };
2187    // A constraint needs a name (Cypher DDL requires one), so there is none to invent.
2188    let name = match schema_name(&body) {
2189        Ok(Some(n)) => n.to_string(),
2190        Ok(None) => return err_response("name is required for a constraint"),
2191        Err(e) => return err_response(e),
2192    };
2193    match body.get("type") {
2194        None | Some(Js::Null) => {}
2195        Some(v) if v.as_str() == Some("unique") => {}
2196        Some(_) => return err_response("type must be \"unique\", the only constraint type"),
2197    }
2198    let db = state.db.clone();
2199    let reply = json!({
2200        "name": name, "type": "unique", "label": label, "property": property
2201    });
2202    match tokio::task::spawn_blocking(move || {
2203        db.write()
2204            .schema_create_constraint(&name, &label, &property)
2205    })
2206    .await
2207    {
2208        Ok(Ok(())) => (StatusCode::CREATED, Json(reply)).into_response(),
2209        Ok(Err(e)) => schema_err(e),
2210        Err(_) => err_response("write task panicked"),
2211    }
2212}
2213
2214async fn schema_constraint_drop(
2215    State(state): State<AppState>,
2216    Extension(identity): Extension<AuthIdentity>,
2217    Path(name): Path<String>,
2218) -> Response {
2219    if let Some(refusal) = refuse_schema_write(&identity) {
2220        return refusal;
2221    }
2222    let db = state.db.clone();
2223    match tokio::task::spawn_blocking(move || db.write().schema_drop_constraint(&name)).await {
2224        Ok(Ok(())) => StatusCode::NO_CONTENT.into_response(),
2225        Ok(Err(e)) => schema_err(e),
2226        Err(_) => err_response("write task panicked"),
2227    }
2228}
2229
2230/// An unmasked, live (non-`as_of`) read query, on the lock-free reader. An
2231/// equality lookup on a declared index is served from the snapshot's own
2232/// index, so this holds for indexed stores too.
2233fn read_query(
2234    db: &SharedDb,
2235    cypher: &str,
2236    params: &BTreeMap<String, Value>,
2237) -> mushroomdb::Result<ResultSet> {
2238    db.reader().query(cypher, params)
2239}
2240
2241/// The response for a full-token masked `/query`, from either path.
2242fn masked_query_response(out: mushroomdb::Result<ResultSet>, format: &str) -> Response {
2243    match out {
2244        Ok(rs) => format_query_result(rs, format),
2245        Err(GraphError::MaskedReadOnly) => (
2246            StatusCode::BAD_REQUEST,
2247            Json(json!({"error": "masked queries are read-only"})),
2248        )
2249            .into_response(),
2250        Err(e) => graph_err(e),
2251    }
2252}
2253
2254/// [`NodeMask::from_keys`] against a snapshot: keys it does not hold are
2255/// ignored, and the mode is `Omit`.
2256fn snapshot_mask_from_keys<'a>(
2257    snap: &ReaderSnapshot,
2258    keys: impl IntoIterator<Item = &'a str>,
2259) -> NodeMask {
2260    NodeMask::from_ids(keys.into_iter().filter_map(|k| snap.resolve_key(k)))
2261}
2262
2263/// The client mask a full-token `/node/*` route reads from `mask=` and
2264/// `stub_hidden=`, resolved against `snap`.
2265fn snapshot_client_mask(snap: &ReaderSnapshot, mask_param: &str, stub_hidden: bool) -> NodeMask {
2266    let keys = mask_param
2267        .split(',')
2268        .map(str::trim)
2269        .filter(|s| !s.is_empty());
2270    let m = snapshot_mask_from_keys(snap, keys);
2271    if stub_hidden {
2272        m.with_mode(MaskMode::Stub)
2273    } else {
2274        m
2275    }
2276}
2277
2278/// `GET /stats` — store-wide counts, including the `namespaces` roster.
2279///
2280/// Role tokens are denied, as they always were, and that is also what settles
2281/// the namespace roster on this surface: the roster names every namespace and
2282/// its live count, which is a list of other tenants, so narrowing it for a role
2283/// token would be a second answer to a question this route never answers for one
2284/// at all. A full-access token sees the whole roster; a tenant-scoped client gets
2285/// 403 here and asks MCP `stats` with its `role`, which narrows.
2286async fn stats(
2287    State(state): State<AppState>,
2288    Extension(identity): Extension<AuthIdentity>,
2289) -> Response {
2290    // v1: deny role tokens — raw counts leak graph size beyond the role's
2291    // subgraph, and `namespaces` would name every other tenant.
2292    if let AuthIdentity::Role(_) = identity {
2293        return forbidden("role-bound token: /stats requires a full-access token");
2294    }
2295    if let Some(refusal) = refuse_write_tier(&identity) {
2296        return refusal;
2297    }
2298    let snap = {
2299        let g = state.db.read();
2300        g.stats()
2301    };
2302    match serde_json::to_value(&snap) {
2303        Ok(v) => json_ok(v),
2304        Err(e) => err_response(e.to_string()),
2305    }
2306}
2307
2308/// The continuous-backup counters `GET /metrics` reports as
2309/// `mushroomdb_backup_*`. The serve loop registers them with
2310/// [`register_backup_metrics`] when backup is configured; without that, no
2311/// backup series is emitted at all (absent, not zero, so an alert on
2312/// `mushroomdb_backup_lag_seconds` cannot be satisfied by a store that was
2313/// never backed up).
2314#[derive(Debug, Clone)]
2315pub struct BackupMetrics {
2316    /// Failed backup steps since start.
2317    pub error_count: Arc<AtomicU64>,
2318    /// Unix seconds of the last moment everything durable was shipped; 0
2319    /// before the first.
2320    pub last_success_ts: Arc<AtomicU64>,
2321    /// Seconds the oldest durable, unshipped WAL byte has waited.
2322    pub lag_s: Arc<AtomicU64>,
2323    /// Whether the worker reports itself healthy.
2324    pub healthy: Arc<AtomicBool>,
2325    /// Whether the worker holds the bucket lease right now.
2326    pub lease_held: Arc<AtomicBool>,
2327}
2328
2329/// The Bolt listener's counters `GET /metrics` reports as
2330/// `mushroomdb_bolt_connections_*`; absent when Bolt is off.
2331#[derive(Debug, Clone)]
2332pub struct BoltMetrics {
2333    /// Connections being served now.
2334    pub open: Arc<AtomicU64>,
2335    /// Connections admitted since the listener started.
2336    pub accepted: Arc<AtomicU64>,
2337}
2338
2339static BOLT_METRICS: std::sync::Mutex<Option<BoltMetrics>> = std::sync::Mutex::new(None);
2340
2341/// Report `metrics` on `GET /metrics` from now on. Like
2342/// [`register_backup_metrics`] this is per process: one process serves one
2343/// store's metrics.
2344pub fn register_bolt_metrics(metrics: BoltMetrics) {
2345    *BOLT_METRICS.lock().unwrap_or_else(PoisonError::into_inner) = Some(metrics);
2346}
2347
2348fn registered_bolt_metrics() -> Option<BoltSample> {
2349    let guard = BOLT_METRICS.lock().unwrap_or_else(PoisonError::into_inner);
2350    guard.as_ref().map(|m| BoltSample {
2351        open: m.open.load(Ordering::Acquire),
2352        accepted_total: m.accepted.load(Ordering::Acquire),
2353    })
2354}
2355
2356/// One reading of the Bolt counters.
2357struct BoltSample {
2358    open: u64,
2359    accepted_total: u64,
2360}
2361
2362static BACKUP_METRICS: std::sync::Mutex<Option<BackupMetrics>> = std::sync::Mutex::new(None);
2363
2364/// Report `metrics` on `GET /metrics` from now on. Replaces an earlier call.
2365///
2366/// The registration is process-wide: one process serves one store's metrics,
2367/// as `mushroomdb serve` does. Serving several stores from one process would
2368/// need these threaded through `AppState` instead.
2369pub fn register_backup_metrics(metrics: BackupMetrics) {
2370    *BACKUP_METRICS
2371        .lock()
2372        .unwrap_or_else(PoisonError::into_inner) = Some(metrics);
2373}
2374
2375fn registered_backup_metrics() -> Option<BackupSample> {
2376    let guard = BACKUP_METRICS
2377        .lock()
2378        .unwrap_or_else(PoisonError::into_inner);
2379    guard.as_ref().map(|m| BackupSample {
2380        upload_errors_total: m.error_count.load(Ordering::Acquire),
2381        last_success_ts: m.last_success_ts.load(Ordering::Acquire),
2382        lag_s: m.lag_s.load(Ordering::Acquire),
2383        healthy: m.healthy.load(Ordering::Acquire),
2384        lease_held: m.lease_held.load(Ordering::Acquire),
2385    })
2386}
2387
2388fn unix_seconds_now() -> u64 {
2389    std::time::SystemTime::now()
2390        .duration_since(std::time::UNIX_EPOCH)
2391        .map_or(0, |d| d.as_secs())
2392}
2393
2394/// One reading of the backup counters.
2395struct BackupSample {
2396    upload_errors_total: u64,
2397    last_success_ts: u64,
2398    lag_s: u64,
2399    healthy: bool,
2400    lease_held: bool,
2401}
2402
2403/// Everything the text exposition reports, read once.
2404struct PrometheusSample {
2405    nodes_live: u64,
2406    nodes_tombstoned: u64,
2407    edges_live: u64,
2408    commits_total: u64,
2409    wal_size_bytes: Option<u64>,
2410    rss_bytes: Option<u64>,
2411    process_start_time_seconds: u64,
2412    slow_queries_total: u64,
2413    transactions_open: u64,
2414    bolt: Option<BoltSample>,
2415    backup: Option<BackupSample>,
2416}
2417
2418/// Whether `Accept` asks for the Prometheus text format rather than JSON.
2419///
2420/// JSON is the default: a caller with no `Accept`, `*/*`, a browser's
2421/// `text/html,...` or a client that lists `application/json` among others
2422/// (`application/json, text/plain, */*`) gets JSON. Only a caller that names
2423/// `text/plain` or `application/openmetrics-text` and not JSON gets text, which
2424/// is what a Prometheus scraper sends. Quality values are not weighed.
2425fn wants_prometheus_text(headers: &HeaderMap) -> bool {
2426    let mut text = false;
2427    for value in headers.get_all(header::ACCEPT) {
2428        let Ok(value) = value.to_str() else { continue };
2429        for range in value.split(',') {
2430            let media = range
2431                .split(';')
2432                .next()
2433                .unwrap_or("")
2434                .trim()
2435                .to_ascii_lowercase();
2436            match media.as_str() {
2437                "application/json" => return false,
2438                "text/plain" | "application/openmetrics-text" => text = true,
2439                _ => {}
2440            }
2441        }
2442    }
2443    text
2444}
2445
2446/// One metric family in the text exposition format, version 0.0.4.
2447fn write_family(out: &mut String, name: &str, kind: &str, help: &str, value: impl Display) {
2448    let _ = write!(
2449        out,
2450        "# HELP {name} {help}\n# TYPE {name} {kind}\n{name} {value}\n"
2451    );
2452}
2453
2454/// The Prometheus text exposition (version 0.0.4), written by hand: no
2455/// labels, so no cardinality to manage, and a gauge whose source is
2456/// unavailable on this platform (`rss_bytes`) is left out rather than
2457/// reported as 0.
2458fn format_prometheus(s: &PrometheusSample) -> String {
2459    let mut out = String::with_capacity(2048);
2460    write_family(
2461        &mut out,
2462        "mushroomdb_nodes_live",
2463        "gauge",
2464        "Live (non-tombstoned) nodes.",
2465        s.nodes_live,
2466    );
2467    write_family(
2468        &mut out,
2469        "mushroomdb_nodes_tombstoned",
2470        "gauge",
2471        "Soft-deleted nodes not yet compacted away.",
2472        s.nodes_tombstoned,
2473    );
2474    write_family(
2475        &mut out,
2476        "mushroomdb_edges_live",
2477        "gauge",
2478        "Edges in the store.",
2479        s.edges_live,
2480    );
2481    write_family(
2482        &mut out,
2483        "mushroomdb_commits_total",
2484        "counter",
2485        "Commits since the store was created (the commit sequence number).",
2486        s.commits_total,
2487    );
2488    if let Some(bytes) = s.wal_size_bytes {
2489        write_family(
2490            &mut out,
2491            "mushroomdb_wal_size_bytes",
2492            "gauge",
2493            "On-disk size of the write-ahead log in bytes.",
2494            bytes,
2495        );
2496    }
2497    if let Some(bytes) = s.rss_bytes {
2498        write_family(
2499            &mut out,
2500            "mushroomdb_rss_bytes",
2501            "gauge",
2502            "Resident set size of the server process in bytes.",
2503            bytes,
2504        );
2505    }
2506    write_family(
2507        &mut out,
2508        "mushroomdb_process_start_time_seconds",
2509        "gauge",
2510        "Start time of the server as seconds since the Unix epoch.",
2511        s.process_start_time_seconds,
2512    );
2513    write_family(
2514        &mut out,
2515        "mushroomdb_slow_queries_total",
2516        "counter",
2517        "Read queries that met the slow-query threshold since start.",
2518        s.slow_queries_total,
2519    );
2520    write_family(
2521        &mut out,
2522        "mushroomdb_http_transactions_open",
2523        "gauge",
2524        "Open interactive HTTP transactions.",
2525        s.transactions_open,
2526    );
2527    if let Some(b) = &s.bolt {
2528        write_family(
2529            &mut out,
2530            "mushroomdb_bolt_connections_open",
2531            "gauge",
2532            "Bolt connections being served now.",
2533            b.open,
2534        );
2535        write_family(
2536            &mut out,
2537            "mushroomdb_bolt_connections_accepted_total",
2538            "counter",
2539            "Bolt connections admitted since the listener started.",
2540            b.accepted_total,
2541        );
2542    }
2543    if let Some(b) = &s.backup {
2544        write_family(
2545            &mut out,
2546            "mushroomdb_backup_upload_errors_total",
2547            "counter",
2548            "Failed continuous-backup steps since start.",
2549            b.upload_errors_total,
2550        );
2551        write_family(
2552            &mut out,
2553            "mushroomdb_backup_last_success_timestamp_seconds",
2554            "gauge",
2555            "Unix time of the last moment all durable data was shipped; 0 before the first.",
2556            b.last_success_ts,
2557        );
2558        write_family(
2559            &mut out,
2560            "mushroomdb_backup_lag_seconds",
2561            "gauge",
2562            "Seconds the oldest durable, unshipped WAL byte has waited; 0 when all is shipped.",
2563            b.lag_s,
2564        );
2565        write_family(
2566            &mut out,
2567            "mushroomdb_backup_healthy",
2568            "gauge",
2569            "1 while the backup worker reports itself healthy, else 0.",
2570            u8::from(b.healthy),
2571        );
2572        write_family(
2573            &mut out,
2574            "mushroomdb_backup_lease_held",
2575            "gauge",
2576            "1 while this instance holds the backup bucket lease, else 0.",
2577            u8::from(b.lease_held),
2578        );
2579    }
2580    out
2581}
2582
2583async fn metrics(
2584    State(state): State<AppState>,
2585    Extension(identity): Extension<AuthIdentity>,
2586    headers: HeaderMap,
2587) -> Response {
2588    // Mirror /stats auth: deny role tokens — counters leak graph size.
2589    if let AuthIdentity::Role(_) = identity {
2590        return forbidden("role-bound token: /metrics requires a full-access token");
2591    }
2592    if let Some(refusal) = refuse_write_tier(&identity) {
2593        return refusal;
2594    }
2595    let snap = state.db.reader();
2596    let (nodes_live, nodes_tombstoned, edges) = snap.counts();
2597    let commit_seq = snap.commit_seq();
2598    let wal_size_bytes = snap.wal_size_bytes().ok();
2599    let slow_snap = snap.slow_query_snapshot();
2600    if wants_prometheus_text(&headers) {
2601        let text = format_prometheus(&PrometheusSample {
2602            nodes_live: nodes_live as u64,
2603            nodes_tombstoned: nodes_tombstoned as u64,
2604            edges_live: edges,
2605            commits_total: commit_seq,
2606            wal_size_bytes,
2607            rss_bytes: rss_bytes(),
2608            process_start_time_seconds: state.started_unix_s,
2609            slow_queries_total: slow_snap.count,
2610            transactions_open: state.txs.open_count() as u64,
2611            bolt: registered_bolt_metrics(),
2612            backup: registered_backup_metrics(),
2613        });
2614        return (
2615            StatusCode::OK,
2616            [
2617                (header::CONTENT_TYPE, PROMETHEUS_CONTENT_TYPE),
2618                (header::VARY, "Accept"),
2619            ],
2620            text,
2621        )
2622            .into_response();
2623    }
2624    let uptime_s = state.started_at.elapsed().as_secs();
2625    let slow_entries: Vec<Js> = slow_snap
2626        .last
2627        .iter()
2628        .map(|e| {
2629            json!({
2630                "ms": e.ms,
2631                "query": e.query,
2632                "at_commit": e.at_commit,
2633            })
2634        })
2635        .collect();
2636    let mut response = json_ok(json!({
2637        "nodes_live": nodes_live,
2638        "nodes_tombstoned": nodes_tombstoned,
2639        "edges": edges,
2640        "commit_seq": commit_seq,
2641        "wal_size_bytes": wal_size_bytes,
2642        "rss_bytes": rss_bytes(),
2643        "uptime_s": uptime_s,
2644        "slow_queries": {
2645            "threshold_ms": slow_snap.threshold_ms,
2646            "count": slow_snap.count,
2647            "last": slow_entries,
2648        },
2649    }));
2650    response
2651        .headers_mut()
2652        .insert(header::VARY, HeaderValue::from_static("Accept"));
2653    response
2654}
2655
2656/// `Content-Type` of the Prometheus text exposition.
2657const PROMETHEUS_CONTENT_TYPE: &str = "text/plain; version=0.0.4; charset=utf-8";
2658
2659/// Resident set size of the current process in bytes.
2660///
2661/// macOS: mach task_info (MACH_TASK_BASIC_INFO).
2662/// Linux: /proc/self/statm RSS pages × page size.
2663/// Other: always returns None.
2664///
2665/// Never panics; returns None on any failure.
2666fn rss_bytes() -> Option<u64> {
2667    #[cfg(target_os = "macos")]
2668    {
2669        // MACH_TASK_BASIC_INFO flavor = 20.
2670        // struct mach_task_basic_info { u64 virtual, u64 resident,
2671        //   u64 resident_max, [u32;2] user_time, [u32;2] sys_time,
2672        //   i32 policy, i32 suspend_count } = 48 bytes = 12 natural_t(u32)s.
2673        const MACH_TASK_BASIC_INFO: u32 = 20;
2674        const MACH_TASK_BASIC_INFO_COUNT: u32 = 12;
2675
2676        #[repr(C)]
2677        struct MachTaskBasicInfo {
2678            virtual_size: u64,
2679            resident_size: u64,
2680            resident_size_max: u64,
2681            user_time: [u32; 2],
2682            system_time: [u32; 2],
2683            policy: i32,
2684            suspend_count: i32,
2685        }
2686
2687        extern "C" {
2688            fn mach_task_self() -> u32;
2689            fn task_info(
2690                target_task: u32,
2691                flavor: u32,
2692                task_info_out: *mut std::ffi::c_void,
2693                task_info_cnt: *mut u32,
2694            ) -> i32;
2695        }
2696
2697        let mut info: MachTaskBasicInfo = unsafe { std::mem::zeroed() };
2698        let mut count = MACH_TASK_BASIC_INFO_COUNT;
2699        let ret = unsafe {
2700            task_info(
2701                mach_task_self(),
2702                MACH_TASK_BASIC_INFO,
2703                &mut info as *mut _ as *mut _,
2704                &mut count,
2705            )
2706        };
2707        if ret != 0 {
2708            return None; // KERN_SUCCESS = 0
2709        }
2710        return Some(info.resident_size);
2711    }
2712
2713    #[cfg(target_os = "linux")]
2714    {
2715        // /proc/self/statm: "vsize rss ..." in pages.
2716        let content = std::fs::read_to_string("/proc/self/statm").ok()?;
2717        let mut parts = content.split_whitespace();
2718        let _vsize = parts.next()?;
2719        let rss_pages: u64 = parts.next()?.parse().ok()?;
2720        let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) };
2721        if page_size <= 0 {
2722            return None;
2723        }
2724        return Some(rss_pages * page_size as u64);
2725    }
2726
2727    #[allow(unreachable_code)]
2728    None
2729}
2730
2731async fn ingest(
2732    State(state): State<AppState>,
2733    Extension(identity): Extension<AuthIdentity>,
2734    Json(body): Json<Js>,
2735) -> Response {
2736    let label = match body.get("label").and_then(Js::as_str) {
2737        Some(s) => s.to_string(),
2738        None => return err_response("missing label"),
2739    };
2740    let rows = match body.get("rows") {
2741        Some(r) => r,
2742        None => return err_response("missing rows"),
2743    };
2744    let mut converted = match json_to_rows(rows) {
2745        Ok(c) => c,
2746        Err(e) => return graph_err(e),
2747    };
2748    let opts = match ingest_options(body.get("options")) {
2749        Ok(o) => o,
2750        Err(e) => return err_response(e),
2751    };
2752    // `namespace` applies to every node this call creates.
2753    let namespace = match namespace_arg(body.get("namespace")) {
2754        Ok(n) => n,
2755        Err(e) => return err_response(e),
2756    };
2757    if let Err(e) = stamp_namespace(&mut converted.rows, namespace.as_deref()) {
2758        return err_response(e);
2759    }
2760    let taken = std::mem::take(&mut converted.rows);
2761    let edges = match body.get("edges") {
2762        None | Some(Js::Null) => Vec::new(),
2763        Some(raw) => match parse_ingest_edges(raw) {
2764            Ok(e) => e,
2765            Err(e) => return err_response(e),
2766        },
2767    };
2768    let db = state.db.clone();
2769
2770    // Role token: route to ingest_with_edges_authz (scope + mask resolved under
2771    // the write lock, §5 discipline; §7.3 — create_labels required, enforced by
2772    // the engine's per-op authz check inside commit_logged_batch).
2773    if let AuthIdentity::Role(role_name) = &identity {
2774        let role = role_name.clone();
2775        return match blocking_write(move || {
2776            db.write()
2777                .ingest_with_edges_authz(&role, &label, taken, &opts, &edges)
2778        })
2779        .await
2780        {
2781            Ok(r) => {
2782                let report = converted.into_report(r);
2783                match serde_json::to_value(&report) {
2784                    Ok(v) => json_ok(v),
2785                    Err(e) => err_response(e.to_string()),
2786                }
2787            }
2788            Err(resp) => resp,
2789        };
2790    }
2791
2792    let report =
2793        match blocking_write(move || db.write().ingest_with_edges(&label, taken, &opts, &edges))
2794            .await
2795        {
2796            Ok(r) => converted.into_report(r),
2797            Err(resp) => return resp,
2798        };
2799    match serde_json::to_value(&report) {
2800        Ok(v) => json_ok(v),
2801        Err(e) => err_response(e.to_string()),
2802    }
2803}
2804
2805/// `GET /suggest` — profile the database and return rule suggestions.
2806///
2807/// # Locking and blocking strategy
2808///
2809/// `suggest_rules_with_config` is CPU-intensive and synchronous. Running it on a
2810/// Tokio worker thread would starve the executor. This handler offloads the work to
2811/// `tokio::task::spawn_blocking`, which uses the blocking thread-pool. The
2812/// `std::sync::RwLock` read guard is acquired and held inside the blocking task —
2813/// reads don't block other reads; writes wait for the guard to drop. The global
2814/// budget (`SuggestConfig::global_budget_ms`, default 5 s) caps lock-hold time.
2815async fn suggest(
2816    State(state): State<AppState>,
2817    Extension(identity): Extension<AuthIdentity>,
2818) -> Response {
2819    // v1: deny role tokens — suggest scans the full graph and would reveal
2820    // existence of nodes outside the role's mask.
2821    if let AuthIdentity::Role(_) = identity {
2822        return forbidden("role-bound token: /suggest requires a full-access token");
2823    }
2824    let db = state.db.clone();
2825    match tokio::task::spawn_blocking(move || {
2826        let config = SuggestConfig::default();
2827        db.read()
2828            .suggest_rules_with_config(&config, SUGGEST_DEFAULT_SEED)
2829    })
2830    .await
2831    {
2832        Ok(report) => json_ok(serde_json::to_value(&report).unwrap_or_else(|_| json!({}))),
2833        Err(_) => err_response("suggest task panicked"),
2834    }
2835}
2836
2837async fn create_rule(
2838    State(state): State<AppState>,
2839    Extension(identity): Extension<AuthIdentity>,
2840    Json(body): Json<Js>,
2841) -> Response {
2842    if let AuthIdentity::Role(_) = identity {
2843        return forbidden("role-bound token: writes are not permitted");
2844    }
2845    let def = match rule_def_from_json(body) {
2846        Ok(d) => d,
2847        Err(e) => return err_response(e),
2848    };
2849    let name = def.name.clone();
2850    let db = state.db.clone();
2851    if let Err(resp) = blocking_write(move || db.write().create_rule(def)).await {
2852        return resp;
2853    }
2854    // A corpus too large to index in one commit leaves the rule installed but
2855    // deriving nothing, so the route says "accepted", not "done", and hands
2856    // back the progress the caller can poll on `GET /stats`.
2857    //
2858    // Read under a *separate* lock from the write above, so a concurrent pump
2859    // can finish the build in between and this route then answers 200 for a
2860    // create that really did defer. Benign — 200 means "the edges are there",
2861    // which by then they are — and the caller's contract is to poll `GET /stats`
2862    // either way.
2863    let building = state
2864        .db
2865        .read()
2866        .builds_in_progress()
2867        .into_iter()
2868        .find(|b| b.rule == name);
2869    match building {
2870        Some(b) => (
2871            StatusCode::ACCEPTED,
2872            Json(json!({"rule": name, "building": {"indexed": b.indexed, "total": b.total}})),
2873        )
2874            .into_response(),
2875        None => json_ok(json!({"ok": true, "name": name})),
2876    }
2877}
2878
2879async fn explain(
2880    State(state): State<AppState>,
2881    Extension(identity): Extension<AuthIdentity>,
2882    Query(qs): Query<BTreeMap<String, String>>,
2883) -> Response {
2884    // v1: deny role tokens — explain reveals hidden-node linkage through rules.
2885    if let AuthIdentity::Role(_) = identity {
2886        return forbidden(
2887            "role-bound token: /explain requires a full-access token \
2888             (v1: explain may reveal hidden-node linkage; revisit when stubs land)",
2889        );
2890    }
2891    let a = match qs.get("a") {
2892        Some(s) if !s.is_empty() => s.clone(),
2893        _ => return err_response("missing query param a"),
2894    };
2895    let b = match qs.get("b") {
2896        Some(s) if !s.is_empty() => s.clone(),
2897        _ => return err_response("missing query param b"),
2898    };
2899    let out = {
2900        let g = state.db.read();
2901        g.explain(&a, &b)
2902    };
2903    match out {
2904        Ok(v) => match serde_json::to_value(&v) {
2905            Ok(j) => json_ok(j),
2906            Err(e) => err_response(e.to_string()),
2907        },
2908        Err(e) => graph_err(e),
2909    }
2910}
2911
2912async fn node_info(
2913    State(state): State<AppState>,
2914    Extension(identity): Extension<AuthIdentity>,
2915    Path(key): Path<String>,
2916    Query(qs): Query<BTreeMap<String, String>>,
2917) -> Response {
2918    if let AuthIdentity::Role(ref role_name) = identity {
2919        // Role-token path: hard-coded Omit mode; hidden keys are indistinguishable
2920        // from absent keys.  `stub_hidden` query param is silently ignored here —
2921        // role paths must NEVER produce stubs (RBAC invariant).
2922        let snap = state.db.reader();
2923        let role_mask = match snap.mask_for_role(role_name) {
2924            Ok(m) => m,
2925            Err(e) => return role_mask_err(e),
2926        };
2927        if !snap
2928            .resolve_key(&key)
2929            .is_some_and(|id| role_mask.contains_id(id))
2930        {
2931            return key_not_found(key);
2932        }
2933        return match snap.node_info(&key) {
2934            Some(info) => json_ok(node_info_json(&info)),
2935            None => key_not_found(key),
2936        };
2937    }
2938
2939    // Full-token path: optional client mask + stub_hidden via query params.
2940    // `mask=key1,key2` — comma-separated visible keys (empty string = no mask).
2941    // `stub_hidden=true` — opt into MaskMode::Stub for this request.
2942    let mask_param = qs.get("mask").map(String::as_str).unwrap_or("").trim();
2943    if !mask_param.is_empty() {
2944        let stub_hidden = qs
2945            .get("stub_hidden")
2946            .map(|v| v == "true" || v == "1")
2947            .unwrap_or(false);
2948        let snap = state.db.reader();
2949        if let Err(e) = snap.ensure_readable() {
2950            return graph_err(e);
2951        }
2952        let mask = snapshot_client_mask(&snap, mask_param, stub_hidden);
2953        return match snap.node_info_masked(&key, &mask) {
2954            Some(mushroomdb::MaskedNodeResult::Visible(info)) => json_ok(node_info_json(&info)),
2955            Some(mushroomdb::MaskedNodeResult::Restricted) => {
2956                json_ok(crate::json::stub_node_json(&key))
2957            }
2958            None => key_not_found(key),
2959        };
2960    }
2961
2962    let snap = state.db.reader();
2963    if let Err(e) = snap.ensure_readable() {
2964        return graph_err(e);
2965    }
2966    match snap.node_info(&key) {
2967        Some(info) => json_ok(node_info_json(&info)),
2968        None => key_not_found(key),
2969    }
2970}
2971
2972async fn node_edges(
2973    State(state): State<AppState>,
2974    Extension(identity): Extension<AuthIdentity>,
2975    Path(key): Path<String>,
2976    Query(qs): Query<BTreeMap<String, String>>,
2977) -> Response {
2978    if let AuthIdentity::Role(ref role_name) = identity {
2979        // Role-token path: hard-coded Omit mode.  `stub_hidden` query param is
2980        // silently ignored — role paths must NEVER produce stubs (RBAC invariant).
2981        //
2982        // The subject check and the hidden-endpoint filter live in
2983        // `ReaderSnapshot::node_edges_scoped`, so every scoped caller gets them,
2984        // not only this route.
2985        let snap = state.db.reader();
2986        let role_mask = match snap.mask_for_role(role_name) {
2987            Ok(m) => m,
2988            Err(e) => return role_mask_err(e),
2989        };
2990        return match snap.node_edges_scoped(&key, &role_mask) {
2991            Ok(edges) => json_ok(node_edges_json(&edges)),
2992            Err(GraphError::KeyNotFound { key }) => key_not_found(key),
2993            Err(e) => graph_err(e),
2994        };
2995    }
2996
2997    // Full-token path: optional client mask + stub_hidden via query params.
2998    let mask_param = qs.get("mask").map(String::as_str).unwrap_or("").trim();
2999    if !mask_param.is_empty() {
3000        let stub_hidden = qs
3001            .get("stub_hidden")
3002            .map(|v| v == "true" || v == "1")
3003            .unwrap_or(false);
3004        let snap = state.db.reader();
3005        if let Err(e) = snap.ensure_readable() {
3006            return graph_err(e);
3007        }
3008        let mask = snapshot_client_mask(&snap, mask_param, stub_hidden);
3009        return match snap.node_edges_masked(&key, &mask) {
3010            Ok(edges) => json_ok(crate::json::masked_edges_json(&edges)),
3011            Err(GraphError::KeyNotFound { key }) => key_not_found(key),
3012            Err(e) => graph_err(e),
3013        };
3014    }
3015
3016    match state.db.reader().node_edges(&key) {
3017        Ok(edges) => json_ok(node_edges_json(&edges)),
3018        Err(GraphError::KeyNotFound { key }) => key_not_found(key),
3019        Err(e) => graph_err(e),
3020    }
3021}
3022
3023async fn neighborhood(
3024    State(state): State<AppState>,
3025    Extension(identity): Extension<AuthIdentity>,
3026    Path(key): Path<String>,
3027    Query(qs): Query<BTreeMap<String, String>>,
3028) -> Response {
3029    let depth = match resolve_neighborhood_depth(qs.get("depth").map(String::as_str)) {
3030        Ok(d) => d,
3031        Err(e) => return err_response(e),
3032    };
3033    let dir = match qs.get("dir").map(String::as_str).unwrap_or("both") {
3034        s if s.eq_ignore_ascii_case("out") => Dir::Out,
3035        s if s.eq_ignore_ascii_case("in") => Dir::In,
3036        s if s.eq_ignore_ascii_case("both") => Dir::Both,
3037        other => return err_response(format!("unknown dir: {other}")),
3038    };
3039    let edge_type_names: Option<Vec<String>> = qs.get("edge_types").map(|s| {
3040        s.split(',')
3041            .map(str::trim)
3042            .filter(|t| !t.is_empty())
3043            .map(str::to_string)
3044            .collect()
3045    });
3046    let etype_refs: Option<Vec<&str>> = edge_type_names
3047        .as_ref()
3048        .map(|v| v.iter().map(String::as_str).collect());
3049    if let AuthIdentity::Role(ref role_name) = identity {
3050        // Lock-free epoch snapshot: mask and neighborhood BFS on same frozen state
3051        // (constraint 2 — RBAC mask coherence).
3052        let snap = state.db.reader();
3053        let role_mask = match snap.mask_for_role(role_name) {
3054            Ok(m) => m,
3055            Err(e) => return role_mask_err(e),
3056        };
3057        // The scoped BFS: the subject is checked first (a hidden key answers as
3058        // an absent one), and hidden nodes are excluded from results AND cannot
3059        // be used as traversal intermediaries (never-leak invariant).
3060        return match snap.neighborhood_scoped(&key, depth, etype_refs.as_deref(), dir, &role_mask) {
3061            Ok(rs) => json_ok(result_set_json(&rs)),
3062            Err(GraphError::KeyNotFound { key }) => key_not_found(key),
3063            Err(e) => graph_err(e),
3064        };
3065    }
3066    // Full-token path: optional client mask + stub_hidden via query params.
3067    // `mask=key1,key2` — comma-separated visible keys.
3068    // `stub_hidden=true` — opt into MaskMode::Stub; hidden direct neighbours
3069    // appear as stub rows (label: null) in the result; BFS does not expand
3070    // through them in either mode.
3071    let mask_param = qs.get("mask").map(String::as_str).unwrap_or("").trim();
3072    if !mask_param.is_empty() {
3073        let stub_hidden = qs
3074            .get("stub_hidden")
3075            .map(|v| v == "true" || v == "1")
3076            .unwrap_or(false);
3077        let snap = state.db.reader();
3078        if let Err(e) = snap.ensure_readable() {
3079            return graph_err(e);
3080        }
3081        let mask = snapshot_client_mask(&snap, mask_param, stub_hidden);
3082        return match snap.neighborhood_masked(&key, depth, etype_refs.as_deref(), dir, &mask) {
3083            Some(rs) => json_ok(result_set_json(&rs)),
3084            None => key_not_found(key),
3085        };
3086    }
3087
3088    // Unmasked full-token path (no mask param).
3089    let snap = state.db.reader();
3090    if let Err(e) = snap.ensure_readable() {
3091        return graph_err(e);
3092    }
3093    match snap.neighborhood(&key, depth, etype_refs.as_deref(), dir) {
3094        Some(rs) => json_ok(result_set_json(&rs)),
3095        None => key_not_found(key),
3096    }
3097}
3098
3099/// `POST /find_similar` — vector kNN. First-class read, not `/algo/*`, so a
3100/// role token is allowed and a client `mask` intersects the role (never widens).
3101/// Default `min` is `mushroomdb::FIND_SIMILAR_DEFAULT_MIN` (0.8), as on MCP and
3102/// in the Python binding. It was 0.0 here through 0.6.
3103async fn find_similar(
3104    State(state): State<AppState>,
3105    Extension(identity): Extension<AuthIdentity>,
3106    Json(body): Json<Js>,
3107) -> Response {
3108    let field = match body.get("field").and_then(Js::as_str) {
3109        Some(s) if !s.is_empty() => s.to_string(),
3110        _ => return err_response("missing field"),
3111    };
3112    let vector = match parse_find_similar_vector(body.get("vector")) {
3113        Ok(v) => v,
3114        Err(e) => return err_response(e),
3115    };
3116    let k = match body.get("k") {
3117        None | Some(Js::Null) => 10usize,
3118        Some(v) => match v.as_u64() {
3119            Some(n) => n as usize,
3120            None => return err_response("k must be a non-negative integer"),
3121        },
3122    };
3123    let min = match body.get("min") {
3124        None | Some(Js::Null) => mushroomdb::FIND_SIMILAR_DEFAULT_MIN,
3125        Some(v) => match v.as_f64() {
3126            Some(n) => n,
3127            None => return err_response("min must be a number"),
3128        },
3129    };
3130    let label = match body.get("label").and_then(Js::as_str) {
3131        None | Some("") => None,
3132        Some(s) => Some(s.to_string()),
3133    };
3134    let mask_keys = match parse_mask_keys(body.get("mask")) {
3135        Ok(m) => m,
3136        Err(e) => return err_response(e),
3137    };
3138    let where_pred = match parse_where_body(body.get("where")) {
3139        Ok(p) => p,
3140        Err(detail) => return graph_err(GraphError::QueryError { detail }),
3141    };
3142    let exact = match body.get("exact") {
3143        None | Some(Js::Null) => false,
3144        Some(v) => match v.as_bool() {
3145            Some(b) => b,
3146            None => return err_response("exact must be a boolean"),
3147        },
3148    };
3149    let exact = exact || where_pred.is_some();
3150    let role_name = match identity {
3151        AuthIdentity::Role(r) => Some(r),
3152        AuthIdentity::Full | AuthIdentity::Write => None,
3153        // Unreachable on this route — `Anonymous` is only ever inserted for
3154        // `GET /health` — but treated as the narrowest identity rather than
3155        // the widest, so a future open endpoint cannot inherit full access by
3156        // omission.
3157        AuthIdentity::Anonymous => {
3158            return forbidden("this endpoint requires a token");
3159        }
3160    };
3161    let db = state.db.clone();
3162    match tokio::task::spawn_blocking(move || {
3163        let g = db.read();
3164        let role_mask = match role_name.as_deref() {
3165            Some(role) => Some(g.mask_for_role(role)?),
3166            None => None,
3167        };
3168        let client_mask = mask_keys
3169            .as_ref()
3170            .map(|keys| NodeMask::from_keys(&*g, keys.iter().map(String::as_str)));
3171        let effective = match (role_mask, client_mask) {
3172            (Some(role), Some(client)) => Some(role.intersect(&client)),
3173            (Some(role), None) => Some(role),
3174            (None, Some(client)) => Some(client),
3175            (None, None) => None,
3176        };
3177        g.find_similar_vector_filtered(
3178            &field,
3179            label.as_deref(),
3180            &vector,
3181            k,
3182            min,
3183            effective.as_ref(),
3184            where_pred.as_ref(),
3185            exact,
3186        )
3187    })
3188    .await
3189    {
3190        Ok(Ok(hits)) => json_ok(json!({ "hits": hits })),
3191        Ok(Err(e)) => role_mask_err(e),
3192        Err(_) => err_response("find_similar task panicked"),
3193    }
3194}
3195
3196fn refuse_nonobject_algorithm_config(body: &serde_json::Value) -> Option<Response> {
3197    (!body.is_object()).then(|| err_response("algorithm configuration must be a JSON object"))
3198}
3199
3200/// `POST /algo/pagerank` — run PageRank over the unified topology.
3201///
3202/// # Locking and blocking strategy
3203///
3204/// PageRank is CPU-intensive and synchronous. This handler offloads the work
3205/// to `tokio::task::spawn_blocking` (blocking thread-pool), over a lock-free
3206/// [`ReaderSnapshot`], so it holds no guard and never holds a writer up. The
3207/// `budget_ms` field in [`PageRankConfig`] caps its running time.
3208async fn algo_pagerank(
3209    State(state): State<AppState>,
3210    Extension(identity): Extension<AuthIdentity>,
3211    Json(body): Json<serde_json::Value>,
3212) -> Response {
3213    // v1: deny role tokens — algo endpoints scan the full graph.
3214    if let AuthIdentity::Role(_) = identity {
3215        return forbidden("role-bound token: /algo/* requires a full-access token");
3216    }
3217    if let Some(response) = refuse_nonobject_algorithm_config(&body) {
3218        return response;
3219    }
3220    let config: PageRankConfig = match serde_json::from_value(body) {
3221        Ok(c) => c,
3222        Err(e) => return err_response(format!("invalid pagerank config: {e}")),
3223    };
3224    if let Err(e) = config.validate() {
3225        return err_response(format!("invalid pagerank config: {e}"));
3226    }
3227    let db = state.db.clone();
3228    match tokio::task::spawn_blocking(move || db.reader().pagerank(&config)).await {
3229        Ok(Ok(report)) => json_ok(serde_json::to_value(&report).unwrap_or_else(|_| json!({}))),
3230        Ok(Err(e)) => graph_err(e),
3231        Err(_) => err_response("pagerank task panicked"),
3232    }
3233}
3234
3235/// `POST /algo/wcc` — weakly-connected components over the unified topology.
3236///
3237/// Mirrors `algo_pagerank`: spawn_blocking over a lock-free snapshot,
3238/// `budget_ms` in config caps running time.
3239async fn algo_wcc(
3240    State(state): State<AppState>,
3241    Extension(identity): Extension<AuthIdentity>,
3242    Json(body): Json<serde_json::Value>,
3243) -> Response {
3244    if let AuthIdentity::Role(_) = identity {
3245        return forbidden("role-bound token: /algo/* requires a full-access token");
3246    }
3247    if let Some(response) = refuse_nonobject_algorithm_config(&body) {
3248        return response;
3249    }
3250    let config: WccConfig = match serde_json::from_value(body) {
3251        Ok(c) => c,
3252        Err(e) => return err_response(format!("invalid wcc config: {e}")),
3253    };
3254    let db = state.db.clone();
3255    match tokio::task::spawn_blocking(move || db.reader().connected_components(&config)).await {
3256        Ok(Ok(report)) => json_ok(serde_json::to_value(&report).unwrap_or_else(|_| json!({}))),
3257        Ok(Err(e)) => graph_err(e),
3258        Err(_) => err_response("wcc task panicked"),
3259    }
3260}
3261
3262/// `POST /algo/degree` — degree centrality over the unified topology.
3263///
3264/// Mirrors `algo_pagerank`.
3265async fn algo_degree(
3266    State(state): State<AppState>,
3267    Extension(identity): Extension<AuthIdentity>,
3268    Json(body): Json<serde_json::Value>,
3269) -> Response {
3270    if let AuthIdentity::Role(_) = identity {
3271        return forbidden("role-bound token: /algo/* requires a full-access token");
3272    }
3273    if let Some(response) = refuse_nonobject_algorithm_config(&body) {
3274        return response;
3275    }
3276    let config: DegreeConfig = match serde_json::from_value(body) {
3277        Ok(c) => c,
3278        Err(e) => return err_response(format!("invalid degree config: {e}")),
3279    };
3280    let db = state.db.clone();
3281    match tokio::task::spawn_blocking(move || db.reader().degree_centrality(&config)).await {
3282        Ok(Ok(report)) => json_ok(serde_json::to_value(&report).unwrap_or_else(|_| json!({}))),
3283        Ok(Err(e)) => graph_err(e),
3284        Err(_) => err_response("degree task panicked"),
3285    }
3286}
3287
3288// ── Simple BatchOp-mapped endpoints ──────────────────────────────────────────
3289//
3290// These endpoints map 1:1 to BatchOp variants and route through submit_batch
3291// so concurrent writes share one WAL fsync per drain group, keeping reader
3292// p95 latency low under write bursts.  Auth checks (role tokens → 403) run
3293// before enqueue so RBAC enforcement is unchanged.
3294//
3295// Complex paths (/query Cypher writes, /ingest bulk JSON) stay on db.write()
3296// because they are multi-step operations that cannot be pre-expressed as a
3297// Vec<BatchOp> without redesigning the query executor.
3298
3299/// Parse a JSON object into a `Vec<(String, Value)>` prop list.
3300fn props_from_json_obj(v: &serde_json::Value) -> Result<Vec<(String, Value)>, String> {
3301    let obj = match v.as_object() {
3302        Some(o) => o,
3303        None => return Err("props must be a JSON object".into()),
3304    };
3305    let mut out = Vec::with_capacity(obj.len());
3306    for (k, val) in obj {
3307        if let Some(v) = json_to_value(val.clone()).map_err(|e| format!("props {k:?}: {e}"))? {
3308            out.push((k.clone(), v));
3309        }
3310    }
3311    Ok(out)
3312}
3313
3314/// `POST /nodes` — create or upsert a node via the group-commit queue.
3315///
3316/// Body: `{"label": "Person", "key": "alice", "props": {"age": 30}}`
3317async fn create_node(
3318    State(state): State<AppState>,
3319    Extension(identity): Extension<AuthIdentity>,
3320    Json(body): Json<Js>,
3321) -> Response {
3322    let label = match body.get("label").and_then(Js::as_str) {
3323        Some(s) => s.to_string(),
3324        None => return err_response("missing label"),
3325    };
3326    let key = match body.get("key").and_then(Js::as_str) {
3327        Some(s) => s.to_string(),
3328        None => return err_response("missing key"),
3329    };
3330    let mut props = match body.get("props") {
3331        None | Some(Js::Null) => vec![],
3332        Some(v) => match props_from_json_obj(v) {
3333            Ok(p) => p,
3334            Err(e) => return err_response(e),
3335        },
3336    };
3337    // `namespace` is the namespace the node is created in — the same write-once
3338    // `ns` property, named on the call instead of buried in `props`.
3339    match namespace_arg(body.get("namespace")) {
3340        Ok(None) => {}
3341        Ok(Some(ns)) => {
3342            let mut row: BTreeMap<String, Value> = props.into_iter().collect();
3343            if let Err(e) = stamp_namespace_row(&mut row, &ns) {
3344                return err_response(e);
3345            }
3346            props = row.into_iter().collect();
3347        }
3348        Err(e) => return err_response(e),
3349    }
3350    let db = state.db.clone();
3351    if let AuthIdentity::Role(role_name) = &identity {
3352        let role = role_name.clone();
3353        return match blocking_write(move || {
3354            db.submit_batch_authz(role, vec![BatchOp::InsertNode { label, key, props }])
3355        })
3356        .await
3357        {
3358            Ok((nodes, edges)) => json_ok(json!({"ok": true, "nodes": nodes, "edges": edges})),
3359            Err(resp) => resp,
3360        };
3361    }
3362    match blocking_write(move || db.submit_batch(vec![BatchOp::InsertNode { label, key, props }]))
3363        .await
3364    {
3365        Ok((nodes, edges)) => json_ok(json!({"ok": true, "nodes": nodes, "edges": edges})),
3366        Err(resp) => resp,
3367    }
3368}
3369
3370/// `DELETE /node/{key}` — delete a node via the group-commit queue.
3371async fn delete_node(
3372    State(state): State<AppState>,
3373    Extension(identity): Extension<AuthIdentity>,
3374    Path(key): Path<String>,
3375) -> Response {
3376    let db = state.db.clone();
3377    if let AuthIdentity::Role(role_name) = &identity {
3378        let role = role_name.clone();
3379        return match blocking_write(move || {
3380            db.submit_batch_authz(role, vec![BatchOp::DeleteNode { key }])
3381        })
3382        .await
3383        {
3384            Ok(_) => json_ok(json!({"ok": true})),
3385            Err(resp) => resp,
3386        };
3387    }
3388    match blocking_write(move || db.submit_batch(vec![BatchOp::DeleteNode { key }])).await {
3389        Ok(_) => json_ok(json!({"ok": true})),
3390        Err(resp) => resp,
3391    }
3392}
3393
3394/// `POST /edges` — create an edge via the group-commit queue.
3395///
3396/// Body: `{"type": "KNOWS", "src": "alice", "dst": "bob"}`
3397async fn create_edge(
3398    State(state): State<AppState>,
3399    Extension(identity): Extension<AuthIdentity>,
3400    Json(body): Json<Js>,
3401) -> Response {
3402    let edge_type = match body.get("type").and_then(Js::as_str) {
3403        Some(s) => s.to_string(),
3404        None => return err_response("missing type"),
3405    };
3406    let src = match body.get("src").and_then(Js::as_str) {
3407        Some(s) => s.to_string(),
3408        None => return err_response("missing src"),
3409    };
3410    let dst = match body.get("dst").and_then(Js::as_str) {
3411        Some(s) => s.to_string(),
3412        None => return err_response("missing dst"),
3413    };
3414    let db = state.db.clone();
3415    if let AuthIdentity::Role(role_name) = &identity {
3416        let role = role_name.clone();
3417        return match blocking_write(move || {
3418            db.submit_batch_authz(
3419                role,
3420                vec![BatchOp::InsertEdge {
3421                    edge_type,
3422                    src_key: src,
3423                    dst_key: dst,
3424                }],
3425            )
3426        })
3427        .await
3428        {
3429            Ok(_) => json_ok(json!({"ok": true})),
3430            Err(resp) => resp,
3431        };
3432    }
3433    match blocking_write(move || {
3434        db.submit_batch(vec![BatchOp::InsertEdge {
3435            edge_type,
3436            src_key: src,
3437            dst_key: dst,
3438        }])
3439    })
3440    .await
3441    {
3442        Ok(_) => json_ok(json!({"ok": true})),
3443        Err(resp) => resp,
3444    }
3445}
3446
3447/// `DELETE /edges/{etype}/{src}/{dst}` — delete an edge via the group-commit queue.
3448async fn delete_edge(
3449    State(state): State<AppState>,
3450    Extension(identity): Extension<AuthIdentity>,
3451    Path((etype, src, dst)): Path<(String, String, String)>,
3452) -> Response {
3453    let db = state.db.clone();
3454    if let AuthIdentity::Role(role_name) = &identity {
3455        let role = role_name.clone();
3456        return match blocking_write(move || {
3457            db.submit_batch_authz(
3458                role,
3459                vec![BatchOp::DeleteEdge {
3460                    edge_type: etype,
3461                    src_key: src,
3462                    dst_key: dst,
3463                }],
3464            )
3465        })
3466        .await
3467        {
3468            Ok(_) => json_ok(json!({"ok": true})),
3469            Err(resp) => resp,
3470        };
3471    }
3472    match blocking_write(move || {
3473        db.submit_batch(vec![BatchOp::DeleteEdge {
3474            edge_type: etype,
3475            src_key: src,
3476            dst_key: dst,
3477        }])
3478    })
3479    .await
3480    {
3481        Ok(_) => json_ok(json!({"ok": true})),
3482        Err(resp) => resp,
3483    }
3484}
3485
3486/// `POST /nodes/{key}/rename` — rename a node's key via the group-commit queue.
3487///
3488/// Body: `{"new_key": "alice2"}`
3489/// Returns 404 on KeyNotFound, 409 on DuplicateKey, 200 on success.
3490async fn rename_node(
3491    State(state): State<AppState>,
3492    Extension(identity): Extension<AuthIdentity>,
3493    Path(key): Path<String>,
3494    Json(body): Json<Js>,
3495) -> Response {
3496    if let AuthIdentity::Role(_) = identity {
3497        return forbidden("role-bound token: writes are not permitted");
3498    }
3499    let new_key = match body.get("new_key").and_then(Js::as_str) {
3500        Some(s) => s.to_string(),
3501        None => return err_response("missing new_key"),
3502    };
3503    let db = state.db.clone();
3504    match tokio::task::spawn_blocking(move || {
3505        db.submit_batch(vec![BatchOp::RenameNode {
3506            old_key: key,
3507            new_key,
3508        }])
3509    })
3510    .await
3511    {
3512        Ok(Ok(_)) => json_ok(json!({"ok": true})),
3513        Ok(Err(GraphError::KeyNotFound { key })) => key_not_found(key),
3514        Ok(Err(GraphError::DuplicateKey { key })) => conflict_response(key),
3515        Ok(Err(e)) => graph_err(e),
3516        Err(_) => err_response("write task panicked"),
3517    }
3518}
3519
3520/// `POST /edges/upsert` — insert an edge, auto-creating missing endpoints.
3521///
3522/// Body: `{"edge_type":"KNOWS","src_key":"alice","dst_key":"bob","placeholder_label":"Person"}`
3523/// Returns `{"nodes_created": N, "edge_inserted": bool}`.
3524async fn upsert_edge(
3525    State(state): State<AppState>,
3526    Extension(identity): Extension<AuthIdentity>,
3527    Json(body): Json<Js>,
3528) -> Response {
3529    let edge_type = match body.get("edge_type").and_then(Js::as_str) {
3530        Some(s) => s.to_string(),
3531        None => return err_response("missing edge_type"),
3532    };
3533    let src_key = match body.get("src_key").and_then(Js::as_str) {
3534        Some(s) => s.to_string(),
3535        None => return err_response("missing src_key"),
3536    };
3537    let dst_key = match body.get("dst_key").and_then(Js::as_str) {
3538        Some(s) => s.to_string(),
3539        None => return err_response("missing dst_key"),
3540    };
3541    let placeholder_label = match body.get("placeholder_label").and_then(Js::as_str) {
3542        Some(s) => s.to_string(),
3543        None => return err_response("missing placeholder_label"),
3544    };
3545    let db = state.db.clone();
3546    if let AuthIdentity::Role(role_name) = &identity {
3547        let role = role_name.clone();
3548        return match blocking_write(move || {
3549            db.submit_batch_authz(
3550                role,
3551                vec![BatchOp::InsertEdgeUpsert {
3552                    edge_type,
3553                    src_key,
3554                    dst_key,
3555                    placeholder_label,
3556                }],
3557            )
3558        })
3559        .await
3560        {
3561            Ok((nodes, edges)) => json_ok(json!({
3562                "nodes_created": nodes,
3563                "edge_inserted": edges > 0,
3564            })),
3565            Err(resp) => resp,
3566        };
3567    }
3568    match blocking_write(move || {
3569        db.submit_batch(vec![BatchOp::InsertEdgeUpsert {
3570            edge_type,
3571            src_key,
3572            dst_key,
3573            placeholder_label,
3574        }])
3575    })
3576    .await
3577    {
3578        Ok((nodes, edges)) => json_ok(json!({
3579            "nodes_created": nodes,
3580            "edge_inserted": edges > 0,
3581        })),
3582        Err(resp) => resp,
3583    }
3584}
3585
3586/// `PUT /node/{key}/prop/{field}` — set a property via the group-commit queue.
3587///
3588/// Body: `{"value": 42}`
3589async fn set_node_prop(
3590    State(state): State<AppState>,
3591    Extension(identity): Extension<AuthIdentity>,
3592    Path((key, field)): Path<(String, String)>,
3593    Json(body): Json<Js>,
3594) -> Response {
3595    let parsed = body.get("value").map(|v| json_to_value(v.clone()));
3596    let value = match parsed {
3597        Some(Err(e)) => return err_response(format!("value: {e}")),
3598        Some(Ok(Some(v))) => v,
3599        Some(Ok(None)) | None => {
3600            return err_response(
3601                "request body must be a JSON object with a \"value\" field, \
3602                 e.g. {\"value\": \"SanFrancisco\"} or {\"value\": [\"a\", \"b\"]}",
3603            )
3604        }
3605    };
3606    let db = state.db.clone();
3607    if let AuthIdentity::Role(role_name) = &identity {
3608        let role = role_name.clone();
3609        return match blocking_write(move || {
3610            db.submit_batch_authz(role, vec![BatchOp::SetProp { key, field, value }])
3611        })
3612        .await
3613        {
3614            Ok(_) => json_ok(json!({"ok": true})),
3615            Err(resp) => resp,
3616        };
3617    }
3618    match blocking_write(move || db.submit_batch(vec![BatchOp::SetProp { key, field, value }]))
3619        .await
3620    {
3621        Ok(_) => json_ok(json!({"ok": true})),
3622        Err(resp) => resp,
3623    }
3624}
3625
3626// ── History endpoints ─────────────────────────────────────────────────────────
3627//
3628// These are cold-path diagnostic endpoints. They scan the on-disk WAL and
3629// must NOT extend ReaderSnapshot — they use db.read() directly per the
3630// controller ruling (see module doc and task-2-brief.md).
3631//
3632// Role masking: node visibility is checked under the SAME read guard as the
3633// history call (coherent snapshot). A node outside the role's mask responds
3634// identically to an absent node — no existence oracle.
3635
3636/// `GET /node/{key}/history` — return the WAL change history for `key`.
3637///
3638/// Response: `{ key, history: [{commit, change}], total_commits, horizon }`.
3639/// `horizon` is the oldest commit history still reaches; events before it were
3640/// pruned and are not in `history`.
3641/// Role tokens: if `key` is hidden by the role mask, responds with 404
3642/// (same shape as querying an absent key — no existence oracle).
3643async fn node_history_handler(
3644    State(state): State<AppState>,
3645    Extension(identity): Extension<AuthIdentity>,
3646    Path(key): Path<String>,
3647) -> Response {
3648    if let AuthIdentity::Role(ref role_name) = identity {
3649        let g = state.db.read();
3650        let role_mask = match g.mask_for_role(role_name) {
3651            Ok(m) => m,
3652            Err(e) => return role_mask_err(e),
3653        };
3654        // Hidden keys must respond identically to absent keys (no oracle).
3655        if !role_mask.contains_node(&*g, &key) {
3656            return key_not_found(key);
3657        }
3658        let result = match g.node_history(&key) {
3659            Ok(e) => e,
3660            Err(e) => return graph_err(e),
3661        };
3662        // Filter EdgeAdded/EdgeRemoved entries whose `other` endpoint is hidden.
3663        // A role token must not learn about hidden nodes via edge history events —
3664        // mirrors the same protection in `node_edges` (http.rs ~978-989).
3665        use mushroomdb::HistoryChange;
3666        let visible = mushroomdb::HistoryResult {
3667            total_commits: result.total_commits,
3668            horizon: result.horizon,
3669            items: result
3670                .items
3671                .into_iter()
3672                .filter(|entry| match &entry.change {
3673                    HistoryChange::EdgeAdded { other, .. }
3674                    | HistoryChange::EdgeRemoved { other, .. } => {
3675                        role_mask.contains_node(&*g, other)
3676                    }
3677                    _ => true,
3678                })
3679                .collect(),
3680        };
3681        return json_ok(node_history_json(&key, &visible));
3682    }
3683    // Full identity: no masking. Return 404 for absent keys (consistent with
3684    // GET /node/{key} and the Role branch above).
3685    let g = state.db.read();
3686    if !g.has_node(&key) {
3687        return key_not_found(key);
3688    }
3689    let result = match g.node_history(&key) {
3690        Ok(e) => e,
3691        Err(e) => return graph_err(e),
3692    };
3693    json_ok(node_history_json(&key, &result))
3694}
3695
3696/// `GET /history/edge?a=&b=` — return the edge lifecycle between two nodes.
3697///
3698/// Response: `{ a, b, events: [{edge_type, commit, event, rule}], total_commits, horizon }`.
3699/// `horizon` is the oldest commit history still reaches; events before it were
3700/// pruned and are not in `events`.
3701/// Role tokens: BOTH `a` AND `b` must be visible in the role mask, otherwise
3702/// responds with 404 for the first invisible key (no existence oracle).
3703async fn edge_history_handler(
3704    State(state): State<AppState>,
3705    Extension(identity): Extension<AuthIdentity>,
3706    Query(qs): Query<BTreeMap<String, String>>,
3707) -> Response {
3708    let a = match qs.get("a").filter(|s| !s.is_empty()) {
3709        Some(s) => s.clone(),
3710        None => return err_response("missing query param a"),
3711    };
3712    let b = match qs.get("b").filter(|s| !s.is_empty()) {
3713        Some(s) => s.clone(),
3714        None => return err_response("missing query param b"),
3715    };
3716    if let AuthIdentity::Role(ref role_name) = identity {
3717        let g = state.db.read();
3718        let role_mask = match g.mask_for_role(role_name) {
3719            Ok(m) => m,
3720            Err(e) => return role_mask_err(e),
3721        };
3722        // BOTH endpoints must be visible (no oracle for either).
3723        if !role_mask.contains_node(&*g, &a) {
3724            return key_not_found(a);
3725        }
3726        if !role_mask.contains_node(&*g, &b) {
3727            return key_not_found(b);
3728        }
3729        let result = match g.edge_history(&a, &b) {
3730            Ok(r) => r,
3731            Err(e) => return graph_err(e),
3732        };
3733        return json_ok(edge_history_result_json(&a, &b, &result));
3734    }
3735    // Full identity: no masking.
3736    let g = state.db.read();
3737    let result = match g.edge_history(&a, &b) {
3738        Ok(r) => r,
3739        Err(e) => return graph_err(e),
3740    };
3741    json_ok(edge_history_result_json(&a, &b, &result))
3742}
3743
3744/// `GET /history/was_linked?a=&b=&edge_type=&at_commit=` — point-in-time edge check.
3745///
3746/// Response: `{ a, b, edge_type, at_commit, linked }`.
3747/// Returns 400 (not 500) when `at_commit` is outside the visible horizon.
3748/// Role tokens: BOTH `a` AND `b` must be visible, same-as-absent otherwise.
3749async fn was_linked_handler(
3750    State(state): State<AppState>,
3751    Extension(identity): Extension<AuthIdentity>,
3752    Query(qs): Query<BTreeMap<String, String>>,
3753) -> Response {
3754    let a = match qs.get("a").filter(|s| !s.is_empty()) {
3755        Some(s) => s.clone(),
3756        None => return err_response("missing query param a"),
3757    };
3758    let b = match qs.get("b").filter(|s| !s.is_empty()) {
3759        Some(s) => s.clone(),
3760        None => return err_response("missing query param b"),
3761    };
3762    let edge_type = match qs.get("edge_type").filter(|s| !s.is_empty()) {
3763        Some(s) => s.clone(),
3764        None => return err_response("missing query param edge_type"),
3765    };
3766    // `at_commit` takes a 0-based frame index or an RFC 3339 date. A caller
3767    // asking "were they linked on 2026-06-19" should not have to find the frame
3768    // themselves — and a guessed index is how a plausible wrong answer is
3769    // returned.
3770    let at_commit: u64 = match qs.get("at_commit") {
3771        Some(s) => match s.parse::<u64>() {
3772            Ok(n) => n,
3773            Err(_) => match state.db.read().resolve_date(s) {
3774                Ok(n) => n,
3775                Err(e) => return graph_err(e),
3776            },
3777        },
3778        None => return err_response("missing query param at_commit"),
3779    };
3780
3781    if let AuthIdentity::Role(ref role_name) = identity {
3782        let g = state.db.read();
3783        let role_mask = match g.mask_for_role(role_name) {
3784            Ok(m) => m,
3785            Err(e) => return role_mask_err(e),
3786        };
3787        if !role_mask.contains_node(&*g, &a) {
3788            return key_not_found(a);
3789        }
3790        if !role_mask.contains_node(&*g, &b) {
3791            return key_not_found(b);
3792        }
3793        return match g.was_linked(&a, &b, &edge_type, at_commit) {
3794            Ok(linked) => json_ok(json!({
3795                "a": a, "b": b, "edge_type": edge_type,
3796                "at_commit": at_commit, "linked": linked,
3797            })),
3798            Err(e @ GraphError::CommitOutOfRange { .. }) => (
3799                StatusCode::BAD_REQUEST,
3800                Json(json!({"error": e.to_string()})),
3801            )
3802                .into_response(),
3803            Err(e) => graph_err(e),
3804        };
3805    }
3806
3807    // Full identity: no masking.
3808    let g = state.db.read();
3809    match g.was_linked(&a, &b, &edge_type, at_commit) {
3810        Ok(linked) => json_ok(json!({
3811            "a": a, "b": b, "edge_type": edge_type,
3812            "at_commit": at_commit, "linked": linked,
3813        })),
3814        Err(e @ GraphError::CommitOutOfRange { .. }) => (
3815            StatusCode::BAD_REQUEST,
3816            Json(json!({"error": e.to_string()})),
3817        )
3818            .into_response(),
3819        Err(e) => graph_err(e),
3820    }
3821}
3822
3823/// `DELETE /node/{key}/prop/{field}` — remove a property via the group-commit queue.
3824async fn remove_node_prop(
3825    State(state): State<AppState>,
3826    Extension(identity): Extension<AuthIdentity>,
3827    Path((key, field)): Path<(String, String)>,
3828) -> Response {
3829    let db = state.db.clone();
3830    if let AuthIdentity::Role(role_name) = &identity {
3831        let role = role_name.clone();
3832        return match blocking_write(move || {
3833            db.submit_batch_authz(role, vec![BatchOp::RemoveProp { key, field }])
3834        })
3835        .await
3836        {
3837            Ok(_) => json_ok(json!({"ok": true})),
3838            Err(resp) => resp,
3839        };
3840    }
3841    match blocking_write(move || db.submit_batch(vec![BatchOp::RemoveProp { key, field }])).await {
3842        Ok(_) => json_ok(json!({"ok": true})),
3843        Err(resp) => resp,
3844    }
3845}
3846
3847/// Constant-time byte-string equality, used for secret (token) comparison so
3848/// the match does not short-circuit on the first differing byte and leak the
3849/// token through response-timing. Length is compared up front (token length is
3850/// not treated as a secret); the byte loop is branch-free over equal lengths.
3851fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
3852    if a.len() != b.len() {
3853        return false;
3854    }
3855    let mut diff = 0u8;
3856    for (x, y) in a.iter().zip(b.iter()) {
3857        diff |= x ^ y;
3858    }
3859    diff == 0
3860}
3861
3862/// Maximum `depth` accepted by the neighborhood endpoint. Bounds the read-lock
3863/// hold time of a single BFS request on a dense graph (a hostile
3864/// `depth=4294967295` would otherwise traverse the whole graph under the read
3865/// guard, starving writers).
3866const MAX_NEIGHBORHOOD_DEPTH: u32 = 64;
3867
3868/// Parse `POST /find_similar` `vector` — a non-empty array of numbers.
3869fn parse_find_similar_vector(v: Option<&Js>) -> Result<Vec<f64>, String> {
3870    let Some(arr) = v.and_then(Js::as_array) else {
3871        return Err("missing vector".into());
3872    };
3873    let mut out = Vec::with_capacity(arr.len());
3874    for item in arr {
3875        match item.as_f64() {
3876            Some(n) => out.push(n),
3877            None => return Err("vector must be an array of numbers".into()),
3878        }
3879    }
3880    if out.is_empty() {
3881        return Err("vector must be a non-empty array of numbers".into());
3882    }
3883    Ok(out)
3884}
3885
3886/// Parse an optional JSON `mask` array of node keys.
3887fn parse_mask_keys(v: Option<&Js>) -> Result<Option<Vec<String>>, String> {
3888    match v {
3889        None | Some(Js::Null) => Ok(None),
3890        Some(Js::Array(arr)) => {
3891            let mut keys = Vec::with_capacity(arr.len());
3892            for item in arr {
3893                match item.as_str() {
3894                    Some(s) => keys.push(s.to_string()),
3895                    None => return Err("mask must be an array of strings".into()),
3896                }
3897            }
3898            Ok(Some(keys))
3899        }
3900        Some(_) => Err("mask must be an array of strings".into()),
3901    }
3902}
3903
3904/// Parse optional `where` as a `PropPredicate`. Invalid predicates are QueryError.
3905fn parse_where_body(v: Option<&Js>) -> Result<Option<PropPredicate>, String> {
3906    match v {
3907        None | Some(Js::Null) => Ok(None),
3908        Some(w) => {
3909            let pred: PropPredicate =
3910                serde_json::from_value(w.clone()).map_err(|e| format!("where: {e}"))?;
3911            pred.validate_named("where")?;
3912            Ok(Some(pred))
3913        }
3914    }
3915}
3916
3917/// Parse and bound the neighborhood `depth` query parameter.
3918fn resolve_neighborhood_depth(raw: Option<&str>) -> Result<u32, String> {
3919    match raw {
3920        None => Ok(1),
3921        Some(s) => {
3922            let d: u32 = s
3923                .parse()
3924                .map_err(|_| "depth must be an integer".to_string())?;
3925            if d > MAX_NEIGHBORHOOD_DEPTH {
3926                return Err(format!("depth must be ≤ {MAX_NEIGHBORHOOD_DEPTH}"));
3927            }
3928            Ok(d)
3929        }
3930    }
3931}
3932
3933/// Confine a client-supplied backup `dest` to `root`, closing the arbitrary
3934/// filesystem-write vector (a full-access token could otherwise direct the
3935/// backup writer at `/etc/...`, `/root/.ssh/...`, etc.).
3936///
3937/// Rules: reject empty; reject any `..` segment; relative paths resolve under
3938/// `root`; absolute paths must already fall within `root`. The check is lexical
3939/// (no filesystem access) and safe because `..` is rejected before joining.
3940fn confine_backup_dest(dest: &str, root: &std::path::Path) -> Result<PathBuf, String> {
3941    if dest.is_empty() {
3942        return Err("missing or empty \"dest\" field".into());
3943    }
3944    let dest_path = std::path::Path::new(dest);
3945    if dest_path
3946        .components()
3947        .any(|c| matches!(c, std::path::Component::ParentDir))
3948    {
3949        return Err("backup \"dest\" may not contain \"..\" path segments".into());
3950    }
3951    let joined = if dest_path.is_absolute() {
3952        dest_path.to_path_buf()
3953    } else {
3954        root.join(dest_path)
3955    };
3956    if !joined.starts_with(root) {
3957        return Err(format!(
3958            "backup \"dest\" must be within the backup root ({}); \
3959             set MUSHROOMDB_BACKUP_DIR to change it",
3960            root.display()
3961        ));
3962    }
3963    Ok(joined)
3964}
3965
3966/// Directory backups are confined to: `MUSHROOMDB_BACKUP_DIR` if set, else the
3967/// server's current working directory.
3968fn backup_root() -> PathBuf {
3969    std::env::var_os("MUSHROOMDB_BACKUP_DIR")
3970        .map(PathBuf::from)
3971        .or_else(|| std::env::current_dir().ok())
3972        .unwrap_or_else(|| PathBuf::from("."))
3973}
3974
3975/// `POST /backup` — take a consistent backup of the database to `dest`.
3976///
3977/// The read guard is held for the duration of the file copies, which is the
3978/// correct cross-process synchronisation point: the server is the single
3979/// process touching the files, so holding the read lock excludes concurrent
3980/// in-process writers.  This is the safe alternative to running the
3981/// `mushroomdb backup` CLI against a live-served store.
3982///
3983/// Request body: `{"dest": "backup-dir"}`. `dest` is confined to the backup
3984/// root (`MUSHROOMDB_BACKUP_DIR`, else the server's working directory):
3985/// relative paths resolve under it, absolute paths must fall within it, and
3986/// `..` segments are rejected. This prevents a full-access token from writing
3987/// to arbitrary filesystem paths.
3988///
3989/// Responses:
3990/// - `200 OK` — backup completed; body is a `BackupReport` JSON object.
3991/// - `500 Internal Server Error` — backup succeeded but verification failed;
3992///   body is the `BackupReport` JSON object (examine `files` and `bytes`).
3993/// - `400 Bad Request` — missing or invalid `dest`.
3994/// - `403 Forbidden` — role-bound token; this endpoint requires a full-access token.
3995async fn backup(
3996    State(state): State<AppState>,
3997    Extension(identity): Extension<AuthIdentity>,
3998    Json(body): Json<Js>,
3999) -> Response {
4000    if let AuthIdentity::Role(_) = identity {
4001        return forbidden("role-bound token: /backup requires a full-access token");
4002    }
4003    if let Some(refusal) = refuse_write_tier(&identity) {
4004        return refusal;
4005    }
4006    let root = backup_root();
4007    let dest = match body.get("dest").and_then(Js::as_str) {
4008        Some(s) => match confine_backup_dest(s, &root) {
4009            Ok(p) => p,
4010            Err(e) => return err_response(e),
4011        },
4012        None => return err_response("missing or empty \"dest\" field"),
4013    };
4014    let db = state.db.clone();
4015    let report: BackupReport = match tokio::task::spawn_blocking(move || {
4016        // The read guard is held inside spawn_blocking so file copies happen
4017        // with the write lock excluded.  The guard drops at end of closure.
4018        let g = db.read();
4019        g.backup_to(&dest)
4020    })
4021    .await
4022    {
4023        Ok(Ok(r)) => r,
4024        Ok(Err(e)) => return graph_err(e),
4025        Err(_) => return err_response("backup task panicked"),
4026    };
4027
4028    let body = match serde_json::to_value(BackupReportJson::from(&report)) {
4029        Ok(v) => v,
4030        Err(e) => return err_response(e.to_string()),
4031    };
4032
4033    if report.verified {
4034        json_ok(body)
4035    } else {
4036        (StatusCode::INTERNAL_SERVER_ERROR, Json(body)).into_response()
4037    }
4038}
4039
4040/// JSON-serialisable projection of [`BackupReport`].
4041#[derive(serde::Serialize)]
4042struct BackupReportJson<'a> {
4043    files: &'a [String],
4044    bytes: u64,
4045    verified: bool,
4046}
4047
4048impl<'a> From<&'a BackupReport> for BackupReportJson<'a> {
4049    fn from(r: &'a BackupReport) -> Self {
4050        Self {
4051            files: &r.files,
4052            bytes: r.bytes,
4053            verified: r.verified,
4054        }
4055    }
4056}
4057
4058#[cfg(test)]
4059mod tests {
4060    fn prom_sample(
4061        backup: Option<BackupSample>,
4062        bolt: Option<BoltSample>,
4063        rss: Option<u64>,
4064    ) -> PrometheusSample {
4065        PrometheusSample {
4066            nodes_live: 3,
4067            nodes_tombstoned: 1,
4068            edges_live: 2,
4069            commits_total: 9,
4070            wal_size_bytes: Some(4096),
4071            rss_bytes: rss,
4072            process_start_time_seconds: 1_700_000_000,
4073            slow_queries_total: 4,
4074            transactions_open: 2,
4075            bolt,
4076            backup,
4077        }
4078    }
4079
4080    #[test]
4081    fn prometheus_text_has_help_and_type_for_every_sample() {
4082        let text = format_prometheus(&prom_sample(None, None, Some(1024)));
4083        let samples: Vec<&str> = text.lines().filter(|l| !l.starts_with('#')).collect();
4084        assert!(samples.contains(&"mushroomdb_nodes_live 3"));
4085        assert!(samples.contains(&"mushroomdb_commits_total 9"));
4086        assert!(samples.contains(&"mushroomdb_http_transactions_open 2"));
4087        for line in &samples {
4088            let name = line.split(' ').next().unwrap();
4089            assert!(text.contains(&format!("# HELP {name} ")), "{name} HELP");
4090            assert!(text.contains(&format!("# TYPE {name} ")), "{name} TYPE");
4091        }
4092        assert!(!text.contains("backup"));
4093        assert!(!text.contains("bolt"), "no Bolt series with Bolt off");
4094    }
4095
4096    #[test]
4097    fn prometheus_text_reports_bolt_connections_when_bolt_is_on() {
4098        let text = format_prometheus(&prom_sample(
4099            None,
4100            Some(BoltSample {
4101                open: 3,
4102                accepted_total: 41,
4103            }),
4104            None,
4105        ));
4106        assert!(text.contains("# TYPE mushroomdb_bolt_connections_open gauge\n"));
4107        assert!(text.contains("\nmushroomdb_bolt_connections_open 3\n"));
4108        assert!(text.contains("# TYPE mushroomdb_bolt_connections_accepted_total counter\n"));
4109        assert!(text.contains("\nmushroomdb_bolt_connections_accepted_total 41\n"));
4110    }
4111
4112    #[test]
4113    fn prometheus_text_omits_an_unavailable_gauge_and_adds_backup_series() {
4114        let text = format_prometheus(&prom_sample(
4115            Some(BackupSample {
4116                upload_errors_total: 5,
4117                last_success_ts: 1_700_000_123,
4118                lag_s: 7,
4119                healthy: false,
4120                lease_held: true,
4121            }),
4122            None,
4123            None,
4124        ));
4125        assert!(!text.contains("mushroomdb_rss_bytes"));
4126        assert!(text.contains("# TYPE mushroomdb_backup_upload_errors_total counter\n"));
4127        assert!(text.contains("\nmushroomdb_backup_upload_errors_total 5\n"));
4128        assert!(text.contains("\nmushroomdb_backup_last_success_timestamp_seconds 1700000123\n"));
4129        assert!(text.contains("\nmushroomdb_backup_lag_seconds 7\n"));
4130        assert!(text.contains("\nmushroomdb_backup_healthy 0\n"));
4131        assert!(text.contains("# TYPE mushroomdb_backup_lease_held gauge\n"));
4132        assert!(text.contains("\nmushroomdb_backup_lease_held 1\n"));
4133    }
4134
4135    #[test]
4136    fn accept_negotiation_defaults_to_json() {
4137        let accept = |v: &str| {
4138            let mut h = HeaderMap::new();
4139            h.insert(header::ACCEPT, HeaderValue::from_str(v).unwrap());
4140            wants_prometheus_text(&h)
4141        };
4142        assert!(!wants_prometheus_text(&HeaderMap::new()));
4143        assert!(!accept("*/*"));
4144        assert!(!accept("application/json, text/plain, */*"));
4145        assert!(!accept("text/html,application/xhtml+xml"));
4146        assert!(accept("text/plain"));
4147        assert!(accept("TEXT/PLAIN;version=0.0.4;q=0.5"));
4148        assert!(accept(
4149            "application/openmetrics-text;version=1.0.0,*/*;q=0.1"
4150        ));
4151    }
4152
4153    use super::*;
4154    use crate::json::result_set_json;
4155    use mushroomdb::{DegreeConfig, PageRankConfig, ResultSet, Value, WccConfig};
4156
4157    #[tokio::test]
4158    async fn an_unwritable_directory_is_a_500_with_a_short_client_message() {
4159        let resp = graph_err(GraphError::Unwritable {
4160            path: "/secret/dir/LOCK".into(),
4161            detail: "uid 65532 owner 0 /secret/dir".into(),
4162            read_only_filesystem: false,
4163        });
4164        assert_eq!(resp.status(), StatusCode::INTERNAL_SERVER_ERROR);
4165        let body = axum::body::to_bytes(resp.into_body(), 4096).await.unwrap();
4166        let text = String::from_utf8(body.to_vec()).unwrap();
4167        assert!(
4168            text.contains(GraphError::UNWRITABLE_CLIENT_MESSAGE),
4169            "{text}"
4170        );
4171        assert!(
4172            !text.contains("secret") && !text.contains("65532"),
4173            "{text}"
4174        );
4175    }
4176
4177    #[test]
4178    fn nan_float_cell_serializes_as_null() {
4179        let mut rs = ResultSet::new(vec!["n".into()]);
4180        rs.push_row(vec![Some(Value::Float(f64::NAN))]);
4181        let j = result_set_json(&rs);
4182        assert_eq!(j["rows"][0][0], Js::Null);
4183    }
4184
4185    /// Verify that `POST /algo/pagerank` accepts an empty JSON body `{}` and
4186    /// applies server defaults (regression guard for `#[serde(default)]`).
4187    #[test]
4188    fn pagerank_config_empty_body_uses_defaults() {
4189        let config: PageRankConfig = serde_json::from_str("{}").unwrap();
4190        let default = PageRankConfig::default();
4191        assert_eq!(config.damping, default.damping);
4192        assert_eq!(config.max_iters, default.max_iters);
4193        assert_eq!(config.tol, default.tol);
4194        assert_eq!(config.budget_ms, default.budget_ms);
4195        assert_eq!(config.edge_type, default.edge_type);
4196    }
4197
4198    /// Same guard for `POST /algo/wcc`.
4199    #[test]
4200    fn wcc_config_empty_body_uses_defaults() {
4201        let config: WccConfig = serde_json::from_str("{}").unwrap();
4202        let default = WccConfig::default();
4203        assert_eq!(config.budget_ms, default.budget_ms);
4204        assert_eq!(config.edge_type, default.edge_type);
4205    }
4206
4207    /// Same guard for `POST /algo/degree`.
4208    #[test]
4209    fn degree_config_empty_body_uses_defaults() {
4210        let config: DegreeConfig = serde_json::from_str("{}").unwrap();
4211        let default = DegreeConfig::default();
4212        assert_eq!(config.budget_ms, default.budget_ms);
4213        assert_eq!(config.edge_type, default.edge_type);
4214    }
4215
4216    #[test]
4217    fn backup_dest_rejects_empty() {
4218        assert!(confine_backup_dest("", std::path::Path::new("/srv/backups")).is_err());
4219    }
4220
4221    #[test]
4222    fn constant_time_eq_matches_equality() {
4223        assert!(constant_time_eq(b"secret-token", b"secret-token"));
4224        assert!(constant_time_eq(b"", b""));
4225        assert!(!constant_time_eq(b"secret-token", b"secret-toke")); // shorter
4226        assert!(!constant_time_eq(b"secret-token", b"secret-tokex")); // last byte differs
4227        assert!(!constant_time_eq(b"secret-token", b"Xecret-token")); // first byte differs
4228        assert!(!constant_time_eq(b"", b"x"));
4229    }
4230
4231    #[test]
4232    fn depth_defaults_to_one_when_absent() {
4233        assert_eq!(resolve_neighborhood_depth(None).unwrap(), 1);
4234    }
4235
4236    #[test]
4237    fn depth_within_cap_is_accepted() {
4238        assert_eq!(resolve_neighborhood_depth(Some("10")).unwrap(), 10);
4239        assert_eq!(
4240            resolve_neighborhood_depth(Some(&MAX_NEIGHBORHOOD_DEPTH.to_string())).unwrap(),
4241            MAX_NEIGHBORHOOD_DEPTH
4242        );
4243    }
4244
4245    #[test]
4246    fn depth_over_cap_is_rejected() {
4247        assert!(resolve_neighborhood_depth(Some("65")).is_err());
4248        assert!(resolve_neighborhood_depth(Some("4294967295")).is_err());
4249    }
4250
4251    #[test]
4252    fn depth_non_integer_is_rejected() {
4253        assert!(resolve_neighborhood_depth(Some("abc")).is_err());
4254    }
4255
4256    #[test]
4257    fn backup_dest_rejects_parent_traversal() {
4258        let root = std::path::Path::new("/srv/backups");
4259        assert!(confine_backup_dest("../../etc/cron.d", root).is_err());
4260        assert!(confine_backup_dest("ok/../../../etc", root).is_err());
4261    }
4262
4263    #[test]
4264    fn backup_dest_rejects_absolute_outside_root() {
4265        let root = std::path::Path::new("/srv/backups");
4266        assert!(confine_backup_dest("/etc/cron.d", root).is_err());
4267        assert!(confine_backup_dest("/root/.ssh/authorized_keys", root).is_err());
4268    }
4269
4270    #[test]
4271    fn backup_dest_allows_relative_within_root() {
4272        let root = std::path::Path::new("/srv/backups");
4273        assert_eq!(
4274            confine_backup_dest("nightly", root).unwrap(),
4275            PathBuf::from("/srv/backups/nightly")
4276        );
4277        assert_eq!(
4278            confine_backup_dest("2026/aug", root).unwrap(),
4279            PathBuf::from("/srv/backups/2026/aug")
4280        );
4281    }
4282
4283    #[test]
4284    fn backup_dest_allows_absolute_within_root() {
4285        let root = std::path::Path::new("/srv/backups");
4286        assert_eq!(
4287            confine_backup_dest("/srv/backups/x", root).unwrap(),
4288            PathBuf::from("/srv/backups/x")
4289        );
4290    }
4291}
4292
4293#[cfg(test)]
4294mod read_parity_tests {
4295    //! TX-5 differential: every HTTP read route that moved from the read lock to
4296    //! the lock-free reader answers byte-for-byte what the read-lock path answered.
4297    //!
4298    //! The `legacy_*` functions are the pre-TX-5 handler bodies, kept here as the
4299    //! oracle: each runs under `db.read()` against the `GraphDb` handle, exactly as
4300    //! the route did before it moved. Every case sends the same request through the
4301    //! live router and compares status, content type and body bytes.
4302    //!
4303    //! The matrix runs over the house-rule axes: one batch vs separate commits, a
4304    //! never-closed handle with a delta tail, a snapshotted store with an overlay
4305    //! on top, and a reopened store (WAL replay).
4306
4307    use super::*;
4308    use crate::json::{masked_edges_json, node_edges_json, node_info_json, result_set_json};
4309    use axum::body::{to_bytes, Body};
4310    use mushroomdb::{Predicate, RoleDef, RuleDef};
4311    use tower::ServiceExt;
4312
4313    // ── The oracle: the read-lock bodies as they were before TX-5 ────────────────
4314
4315    fn legacy_client_mask(
4316        db: &mushroomdb::GraphDb<mushroomdb::RealFs>,
4317        mask_param: &str,
4318        stub: bool,
4319    ) -> NodeMask {
4320        let keys = mask_param
4321            .split(',')
4322            .map(str::trim)
4323            .filter(|s| !s.is_empty());
4324        let m = NodeMask::from_keys(db, keys);
4325        if stub {
4326            m.with_mode(MaskMode::Stub)
4327        } else {
4328            m
4329        }
4330    }
4331
4332    fn legacy_node_info(db: &SharedDb, key: &str, mask: &str, stub: bool) -> Response {
4333        let g = db.read();
4334        if !mask.is_empty() {
4335            let mask = legacy_client_mask(&g, mask, stub);
4336            return match g.node_info_masked(key, &mask) {
4337                Some(mushroomdb::MaskedNodeResult::Visible(info)) => json_ok(node_info_json(&info)),
4338                Some(mushroomdb::MaskedNodeResult::Restricted) => {
4339                    json_ok(crate::json::stub_node_json(key))
4340                }
4341                None => key_not_found(key.to_string()),
4342            };
4343        }
4344        match g.node_info(key) {
4345            Some(info) => json_ok(node_info_json(&info)),
4346            None => key_not_found(key.to_string()),
4347        }
4348    }
4349
4350    fn legacy_node_edges(db: &SharedDb, key: &str, mask: &str, stub: bool) -> Response {
4351        let g = db.read();
4352        if !mask.is_empty() {
4353            let mask = legacy_client_mask(&g, mask, stub);
4354            return match g.node_edges_masked(key, &mask) {
4355                Ok(edges) => json_ok(masked_edges_json(&edges)),
4356                Err(GraphError::KeyNotFound { key }) => key_not_found(key),
4357                Err(e) => graph_err(e),
4358            };
4359        }
4360        match g.node_edges(key) {
4361            Ok(edges) => json_ok(node_edges_json(&edges)),
4362            Err(GraphError::KeyNotFound { key }) => key_not_found(key),
4363            Err(e) => graph_err(e),
4364        }
4365    }
4366
4367    #[allow(clippy::too_many_arguments)]
4368    fn legacy_neighborhood(
4369        db: &SharedDb,
4370        key: &str,
4371        depth: u32,
4372        edge_types: Option<&[&str]>,
4373        dir: Dir,
4374        mask: &str,
4375        stub: bool,
4376    ) -> Response {
4377        let g = db.read();
4378        if !mask.is_empty() {
4379            let mask = legacy_client_mask(&g, mask, stub);
4380            return match g.neighborhood_masked(key, depth, edge_types, dir, &mask) {
4381                Some(rs) => json_ok(result_set_json(&rs)),
4382                None => key_not_found(key.to_string()),
4383            };
4384        }
4385        match g.node_ref(key) {
4386            Some(n) => json_ok(result_set_json(&n.neighborhood(depth, edge_types, dir))),
4387            None => key_not_found(key.to_string()),
4388        }
4389    }
4390
4391    /// The full-token, live `/query` read paths (unmasked, and client mask and/or
4392    /// namespace), after the request has been parsed.
4393    fn legacy_query(
4394        db: &SharedDb,
4395        cypher: &str,
4396        mask_keys: Option<&[String]>,
4397        namespace: Option<&str>,
4398        stub: bool,
4399        format: &str,
4400    ) -> Response {
4401        let params = BTreeMap::new();
4402        if mask_keys.is_some() || namespace.is_some() {
4403            let db = db.read();
4404            let m = match (mask_keys, namespace) {
4405                (Some(keys), Some(ns)) => {
4406                    NodeMask::from_keys(&*db, keys.iter().map(String::as_str))
4407                        .intersect(&db.mask_for_namespace(ns))
4408                }
4409                (Some(keys), None) => NodeMask::from_keys(&*db, keys.iter().map(String::as_str)),
4410                (None, Some(ns)) => db.mask_for_namespace(ns),
4411                (None, None) => unreachable!(),
4412            };
4413            let mask = if stub { m.with_mode(MaskMode::Stub) } else { m };
4414            return match db.query_masked(cypher, &params, &mask) {
4415                Ok(rs) => format_query_result(rs, format),
4416                Err(GraphError::MaskedReadOnly) => (
4417                    StatusCode::BAD_REQUEST,
4418                    Json(json!({"error": "masked queries are read-only"})),
4419                )
4420                    .into_response(),
4421                Err(e) => graph_err(e),
4422            };
4423        }
4424        // The route classifies the statement before it picks a lock; a statement
4425        // that does not lex is refused there, on either path.
4426        match is_write_query(cypher) {
4427            Ok(false) => {}
4428            Ok(true) => unreachable!("writes are not read routes"),
4429            Err(e) => return err_response(e),
4430        }
4431        match db.read().query(cypher, &params) {
4432            Ok(rs) => format_query_result(rs, format),
4433            Err(e) => graph_err(e),
4434        }
4435    }
4436
4437    /// The role-token read of `/query`, resolved the read-lock way: the role's mask
4438    /// from the live handle, intersected as the route intersects it, and the query
4439    /// run by `GraphDb::query_masked`. The route itself has been on the reader since
4440    /// 0.6; this pins that its answer, `UNION` included, is the write handle's.
4441    fn legacy_role_query(
4442        db: &SharedDb,
4443        role: &str,
4444        cypher: &str,
4445        mask_keys: Option<&[String]>,
4446        namespace: Option<&str>,
4447    ) -> Response {
4448        // The role branch's own checks, which run before any lock on either path.
4449        match is_write_query(cypher) {
4450            Ok(false) => {}
4451            Ok(true) => unreachable!("writes are not read routes"),
4452            Err(e) => return err_response(e),
4453        }
4454        let g = db.read();
4455        let role_mask = match g.mask_for_role(role) {
4456            Ok(m) => m,
4457            Err(e) => return role_mask_err(e),
4458        };
4459        let mask = match mask_keys {
4460            Some(keys) => {
4461                role_mask.intersect(&NodeMask::from_keys(&*g, keys.iter().map(String::as_str)))
4462            }
4463            None => role_mask,
4464        };
4465        let mask = match namespace {
4466            Some(ns) => mask.intersect(&g.mask_for_namespace(ns)),
4467            None => mask,
4468        };
4469        match g.query_masked(cypher, &BTreeMap::new(), &mask) {
4470            Ok(rs) => format_query_result(rs, "json"),
4471            Err(GraphError::MaskedReadOnly) => (
4472                StatusCode::BAD_REQUEST,
4473                Json(json!({"error": "masked queries are read-only"})),
4474            )
4475                .into_response(),
4476            Err(e) => graph_err(e),
4477        }
4478    }
4479
4480    // ── Plumbing ──────────────────────────────────────────────────────────────────
4481
4482    #[derive(Debug, PartialEq)]
4483    struct Answer {
4484        status: StatusCode,
4485        content_type: Option<String>,
4486        body: Vec<u8>,
4487    }
4488
4489    async fn answer(res: Response) -> Answer {
4490        let status = res.status();
4491        let content_type = res
4492            .headers()
4493            .get(header::CONTENT_TYPE)
4494            .and_then(|v| v.to_str().ok())
4495            .map(str::to_string);
4496        let body = to_bytes(res.into_body(), usize::MAX)
4497            .await
4498            .unwrap()
4499            .to_vec();
4500        Answer {
4501            status,
4502            content_type,
4503            body,
4504        }
4505    }
4506
4507    async fn routed(app: &Router, req: axum::http::Request<Body>) -> Answer {
4508        answer(app.clone().oneshot(req).await.unwrap()).await
4509    }
4510
4511    fn get(uri: &str, token: Option<&str>) -> axum::http::Request<Body> {
4512        let mut b = axum::http::Request::builder().method("GET").uri(uri);
4513        if let Some(t) = token {
4514            b = b.header(header::AUTHORIZATION, format!("Bearer {t}"));
4515        }
4516        b.body(Body::empty()).unwrap()
4517    }
4518
4519    fn post(uri: &str, body: Js, token: Option<&str>) -> axum::http::Request<Body> {
4520        let mut b = axum::http::Request::builder()
4521            .method("POST")
4522            .uri(uri)
4523            .header(header::CONTENT_TYPE, "application/json");
4524        if let Some(t) = token {
4525            b = b.header(header::AUTHORIZATION, format!("Bearer {t}"));
4526        }
4527        b.body(Body::from(body.to_string())).unwrap()
4528    }
4529
4530    fn tmp(name: &str) -> PathBuf {
4531        let nanos = std::time::SystemTime::now()
4532            .duration_since(std::time::UNIX_EPOCH)
4533            .unwrap()
4534            .as_nanos();
4535        let d =
4536            std::env::temp_dir().join(format!("tx5-parity-{name}-{}-{nanos}", std::process::id()));
4537        let _ = std::fs::remove_dir_all(&d);
4538        d
4539    }
4540
4541    const FULL: &str = "admin";
4542    const ROLE: &str = "role-tok";
4543
4544    /// How a fixture reaches its final state.
4545    #[derive(Clone, Copy, Debug)]
4546    enum Build {
4547        /// Every write its own commit, on a handle that never closed: a long
4548        /// delta tail over the open-time fold.
4549        SeparateCommits,
4550        /// The same graph written as one batch.
4551        OneBatch,
4552        /// Snapshotted part-way, then more writes: an mmap base with an overlay
4553        /// and a tail on top.
4554        Snapshotted,
4555        /// Written, closed, and reopened: the state comes from WAL replay.
4556        Reopened,
4557        /// Equality indexes declared and folded, then the second half on top:
4558        /// indexed lookups answered from the fold's index plus the tail.
4559        Indexed,
4560    }
4561
4562    fn rule() -> RuleDef {
4563        RuleDef {
4564            name: "works_at".into(),
4565            src_label: "Person".into(),
4566            dst_label: "Org".into(),
4567            predicate: Predicate::KeyMatch {
4568                field: "org_id".into(),
4569            },
4570            edge_type: "WORKS_AT".into(),
4571            weight_prop: None,
4572            max_edges: None,
4573            approximate: false,
4574            via_label: None,
4575            via_edge: None,
4576            via_dir: None,
4577            namespace: None,
4578        }
4579    }
4580
4581    /// A weighted rule: its `score` is written on each edge by the rule engine,
4582    /// with no WAL record, and refreshed in place when an endpoint changes.
4583    fn weighted_rule() -> RuleDef {
4584        RuleDef {
4585            name: "skill_fit".into(),
4586            src_label: "Person".into(),
4587            dst_label: "Org".into(),
4588            predicate: Predicate::Overlap {
4589                field: "skills".into(),
4590                min: 0.3,
4591            },
4592            edge_type: "FITS".into(),
4593            weight_prop: Some("score".into()),
4594            ..rule()
4595        }
4596    }
4597
4598    fn skills(list: &[&str]) -> Value {
4599        Value::List(list.iter().map(|s| Value::Str(s.to_string())).collect())
4600    }
4601
4602    fn person(key: &str, org: &str, ns: &str) -> BatchOp {
4603        let mine: &[&str] = match key.len() % 3 {
4604            0 => &["rust", "sql"],
4605            1 => &["rust", "go", "sql"],
4606            _ => &["go"],
4607        };
4608        BatchOp::InsertNode {
4609            label: "Person".into(),
4610            key: key.into(),
4611            props: vec![
4612                ("name".into(), Value::Str(key.to_uppercase())),
4613                ("org_id".into(), Value::Str(org.into())),
4614                ("age".into(), Value::Int(key.len() as i64 * 7)),
4615                ("ns".into(), Value::Str(ns.into())),
4616                ("skills".into(), skills(mine)),
4617            ],
4618        }
4619    }
4620
4621    fn edge(t: &str, s: &str, d: &str) -> BatchOp {
4622        BatchOp::InsertEdge {
4623            edge_type: t.into(),
4624            src_key: s.into(),
4625            dst_key: d.into(),
4626        }
4627    }
4628
4629    /// The graph every fixture holds: two orgs, people in two namespaces, a
4630    /// rule-derived `WORKS_AT` per person, asserted `KNOWS` edges (each inside one
4631    /// namespace, as the store requires) including a self-loop and a 2-cycle, a
4632    /// property removed, and a node deleted.
4633    fn first_half() -> Vec<BatchOp> {
4634        vec![
4635            BatchOp::InsertNode {
4636                label: "Org".into(),
4637                key: "o1".into(),
4638                props: vec![
4639                    ("name".into(), Value::Str("Acme".into())),
4640                    ("ns".into(), Value::Str("tenant-a".into())),
4641                    ("skills".into(), skills(&["rust", "sql", "go"])),
4642                ],
4643            },
4644            BatchOp::InsertNode {
4645                label: "Org".into(),
4646                key: "o2".into(),
4647                props: vec![
4648                    ("ns".into(), Value::Str("tenant-b".into())),
4649                    ("skills".into(), skills(&["go", "sql"])),
4650                ],
4651            },
4652            person("alice", "o1", "tenant-a"),
4653            person("bob", "o1", "tenant-a"),
4654            person("carol", "o2", "tenant-b"),
4655        ]
4656    }
4657
4658    fn second_half() -> Vec<BatchOp> {
4659        vec![
4660            person("dave", "o2", "tenant-b"),
4661            person("erin", "o1", "tenant-a"),
4662            edge("KNOWS", "alice", "bob"),
4663            edge("KNOWS", "erin", "bob"),
4664            edge("KNOWS", "bob", "alice"),
4665            edge("KNOWS", "carol", "dave"),
4666            edge("KNOWS", "dave", "dave"),
4667            edge("LIKES", "erin", "alice"),
4668            edge("LIKES", "alice", "o1"),
4669            edge("LIKES", "dave", "o2"),
4670            BatchOp::SetProp {
4671                key: "bob".into(),
4672                field: "age".into(),
4673                value: Value::Int(41),
4674            },
4675            // Moves bob's `FITS` score in place: no edge is derived or retracted.
4676            BatchOp::SetProp {
4677                key: "bob".into(),
4678                field: "skills".into(),
4679                value: skills(&["rust", "sql", "go"]),
4680            },
4681            BatchOp::RemoveProp {
4682                key: "carol".into(),
4683                field: "age".into(),
4684            },
4685            person("zed", "o1", "tenant-a"),
4686            edge("KNOWS", "zed", "alice"),
4687            BatchOp::DeleteNode { key: "zed".into() },
4688            // Moves an indexed value: the fold's index still says CAROL.
4689            BatchOp::SetProp {
4690                key: "carol".into(),
4691                field: "name".into(),
4692                value: Value::Str("CAROLINE".into()),
4693            },
4694        ]
4695    }
4696
4697    fn write_ops(db: &SharedDb, ops: Vec<BatchOp>, one_batch: bool) {
4698        if one_batch {
4699            db.submit_batch(ops).unwrap();
4700        } else {
4701            for op in ops {
4702                db.submit_batch(vec![op]).unwrap();
4703            }
4704        }
4705    }
4706
4707    fn roles() -> mushroomdb::schema::Schema {
4708        mushroomdb::schema::Schema {
4709            roles: vec![RoleDef {
4710                name: "analyst".into(),
4711                labels: vec!["Person".into()],
4712                keys: vec!["o1".into()],
4713                visible_where: None,
4714                namespaces: None,
4715                write: None,
4716            }],
4717            ..Default::default()
4718        }
4719    }
4720
4721    fn fixture(name: &str, build: Build) -> (Router, SharedDb) {
4722        let dir = tmp(name);
4723        let db = SharedDb::open(&dir).unwrap();
4724        db.write().apply_schema(&roles()).unwrap();
4725        db.write().create_rule(rule()).unwrap();
4726        db.write().create_rule(weighted_rule()).unwrap();
4727        match build {
4728            Build::SeparateCommits => {
4729                write_ops(&db, first_half(), false);
4730                write_ops(&db, second_half(), false);
4731            }
4732            Build::OneBatch => {
4733                let mut ops = first_half();
4734                ops.extend(second_half());
4735                write_ops(&db, ops, true);
4736            }
4737            Build::Snapshotted => {
4738                write_ops(&db, first_half(), false);
4739                db.write().snapshot().unwrap();
4740                write_ops(&db, second_half(), false);
4741            }
4742            Build::Reopened => {
4743                write_ops(&db, first_half(), false);
4744                write_ops(&db, second_half(), false);
4745            }
4746            Build::Indexed => {
4747                db.write().enable_index("Person", "name").unwrap();
4748                db.write().enable_index("Org", "name").unwrap();
4749                write_ops(&db, first_half(), false);
4750                // Enough commits to fold, so the indexes and the first half are
4751                // in the fold and the second half is the tail.
4752                for i in 0..mushroomdb::FOLD_EVERY_K {
4753                    db.submit_batch(vec![BatchOp::InsertNode {
4754                        label: "Pad".into(),
4755                        key: format!("pad{i}"),
4756                        props: vec![],
4757                    }])
4758                    .unwrap();
4759                }
4760                write_ops(&db, second_half(), false);
4761                assert!(db.reader().queries_use_index(), "{name}: index unused");
4762            }
4763        }
4764        let db = match build {
4765            Build::Reopened => {
4766                drop(db);
4767                SharedDb::open(&dir).unwrap()
4768            }
4769            _ => db,
4770        };
4771        let app = router_with_role_tokens(
4772            db.clone(),
4773            Some(FULL.into()),
4774            HashMap::from([(ROLE.to_string(), "analyst".to_string())]),
4775        );
4776        (app, db)
4777    }
4778
4779    const KEYS: &[&str] = &["alice", "bob", "carol", "dave", "o1", "o2", "zed", "nobody"];
4780    const MASKS: &[(&str, bool)] = &[
4781        ("", false),
4782        ("alice,carol,o1", false),
4783        ("alice,carol,o1", true),
4784        ("dave", true),
4785        ("nobody", false),
4786    ];
4787
4788    async fn node_routes_match(app: &Router, db: &SharedDb, label: &str) -> usize {
4789        let mut cases = 0;
4790        for key in KEYS {
4791            for (mask, stub) in MASKS {
4792                let qs = if mask.is_empty() {
4793                    String::new()
4794                } else {
4795                    format!("?mask={mask}&stub_hidden={stub}")
4796                };
4797                let got = routed(app, get(&format!("/node/{key}{qs}"), Some(FULL))).await;
4798                let want = answer(legacy_node_info(db, key, mask, *stub)).await;
4799                assert_eq!(got, want, "{label}: GET /node/{key}{qs}");
4800
4801                let got = routed(app, get(&format!("/node/{key}/edges{qs}"), Some(FULL))).await;
4802                let want = answer(legacy_node_edges(db, key, mask, *stub)).await;
4803                assert_eq!(got, want, "{label}: GET /node/{key}/edges{qs}");
4804                cases += 2;
4805
4806                for (depth, dir, dir_name, types) in [
4807                    (1, Dir::Both, "both", None),
4808                    (2, Dir::Out, "out", None),
4809                    (3, Dir::In, "in", Some("KNOWS")),
4810                    (2, Dir::Both, "both", Some("WORKS_AT,LIKES")),
4811                ] {
4812                    let mut uri = format!("/node/{key}/neighborhood?depth={depth}&dir={dir_name}");
4813                    if let Some(t) = types {
4814                        uri.push_str(&format!("&edge_types={t}"));
4815                    }
4816                    if !mask.is_empty() {
4817                        uri.push_str(&format!("&mask={mask}&stub_hidden={stub}"));
4818                    }
4819                    let names: Option<Vec<&str>> = types.map(|t| t.split(',').collect());
4820                    let got = routed(app, get(&uri, Some(FULL))).await;
4821                    let want = answer(legacy_neighborhood(
4822                        db,
4823                        key,
4824                        depth,
4825                        names.as_deref(),
4826                        dir,
4827                        mask,
4828                        *stub,
4829                    ))
4830                    .await;
4831                    assert_eq!(got, want, "{label}: GET {uri}");
4832                    cases += 1;
4833                }
4834            }
4835        }
4836        cases
4837    }
4838
4839    const CYPHERS: &[&str] = &[
4840        "MATCH (n:Person {name: 'BOB'}) RETURN n.key, n.age",
4841        "MATCH (n:Person) WHERE n.name = 'CAROLINE' RETURN n.key",
4842        "MATCH (n:Person {name: 'CAROL'}) RETURN n.key",
4843        "MATCH (n:Person {name: 'ZED'}) RETURN n.key",
4844        "MATCH (n:Person {name: 'ERIN'}) RETURN n.key",
4845        "MATCH (o:Org {name: 'Acme'})<-[:WORKS_AT]-(p) RETURN p.key",
4846        "MATCH (n:Person) RETURN n.name ORDER BY n.name",
4847        "MATCH (n:Person)-[:KNOWS]->(m) RETURN n.name, m.name ORDER BY n.name, m.name",
4848        "MATCH (n:Person)-[:WORKS_AT]->(o:Org) RETURN n.name, o.name",
4849        "MATCH (n:Person)-[r:FITS]->(o:Org) RETURN n.name, o.name, r.score ORDER BY n.name, o.name",
4850        "MATCH (n:Person) WHERE n.age > 30 RETURN n.name, n.age",
4851        "MATCH (n:Person) RETURN count(n) AS c",
4852        "MATCH (n:Person) RETURN n.ns AS ns UNION MATCH (o:Org) RETURN o.ns AS ns",
4853        "MATCH (n:Person) RETURN n.org_id AS x UNION ALL MATCH (n:Person) RETURN n.org_id AS x",
4854        "MATCH (n:Person) RETURN n.name UNION MATCH (o:Org) RETURN o.key",
4855        "MATCH (n:Person RETURN n",
4856        "MATCH (n:Person) RETURN",
4857        "MATCH (n:Person) RETURN n.name UNION",
4858        "MATCH (n) RETURN nosuchfn(n.name)",
4859        "MATCH (n:Person) RETURN m.name",
4860        "MATCH (n) RETURN \u{1}",
4861        "MATCH (n:Person {key: 'zed'}) RETURN n",
4862        "CREATE (x:Person {key: 'mallory'})",
4863    ];
4864
4865    async fn query_routes_match(app: &Router, db: &SharedDb, label: &str) -> usize {
4866        let mut cases = 0;
4867        let masks: [Option<Vec<String>>; 3] = [
4868            None,
4869            Some(vec!["alice".into(), "carol".into(), "o2".into()]),
4870            Some(vec!["nobody".into()]),
4871        ];
4872        for cypher in CYPHERS {
4873            for mask in &masks {
4874                for namespace in [None, Some("tenant-a"), Some("no-such-ns")] {
4875                    for stub in [false, true] {
4876                        // A write without a mask or namespace is not a read route.
4877                        if cypher.starts_with("CREATE") && mask.is_none() && namespace.is_none() {
4878                            continue;
4879                        }
4880                        if stub && mask.is_none() && namespace.is_none() {
4881                            continue;
4882                        }
4883                        for format in ["json", ""] {
4884                            let mut body = json!({ "cypher": cypher });
4885                            if let Some(m) = mask {
4886                                body["mask"] = json!(m);
4887                            }
4888                            if let Some(ns) = namespace {
4889                                body["namespace"] = json!(ns);
4890                            }
4891                            if stub {
4892                                body["stub_hidden"] = json!(true);
4893                            }
4894                            let uri = if format.is_empty() {
4895                                "/query".to_string()
4896                            } else {
4897                                format!("/query?format={format}")
4898                            };
4899                            let got = routed(app, post(&uri, body.clone(), Some(FULL))).await;
4900                            let want = answer(legacy_query(
4901                                db,
4902                                cypher,
4903                                mask.as_deref(),
4904                                namespace,
4905                                stub,
4906                                format,
4907                            ))
4908                            .await;
4909                            assert_eq!(got, want, "{label}: POST {uri} {body}");
4910                            cases += 1;
4911                        }
4912                    }
4913                }
4914            }
4915        }
4916        cases
4917    }
4918
4919    async fn role_queries_match(app: &Router, db: &SharedDb, label: &str) -> usize {
4920        let mut cases = 0;
4921        let masks: [Option<Vec<String>>; 2] = [None, Some(vec!["alice".into(), "o1".into()])];
4922        for cypher in CYPHERS.iter().filter(|c| !c.starts_with("CREATE")) {
4923            for mask in &masks {
4924                for namespace in [None, Some("tenant-a")] {
4925                    let mut body = json!({ "cypher": cypher });
4926                    if let Some(m) = mask {
4927                        body["mask"] = json!(m);
4928                    }
4929                    if let Some(ns) = namespace {
4930                        body["namespace"] = json!(ns);
4931                    }
4932                    let got =
4933                        routed(app, post("/query?format=json", body.clone(), Some(ROLE))).await;
4934                    let want = answer(legacy_role_query(
4935                        db,
4936                        "analyst",
4937                        cypher,
4938                        mask.as_deref(),
4939                        namespace,
4940                    ))
4941                    .await;
4942                    assert_eq!(got, want, "{label}: role POST /query {body}");
4943                    cases += 1;
4944                }
4945            }
4946        }
4947        cases
4948    }
4949
4950    async fn all_routes_match(build: Build, name: &str) {
4951        let (app, db) = fixture(name, build);
4952        // The fixture must exercise what it claims to: a tail the reader applies.
4953        if !matches!(build, Build::Reopened) {
4954            assert!(
4955                !db.read().reader().deltas.is_empty(),
4956                "{name}: no delta tail"
4957            );
4958        }
4959        let label = format!("{build:?}");
4960        let n = node_routes_match(&app, &db, &label).await
4961            + query_routes_match(&app, &db, &label).await
4962            + role_queries_match(&app, &db, &label).await
4963            + counter_and_algo_routes_match(&app, &db, &label).await;
4964        assert!(n > 500, "{label}: only {n} cases ran");
4965    }
4966
4967    // ── /health, /metrics and /algo/* (moved in fix round 1) ──────────────────
4968
4969    fn legacy_health(db: &SharedDb) -> Response {
4970        let (nodes, edges) = {
4971            let g = db.read();
4972            let s = g.stats();
4973            (s.nodes_live, s.edges)
4974        };
4975        json_ok(json!({
4976            "ok": true,
4977            "nodes": nodes,
4978            "edges": edges,
4979            "addr": default_advertise_addr().to_string(),
4980        }))
4981    }
4982
4983    /// `/metrics` before TX-5, minus `rss_bytes` and `uptime_s`, which move
4984    /// between any two calls.
4985    fn legacy_metrics(db: &SharedDb) -> Js {
4986        let g = db.read();
4987        let s = g.stats();
4988        let slow = g.slow_query_snapshot();
4989        json!({
4990            "nodes_live": s.nodes_live,
4991            "nodes_tombstoned": s.nodes_tombstoned,
4992            "edges": s.edges,
4993            "commit_seq": g.commit_seq(),
4994            "wal_size_bytes": g.wal_size_bytes().ok(),
4995            "slow_queries": {
4996                "threshold_ms": slow.threshold_ms,
4997                "count": slow.count,
4998                "last": slow.last.iter().map(|e| json!({
4999                    "ms": e.ms, "query": e.query, "at_commit": e.at_commit,
5000                })).collect::<Vec<_>>(),
5001            },
5002        })
5003    }
5004
5005    fn legacy_algo(db: &SharedDb, route: &str, body: &Js) -> Response {
5006        let g = db.read();
5007        let v = match route {
5008            "pagerank" => {
5009                serde_json::to_value(g.pagerank(&serde_json::from_value(body.clone()).unwrap()))
5010            }
5011            "wcc" => serde_json::to_value(
5012                g.connected_components(&serde_json::from_value(body.clone()).unwrap()),
5013            ),
5014            _ => serde_json::to_value(
5015                g.degree_centrality(&serde_json::from_value(body.clone()).unwrap()),
5016            ),
5017        };
5018        json_ok(v.unwrap_or_else(|_| json!({})))
5019    }
5020
5021    async fn counter_and_algo_routes_match(app: &Router, db: &SharedDb, label: &str) -> usize {
5022        let mut cases = 0;
5023        let got = routed(app, get("/health", Some(FULL))).await;
5024        assert_eq!(got, answer(legacy_health(db)).await, "{label}: /health");
5025        let got = routed(app, get("/health", None)).await;
5026        assert_eq!(got.body, br#"{"ok":true}"#, "{label}: anonymous /health");
5027
5028        let got = routed(app, get("/metrics", Some(FULL))).await;
5029        assert_eq!(got.status, StatusCode::OK);
5030        let mut v: Js = serde_json::from_slice(&got.body).unwrap();
5031        let obj = v.as_object_mut().unwrap();
5032        assert!(obj.remove("rss_bytes").is_some() && obj.remove("uptime_s").is_some());
5033        assert_eq!(v, legacy_metrics(db), "{label}: /metrics");
5034        cases += 3;
5035
5036        for route in ["pagerank", "wcc", "degree"] {
5037            for body in [
5038                json!({}),
5039                json!({"edge_type": "KNOWS"}),
5040                json!({"edge_type": "FITS"}),
5041            ] {
5042                let got = routed(
5043                    app,
5044                    post(&format!("/algo/{route}"), body.clone(), Some(FULL)),
5045                )
5046                .await;
5047                let want = answer(legacy_algo(db, route, &body)).await;
5048                assert_eq!(got, want, "{label}: /algo/{route} {body}");
5049                cases += 1;
5050            }
5051        }
5052        cases
5053    }
5054
5055    #[tokio::test]
5056    async fn reader_routes_match_the_read_lock_separate_commits() {
5057        all_routes_match(Build::SeparateCommits, "separate").await;
5058    }
5059
5060    #[tokio::test]
5061    async fn reader_routes_match_the_read_lock_one_batch() {
5062        all_routes_match(Build::OneBatch, "batch").await;
5063    }
5064
5065    #[tokio::test]
5066    async fn reader_routes_match_the_read_lock_snapshotted() {
5067        all_routes_match(Build::Snapshotted, "snapshotted").await;
5068    }
5069
5070    #[tokio::test]
5071    async fn reader_routes_match_the_read_lock_reopened() {
5072        all_routes_match(Build::Reopened, "reopened").await;
5073    }
5074
5075    /// The cases are only a differential if the two sides can disagree: `UNION` on
5076    /// the role path, which the reader refused before TX-5, now answers rows.
5077    #[tokio::test]
5078    async fn union_answers_on_the_reader_for_both_identities() {
5079        let (app, _db) = fixture("union", Build::SeparateCommits);
5080        let cypher = "MATCH (n:Person) RETURN n.ns AS ns UNION MATCH (o:Org) RETURN o.ns AS ns";
5081        for token in [FULL, ROLE] {
5082            let got = routed(
5083                &app,
5084                post(
5085                    "/query?format=json",
5086                    json!({ "cypher": cypher }),
5087                    Some(token),
5088                ),
5089            )
5090            .await;
5091            assert_eq!(got.status, StatusCode::OK, "{token}: {:?}", got.body);
5092            let v: Js = serde_json::from_slice(&got.body).unwrap();
5093            assert!(
5094                !v["rows"].as_array().unwrap().is_empty(),
5095                "{token}: UNION returned no rows: {v}"
5096            );
5097        }
5098    }
5099
5100    /// A store whose indexes are in the fold, with a tail that moves, deletes
5101    /// and adds indexed values: the reader serves the lookups from the fold's
5102    /// index layered with the tail and answers what the read lock answered.
5103    #[tokio::test]
5104    async fn reader_routes_match_the_read_lock_indexed() {
5105        all_routes_match(Build::Indexed, "indexed").await;
5106    }
5107
5108    /// An unmasked reader query still lands in the `/metrics` slow-query ring.
5109    #[tokio::test]
5110    async fn a_slow_reader_query_is_still_recorded() {
5111        let (app, db) = fixture("slow", Build::SeparateCommits);
5112        let bulk = (0..5_000)
5113            .map(|i| BatchOp::InsertNode {
5114                label: "Bulk".into(),
5115                key: format!("bulk{i}"),
5116                props: vec![("n".into(), Value::Int(i))],
5117            })
5118            .collect();
5119        db.submit_batch(bulk).unwrap();
5120        let before = db.read().slow_query_snapshot().count;
5121        // Threshold 1 ms: a filtered scan of 5 000 nodes reaches it, and the count
5122        // is what is asserted, not the duration.
5123        db.write().set_slow_query_threshold_ms(1);
5124        let cypher = "MATCH (n:Bulk) WHERE n.n >= 0 RETURN count(n) AS c";
5125        let mut recorded = false;
5126        for _ in 0..20 {
5127            let got = routed(
5128                &app,
5129                post(
5130                    "/query?format=json",
5131                    json!({ "cypher": cypher }),
5132                    Some(FULL),
5133                ),
5134            )
5135            .await;
5136            assert_eq!(got.status, StatusCode::OK);
5137            if db.read().slow_query_snapshot().count > before {
5138                recorded = true;
5139                break;
5140            }
5141        }
5142        assert!(recorded, "a slow reader query never reached the ring");
5143        let last = db.read().slow_query_snapshot().last;
5144        assert_eq!(last.last().map(|e| e.query.as_str()), Some(cypher));
5145    }
5146}