Skip to main content

server/
http.rs

1//! Thin HTTP wrapper over [`SharedDb`]. Every endpoint is a lock, a public
2//! core-api call, then a response — no business logic.
3//!
4//! # Single-sink design
5//!
6//! The HTTP router is the designated broadcast producer. [`router`] installs
7//! one `broadcast::Sender` as the [`mushroomdb::GraphDb`] event sink. MCP and
8//! CLI mutations on the same [`SharedDb`] fire into that same sink (one
9//! producer, many `/watch` subscribers). A second [`router`] call replaces
10//! the sink and terminates every existing subscriber with
11//! [`tokio::sync::broadcast::error::RecvError::Closed`].
12
13use crate::json::{
14    edge_history_result_json, namespace_arg, node_edges_json, node_history_json, node_info_json,
15    params_from_json, parse_ingest_edges, result_set_json, rule_def_from_json, stamp_namespace,
16    stamp_namespace_row,
17};
18use crate::{AppState, AuthIdentity, ReadOnlyKey};
19use arrow_bridge::to_ipc_bytes;
20use axum::extract::{Extension, Path, Query, Request, State};
21use axum::http::{header, HeaderMap, HeaderValue, Method, StatusCode};
22use axum::middleware::{self, Next};
23use axum::response::{IntoResponse, Response};
24use axum::routing::{get, post};
25use axum::{Json, Router};
26use mushroomdb::{
27    is_write_query, json_to_rows, json_to_value, AccessKeys, AsOfScope, AutoFk, BackupReport,
28    BatchOp, DbEvent, DegreeConfig, Dir, GraphError, IngestOptions, KeyTier, MaskMode, NodeMask,
29    PageRankConfig, PropPredicate, ReaderSnapshot, ResultSet, SchemaError, SchemaErrorKind,
30    SharedDb, SuggestConfig, Value, WccConfig, SUGGEST_DEFAULT_SEED,
31};
32use serde_json::{json, Value as Js};
33use std::collections::{BTreeMap, HashMap};
34use std::fmt::{Display, Write as _};
35use std::net::SocketAddr;
36use std::path::PathBuf;
37use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
38use std::sync::{Arc, PoisonError};
39use tower_http::services::ServeDir;
40
41// The `/tx/*` routes and the open-transaction store. Declared here, not in
42// `lib.rs`, so the module reaches this file's private helpers (`graph_err`,
43// `forbidden`, ...) instead of widening each of them to `pub(crate)`.
44#[path = "tx.rs"]
45mod tx;
46pub(crate) use tx::TxStore;
47pub use tx::{TxLimits, TX_MAX_ENV, TX_MAX_LIFETIME_ENV, TX_TIMEOUT_ENV};
48
49/// Build the HTTP router over `db`. The graph read endpoints (`/query`,
50/// `/node/{key}`, its `/edges` and `/neighborhood`), `/health`, `/metrics` and
51/// `/algo/*` run on a lock-free [`ReaderSnapshot`], whose capture never waits
52/// behind a writer (see [`SharedDb::reader`]). `/stats`, `/explain`,
53/// `/suggest`, `/find_similar` and the history routes read the rule engine or
54/// the WAL and take the read lock; `/ingest` takes the write lock. Guards are
55/// dropped before any `.await`.
56/// `GET /watch` upgrades to a WebSocket fed by the post-commit sink.
57///
58/// **Call at most once per [`SharedDb`].** A second call replaces the sink
59/// and terminates all existing `/watch` subscribers with
60/// [`tokio::sync::broadcast::error::RecvError::Closed`].
61///
62/// Installing the watch sink replaces any previously installed
63/// [`mushroomdb::GraphDb::set_event_sink`]. The sink only
64/// `broadcast::Sender::send`s (non-blocking) and never re-enters `db`.
65///
66/// # Blocking
67///
68/// [`SharedDb`] uses a std [`std::sync::RwLock`]. Write handlers
69/// (`query_write`, `/ingest`, `create_rule`) run in
70/// `tokio::task::spawn_blocking` and drop the write guard before `.await`.
71/// Reads stay on the worker (neighborhood is µs). `suggest` and `algo`
72/// already use the blocking pool. A live (non-`as_of`) graph read holds no
73/// guard while it executes, so a long read never holds the writer up, and an
74/// indexed equality lookup is served from the snapshot's own index.
75///
76/// Every `router_*` entry point reads `MUSHROOMDB_CORS_ORIGINS` (except
77/// [`router_with_cors`], which takes the layer explicitly). Fail-fast on an
78/// invalid value holds only for the CLI; a library caller gets CORS off and an
79/// error log. Validate with [`crate::cors_layer_from_env`] first to fail early.
80pub fn router(db: SharedDb) -> Router {
81    router_with_auth(db, None)
82}
83
84/// [`router`] with an optional bearer/`?token=` requirement on every route
85/// except unauthenticated `GET /health`.
86///
87/// Role enforcement is not active on this entry point; use
88/// [`router_with_role_tokens`] when role-bound tokens are required.
89pub fn router_with_auth(db: SharedDb, token: Option<String>) -> Router {
90    build_app(
91        db,
92        token,
93        HashMap::new(),
94        UiFallback::None,
95        default_advertise_addr(),
96        false,
97    )
98}
99
100/// [`router_with_auth`] whose `GET /ready` follows `ready`: 503 while it is
101/// false, 200 once it is true. Every other router is ready as soon as it is built.
102pub fn router_with_ready_flag(
103    db: SharedDb,
104    token: Option<String>,
105    ready: Arc<AtomicBool>,
106) -> Router {
107    build_app_with_ready(
108        db,
109        token,
110        HashMap::new(),
111        UiFallback::None,
112        default_advertise_addr(),
113        false,
114        ready,
115        None,
116    )
117}
118
119/// [`router_with_role_tokens`] plus the `keys.json` set (see [`AccessKeys`]).
120///
121/// An `admin` key is full access, a `write` key may mutate but is refused
122/// `/backup`, `/stats`, `/metrics`, `/watch` and `/subscribe`, and a `read` key
123/// is the role it names. The env token and `role_tokens` keep working beside
124/// them. `keys` is shared, so a reload of it (see [`crate::watch_sighup`])
125/// reaches every router built from it.
126pub fn router_with_access_keys(
127    db: SharedDb,
128    token: Option<String>,
129    role_tokens: HashMap<String, String>,
130    keys: AccessKeys,
131) -> Router {
132    build_app_with_ready(
133        db,
134        token,
135        role_tokens,
136        UiFallback::None,
137        default_advertise_addr(),
138        false,
139        Arc::new(AtomicBool::new(true)),
140        Some(keys),
141    )
142}
143
144/// [`router_with_auth`] with an explicit CORS layer (see [`crate::parse_cors_origins`]).
145/// `None` adds no CORS headers. The other routers read `MUSHROOMDB_CORS_ORIGINS`
146/// instead; this one ignores the environment.
147pub fn router_with_cors(
148    db: SharedDb,
149    token: Option<String>,
150    cors: Option<tower_http::cors::CorsLayer>,
151) -> Router {
152    build_app_with_cors(
153        db,
154        token,
155        HashMap::new(),
156        UiFallback::None,
157        default_advertise_addr(),
158        false,
159        Arc::new(AtomicBool::new(true)),
160        cors,
161        None,
162        TxStore::new(TxLimits::from_env()),
163    )
164}
165
166/// [`router_with_access_keys`] with the open-transaction limits given instead
167/// of read from `MUSHROOMDB_TX_MAX` and `MUSHROOMDB_TX_TIMEOUT_SECS`. For an
168/// embedder that serves several routers, and for tests that need a short idle
169/// timeout.
170pub fn router_with_tx_limits(
171    db: SharedDb,
172    token: Option<String>,
173    role_tokens: HashMap<String, String>,
174    keys: Option<AccessKeys>,
175    limits: TxLimits,
176) -> Router {
177    build_app_with_ready_and_txs(
178        db,
179        token,
180        role_tokens,
181        UiFallback::None,
182        default_advertise_addr(),
183        false,
184        Arc::new(AtomicBool::new(true)),
185        keys,
186        TxStore::new(limits),
187    )
188}
189
190/// [`router`] with a full-access token and a map of role-bound tokens.
191///
192/// Role-bound tokens (`role_tokens`: bearer → role name) receive masked reads
193/// on `/query` and `/node/*`, and 403 on all write or subscription endpoints.
194/// Unknown token: 401.  Token bound to a role not in the DB at request time: 401.
195pub fn router_with_role_tokens(
196    db: SharedDb,
197    token: Option<String>,
198    role_tokens: HashMap<String, String>,
199) -> Router {
200    build_app(
201        db,
202        token,
203        role_tokens,
204        UiFallback::None,
205        default_advertise_addr(),
206        false,
207    )
208}
209
210/// Same as [`router_with_auth`], then `ServeDir` as the fallback so API routes win.
211///
212/// Role enforcement is not active on this entry point; use
213/// [`router_with_role_tokens`] when role-bound tokens are required.
214pub fn router_with_ui(
215    db: SharedDb,
216    ui_dir: impl AsRef<std::path::Path>,
217    token: Option<String>,
218) -> Router {
219    build_app(
220        db,
221        token,
222        HashMap::new(),
223        UiFallback::Dir(ui_dir.as_ref().to_path_buf()),
224        default_advertise_addr(),
225        false,
226    )
227}
228
229/// Like [`router_with_ui`] but marks the connection as TLS-active so the auth
230/// cookie carries the `Secure` attribute. Use when the binary is serving
231/// directly over HTTPS (i.e. via `serve_tls`) rather than behind a proxy.
232pub fn router_with_ui_tls(
233    db: SharedDb,
234    ui_dir: impl AsRef<std::path::Path>,
235    token: Option<String>,
236) -> Router {
237    build_app(
238        db,
239        token,
240        HashMap::new(),
241        UiFallback::Dir(ui_dir.as_ref().to_path_buf()),
242        default_advertise_addr(),
243        true,
244    )
245}
246
247#[cfg(feature = "embed-ui")]
248static EMBEDDED_UI: include_dir::Dir<'_> =
249    include_dir::include_dir!("$CARGO_MANIFEST_DIR/../../ui/dist");
250
251/// [`router`] plus the `embed-ui` static tree as fallback.
252#[cfg(feature = "embed-ui")]
253pub fn router_with_embedded_ui(db: SharedDb) -> Router {
254    build_app(
255        db,
256        None,
257        HashMap::new(),
258        UiFallback::Embedded,
259        default_advertise_addr(),
260        false,
261    )
262}
263
264#[cfg(feature = "embed-ui")]
265async fn embedded_fallback(uri: axum::http::Uri) -> Response {
266    let rel = if uri.path() == "/" || uri.path().is_empty() {
267        "index.html"
268    } else {
269        uri.path().trim_start_matches('/')
270    };
271    if rel.split('/').any(|seg| seg == "..") {
272        return StatusCode::NOT_FOUND.into_response();
273    }
274    match EMBEDDED_UI.get_file(rel) {
275        Some(file) => (
276            StatusCode::OK,
277            [(header::CONTENT_TYPE, embedded_ctype(rel))],
278            file.contents(),
279        )
280            .into_response(),
281        None => StatusCode::NOT_FOUND.into_response(),
282    }
283}
284
285#[cfg(feature = "embed-ui")]
286fn embedded_ctype(path: &str) -> &'static str {
287    if path.ends_with(".html") {
288        "text/html; charset=utf-8"
289    } else if path.ends_with(".js") {
290        "application/javascript; charset=utf-8"
291    } else if path.ends_with(".css") {
292        "text/css; charset=utf-8"
293    } else if path.ends_with(".woff2") {
294        "font/woff2"
295    } else if path.ends_with(".svg") {
296        "image/svg+xml"
297    } else if path.ends_with(".ico") {
298        "image/x-icon"
299    } else if path.ends_with(".txt") {
300        "text/plain; charset=utf-8"
301    } else {
302        "application/octet-stream"
303    }
304}
305
306/// Bind `addr` (port 0 is ephemeral) and serve.
307///
308/// Sends the resolved local address on `ready` once the listener is accepting.
309/// Does not hold a database lock.
310///
311/// Role enforcement is not active on this entry point; use
312/// [`serve_with_role_tokens`] when role-bound tokens are required.
313#[deprecated(
314    since = "0.2.0",
315    note = "Use `serve_with_role_tokens` instead; this variant silently ignores role-token configuration."
316)]
317#[doc(hidden)]
318pub async fn serve(
319    db: SharedDb,
320    addr: SocketAddr,
321    ready: tokio::sync::oneshot::Sender<SocketAddr>,
322    token: Option<String>,
323) -> std::io::Result<()> {
324    serve_inner(
325        db,
326        addr,
327        ready,
328        UiFallback::None,
329        token,
330        HashMap::new(),
331        None,
332        Arc::new(AtomicBool::new(true)),
333        None,
334        TxLimits::from_env(),
335    )
336    .await
337}
338
339/// [`serve`] with role-bound tokens in addition to the optional full-access token.
340///
341/// `role_tokens` maps bearer values to role names.  See [`router_with_role_tokens`]
342/// for the enforcement semantics.
343pub async fn serve_with_role_tokens(
344    db: SharedDb,
345    addr: SocketAddr,
346    ready: tokio::sync::oneshot::Sender<SocketAddr>,
347    token: Option<String>,
348    role_tokens: HashMap<String, String>,
349) -> std::io::Result<()> {
350    serve_inner(
351        db,
352        addr,
353        ready,
354        UiFallback::None,
355        token,
356        role_tokens,
357        None,
358        Arc::new(AtomicBool::new(true)),
359        None,
360        TxLimits::from_env(),
361    )
362    .await
363}
364
365/// [`serve`] plus a UI dist directory mounted behind the API routes.
366///
367/// Role enforcement is not active on this entry point; use
368/// [`serve_with_ui_and_role_tokens`] when role-bound tokens are required.
369#[deprecated(
370    since = "0.2.0",
371    note = "Use `serve_with_ui_and_role_tokens` instead; this variant silently ignores role-token configuration."
372)]
373#[doc(hidden)]
374pub async fn serve_with_ui(
375    db: SharedDb,
376    addr: SocketAddr,
377    ready: tokio::sync::oneshot::Sender<SocketAddr>,
378    ui_dir: PathBuf,
379    token: Option<String>,
380) -> std::io::Result<()> {
381    serve_inner(
382        db,
383        addr,
384        ready,
385        UiFallback::Dir(ui_dir),
386        token,
387        HashMap::new(),
388        None,
389        Arc::new(AtomicBool::new(true)),
390        None,
391        TxLimits::from_env(),
392    )
393    .await
394}
395
396/// [`serve_with_ui`] with role-bound tokens.
397pub async fn serve_with_ui_and_role_tokens(
398    db: SharedDb,
399    addr: SocketAddr,
400    ready: tokio::sync::oneshot::Sender<SocketAddr>,
401    ui_dir: PathBuf,
402    token: Option<String>,
403    role_tokens: HashMap<String, String>,
404) -> std::io::Result<()> {
405    serve_inner(
406        db,
407        addr,
408        ready,
409        UiFallback::Dir(ui_dir),
410        token,
411        role_tokens,
412        None,
413        Arc::new(AtomicBool::new(true)),
414        None,
415        TxLimits::from_env(),
416    )
417    .await
418}
419
420/// [`serve`] plus the compiled-in UI (no-op fallback if `embed-ui` is off).
421///
422/// `role_tokens` maps bearer values to role names; see [`router_with_role_tokens`]
423/// for enforcement semantics.  Pass `HashMap::new()` when no role tokens are needed.
424#[cfg(feature = "embed-ui")]
425pub async fn serve_with_embedded_ui(
426    db: SharedDb,
427    addr: SocketAddr,
428    ready: tokio::sync::oneshot::Sender<SocketAddr>,
429    token: Option<String>,
430    role_tokens: HashMap<String, String>,
431) -> std::io::Result<()> {
432    serve_inner(
433        db,
434        addr,
435        ready,
436        UiFallback::Embedded,
437        token,
438        role_tokens,
439        None,
440        Arc::new(AtomicBool::new(true)),
441        None,
442        TxLimits::from_env(),
443    )
444    .await
445}
446
447/// The graceful-shutdown variants, added in 0.6.12.
448///
449/// Each mirrors the entry point above it and takes one extra argument: a
450/// receiver that, when it fires **or is dropped**, stops the server accepting
451/// new connections and lets every request already in flight finish. Without
452/// one, the only way to stop a server was to abort the task running it, which
453/// severs in-flight responses and live `/watch` and `/subscribe` subscribers
454/// mid-stream.
455///
456/// These are additive on purpose. The four originals keep their signatures and
457/// their behaviour, so nothing embedding this crate has to change; they simply
458/// pass no signal. The cost is eight entry points where four would do, and that
459/// was the trade chosen over a breaking parameter.
460pub async fn serve_with_shutdown(
461    db: SharedDb,
462    addr: SocketAddr,
463    ready: tokio::sync::oneshot::Sender<SocketAddr>,
464    token: Option<String>,
465    shutdown: tokio::sync::oneshot::Receiver<()>,
466) -> std::io::Result<()> {
467    serve_inner(
468        db,
469        addr,
470        ready,
471        UiFallback::None,
472        token,
473        HashMap::new(),
474        Some(shutdown),
475        Arc::new(AtomicBool::new(true)),
476        None,
477        TxLimits::from_env(),
478    )
479    .await
480}
481
482/// [`serve_with_role_tokens`] that stops gracefully. See [`serve_with_shutdown`].
483pub async fn serve_with_role_tokens_and_shutdown(
484    db: SharedDb,
485    addr: SocketAddr,
486    ready: tokio::sync::oneshot::Sender<SocketAddr>,
487    token: Option<String>,
488    role_tokens: HashMap<String, String>,
489    shutdown: tokio::sync::oneshot::Receiver<()>,
490) -> std::io::Result<()> {
491    serve_with_role_tokens_and_shutdown_with_ready_flag(
492        db,
493        addr,
494        ready,
495        token,
496        role_tokens,
497        shutdown,
498        Arc::new(AtomicBool::new(true)),
499        None,
500        TxLimits::from_env(),
501    )
502    .await
503}
504
505/// [`serve_with_role_tokens_and_shutdown`] whose `GET /ready` follows `ready_flag`.
506#[allow(clippy::too_many_arguments)]
507pub async fn serve_with_role_tokens_and_shutdown_with_ready_flag(
508    db: SharedDb,
509    addr: SocketAddr,
510    ready: tokio::sync::oneshot::Sender<SocketAddr>,
511    token: Option<String>,
512    role_tokens: HashMap<String, String>,
513    shutdown: tokio::sync::oneshot::Receiver<()>,
514    ready_flag: Arc<AtomicBool>,
515    keys: Option<AccessKeys>,
516    tx_limits: TxLimits,
517) -> std::io::Result<()> {
518    serve_inner(
519        db,
520        addr,
521        ready,
522        UiFallback::None,
523        token,
524        role_tokens,
525        Some(shutdown),
526        ready_flag,
527        keys,
528        tx_limits,
529    )
530    .await
531}
532
533/// [`serve_with_ui_and_role_tokens`] that stops gracefully. See [`serve_with_shutdown`].
534pub async fn serve_with_ui_and_role_tokens_and_shutdown(
535    db: SharedDb,
536    addr: SocketAddr,
537    ready: tokio::sync::oneshot::Sender<SocketAddr>,
538    ui_dir: PathBuf,
539    token: Option<String>,
540    role_tokens: HashMap<String, String>,
541    shutdown: tokio::sync::oneshot::Receiver<()>,
542) -> std::io::Result<()> {
543    serve_with_ui_and_role_tokens_and_shutdown_with_ready_flag(
544        db,
545        addr,
546        ready,
547        ui_dir,
548        token,
549        role_tokens,
550        shutdown,
551        Arc::new(AtomicBool::new(true)),
552        None,
553        TxLimits::from_env(),
554    )
555    .await
556}
557
558/// [`serve_with_ui_and_role_tokens_and_shutdown`] whose `GET /ready` follows `ready_flag`.
559#[allow(clippy::too_many_arguments)]
560pub async fn serve_with_ui_and_role_tokens_and_shutdown_with_ready_flag(
561    db: SharedDb,
562    addr: SocketAddr,
563    ready: tokio::sync::oneshot::Sender<SocketAddr>,
564    ui_dir: PathBuf,
565    token: Option<String>,
566    role_tokens: HashMap<String, String>,
567    shutdown: tokio::sync::oneshot::Receiver<()>,
568    ready_flag: Arc<AtomicBool>,
569    keys: Option<AccessKeys>,
570    tx_limits: TxLimits,
571) -> std::io::Result<()> {
572    serve_inner(
573        db,
574        addr,
575        ready,
576        UiFallback::Dir(ui_dir),
577        token,
578        role_tokens,
579        Some(shutdown),
580        ready_flag,
581        keys,
582        tx_limits,
583    )
584    .await
585}
586
587/// [`serve_with_embedded_ui`] that stops gracefully. See [`serve_with_shutdown`].
588#[cfg(feature = "embed-ui")]
589pub async fn serve_with_embedded_ui_and_shutdown(
590    db: SharedDb,
591    addr: SocketAddr,
592    ready: tokio::sync::oneshot::Sender<SocketAddr>,
593    token: Option<String>,
594    role_tokens: HashMap<String, String>,
595    shutdown: tokio::sync::oneshot::Receiver<()>,
596) -> std::io::Result<()> {
597    serve_with_embedded_ui_and_shutdown_with_ready_flag(
598        db,
599        addr,
600        ready,
601        token,
602        role_tokens,
603        shutdown,
604        Arc::new(AtomicBool::new(true)),
605        None,
606        TxLimits::from_env(),
607    )
608    .await
609}
610
611/// [`serve_with_embedded_ui_and_shutdown`] whose `GET /ready` follows `ready_flag`.
612#[cfg(feature = "embed-ui")]
613#[allow(clippy::too_many_arguments)]
614pub async fn serve_with_embedded_ui_and_shutdown_with_ready_flag(
615    db: SharedDb,
616    addr: SocketAddr,
617    ready: tokio::sync::oneshot::Sender<SocketAddr>,
618    token: Option<String>,
619    role_tokens: HashMap<String, String>,
620    shutdown: tokio::sync::oneshot::Receiver<()>,
621    ready_flag: Arc<AtomicBool>,
622    keys: Option<AccessKeys>,
623    tx_limits: TxLimits,
624) -> std::io::Result<()> {
625    serve_inner(
626        db,
627        addr,
628        ready,
629        UiFallback::Embedded,
630        token,
631        role_tokens,
632        Some(shutdown),
633        ready_flag,
634        keys,
635        tx_limits,
636    )
637    .await
638}
639
640enum UiFallback {
641    None,
642    Dir(PathBuf),
643    #[cfg(feature = "embed-ui")]
644    Embedded,
645}
646
647#[allow(clippy::too_many_arguments)]
648async fn serve_inner(
649    db: SharedDb,
650    addr: SocketAddr,
651    ready: tokio::sync::oneshot::Sender<SocketAddr>,
652    ui: UiFallback,
653    token: Option<String>,
654    role_tokens: HashMap<String, String>,
655    shutdown: Option<tokio::sync::oneshot::Receiver<()>>,
656    ready_flag: Arc<AtomicBool>,
657    keys: Option<AccessKeys>,
658    tx_limits: TxLimits,
659) -> std::io::Result<()> {
660    let listener = tokio::net::TcpListener::bind(addr).await?;
661    let local = listener.local_addr()?;
662    if ready.send(local).is_err() {
663        // Caller dropped the readiness receiver; still serve.
664        eprintln!("serve: readiness receiver dropped before bind notify");
665    }
666    let txs = TxStore::new(tx_limits);
667    let app = build_app_with_ready_and_txs(
668        db,
669        token,
670        role_tokens,
671        ui,
672        local,
673        false,
674        ready_flag,
675        keys,
676        txs.clone(),
677    );
678    match shutdown {
679        None => axum::serve(listener, app).await,
680        // Stop accepting, then let every request already in flight finish.
681        // Dropping the sender counts as the signal, so a caller that goes away
682        // shuts the server down rather than stranding it.
683        Some(rx) => {
684            axum::serve(listener, app)
685                .with_graceful_shutdown(async move {
686                    let _ = rx.await;
687                    // The start of the drain: open transactions roll back and
688                    // a new `begin` is refused.
689                    txs.close();
690                })
691                .await
692        }
693    }
694}
695
696/// Serve over TLS (requires the `tls` cargo feature).
697///
698/// Accepts a path to a PEM certificate and a PEM private key. Uses rustls
699/// via `axum-server`. The `ready` sender fires with the bound local address
700/// once the listener is accepting, mirroring the contract of [`serve_inner`].
701///
702/// Pass `token` / `role_tokens` the same way you would for [`serve_with_role_tokens`].
703#[cfg(feature = "tls")]
704pub async fn serve_tls(
705    db: SharedDb,
706    addr: SocketAddr,
707    ready: tokio::sync::oneshot::Sender<SocketAddr>,
708    cert_path: std::path::PathBuf,
709    key_path: std::path::PathBuf,
710    token: Option<String>,
711    role_tokens: HashMap<String, String>,
712) -> std::io::Result<()> {
713    let config = crate::tls::http_rustls_config(&cert_path, &key_path)?;
714    let listener = std::net::TcpListener::bind(addr)?;
715    listener.set_nonblocking(true)?;
716    let local = listener.local_addr()?;
717    if ready.send(local).is_err() {
718        eprintln!("serve_tls: readiness receiver dropped before bind notify");
719    }
720    let app = build_app(db, token, role_tokens, UiFallback::None, local, true);
721    axum_server::from_tcp_rustls(listener, config)?
722        .serve(app.into_make_service())
723        .await
724}
725
726/// [`serve_tls`] that stops gracefully. See [`serve_with_shutdown`].
727///
728/// `axum_server` has its own shutdown mechanism — a [`axum_server::Handle`] —
729/// rather than axum's `with_graceful_shutdown`, so the signal is bridged onto
730/// one here. In-flight requests get [`TLS_GRACE`] to finish before the
731/// listener is closed regardless; an unbounded grace period would let one
732/// stuck request hold a restart open forever.
733#[cfg(feature = "tls")]
734#[allow(clippy::too_many_arguments)]
735pub async fn serve_tls_with_shutdown(
736    db: SharedDb,
737    addr: SocketAddr,
738    ready: tokio::sync::oneshot::Sender<SocketAddr>,
739    cert_path: std::path::PathBuf,
740    key_path: std::path::PathBuf,
741    token: Option<String>,
742    role_tokens: HashMap<String, String>,
743    shutdown: tokio::sync::oneshot::Receiver<()>,
744) -> std::io::Result<()> {
745    serve_tls_with_shutdown_with_ready_flag(
746        db,
747        addr,
748        ready,
749        cert_path,
750        key_path,
751        token,
752        role_tokens,
753        shutdown,
754        Arc::new(AtomicBool::new(true)),
755        None,
756        TxLimits::from_env(),
757    )
758    .await
759}
760
761/// [`serve_tls_with_shutdown`] whose `GET /ready` follows `ready_flag`.
762#[cfg(feature = "tls")]
763#[allow(clippy::too_many_arguments)]
764pub async fn serve_tls_with_shutdown_with_ready_flag(
765    db: SharedDb,
766    addr: SocketAddr,
767    ready: tokio::sync::oneshot::Sender<SocketAddr>,
768    cert_path: std::path::PathBuf,
769    key_path: std::path::PathBuf,
770    token: Option<String>,
771    role_tokens: HashMap<String, String>,
772    shutdown: tokio::sync::oneshot::Receiver<()>,
773    ready_flag: Arc<AtomicBool>,
774    keys: Option<AccessKeys>,
775    tx_limits: TxLimits,
776) -> std::io::Result<()> {
777    let config = crate::tls::http_rustls_config(&cert_path, &key_path)?;
778    let listener = std::net::TcpListener::bind(addr)?;
779    listener.set_nonblocking(true)?;
780    let local = listener.local_addr()?;
781    if ready.send(local).is_err() {
782        eprintln!("serve_tls: readiness receiver dropped before bind notify");
783    }
784    let txs = TxStore::new(tx_limits);
785    let app = build_app_with_ready_and_txs(
786        db,
787        token,
788        role_tokens,
789        UiFallback::None,
790        local,
791        true,
792        ready_flag,
793        keys,
794        txs.clone(),
795    );
796    let handle = axum_server::Handle::new();
797    tokio::spawn({
798        let handle = handle.clone();
799        async move {
800            let _ = shutdown.await;
801            txs.close();
802            handle.graceful_shutdown(Some(TLS_GRACE));
803        }
804    });
805    axum_server::from_tcp_rustls(listener, config)?
806        .handle(handle)
807        .serve(app.into_make_service())
808        .await
809}
810
811/// How long a TLS server lets in-flight requests finish after a shutdown
812/// signal before closing anyway.
813#[cfg(feature = "tls")]
814const TLS_GRACE: std::time::Duration = std::time::Duration::from_secs(30);
815
816fn default_advertise_addr() -> SocketAddr {
817    SocketAddr::from(([127, 0, 0, 1], 8080))
818}
819
820fn build_app(
821    db: SharedDb,
822    token: Option<String>,
823    role_tokens: HashMap<String, String>,
824    ui: UiFallback,
825    addr: SocketAddr,
826    tls_active: bool,
827) -> Router {
828    build_app_with_ready(
829        db,
830        token,
831        role_tokens,
832        ui,
833        addr,
834        tls_active,
835        Arc::new(AtomicBool::new(true)),
836        None,
837    )
838}
839
840/// Reads `MUSHROOMDB_CORS_ORIGINS`. An invalid value does not fail here: the CLI
841/// validates before opening the store; other callers get CORS off plus an error log.
842#[allow(clippy::too_many_arguments)]
843fn build_app_with_ready(
844    db: SharedDb,
845    token: Option<String>,
846    role_tokens: HashMap<String, String>,
847    ui: UiFallback,
848    addr: SocketAddr,
849    tls_active: bool,
850    ready: Arc<AtomicBool>,
851    keys: Option<AccessKeys>,
852) -> Router {
853    build_app_with_ready_and_txs(
854        db,
855        token,
856        role_tokens,
857        ui,
858        addr,
859        tls_active,
860        ready,
861        keys,
862        TxStore::new(TxLimits::from_env()),
863    )
864}
865
866/// [`build_app_with_ready`] over a transaction store the caller holds, so the
867/// serve loop can close it when the shutdown signal arrives.
868#[allow(clippy::too_many_arguments)]
869fn build_app_with_ready_and_txs(
870    db: SharedDb,
871    token: Option<String>,
872    role_tokens: HashMap<String, String>,
873    ui: UiFallback,
874    addr: SocketAddr,
875    tls_active: bool,
876    ready: Arc<AtomicBool>,
877    keys: Option<AccessKeys>,
878    txs: TxStore,
879) -> Router {
880    // The CLI validates the variable before it opens the store; an invalid value
881    // reaching here (library use) fails closed: no CORS, and an error logged.
882    let cors = crate::cors::cors_layer_from_env().unwrap_or_else(|e| {
883        tracing::error!("{e}; CORS disabled");
884        None
885    });
886    build_app_with_cors(
887        db,
888        token,
889        role_tokens,
890        ui,
891        addr,
892        tls_active,
893        ready,
894        cors,
895        keys,
896        txs,
897    )
898}
899
900#[allow(clippy::too_many_arguments)]
901fn build_app_with_cors(
902    db: SharedDb,
903    token: Option<String>,
904    role_tokens: HashMap<String, String>,
905    ui: UiFallback,
906    addr: SocketAddr,
907    tls_active: bool,
908    ready: Arc<AtomicBool>,
909    cors: Option<tower_http::cors::CorsLayer>,
910    keys: Option<AccessKeys>,
911    txs: TxStore,
912) -> Router {
913    debug_assert!(
914        !db.read().has_event_sink(),
915        "router() must be called at most once per SharedDb; a second call \
916         replaces the sink and terminates all existing /watch subscribers \
917         with RecvError::Closed"
918    );
919    let (tx, _) = tokio::sync::broadcast::channel::<String>(1024);
920    // Serialize each MutationEvent to JSON and broadcast the string.
921    {
922        let tx = tx.clone();
923        db.write().set_event_sink(Box::new(move |ev| {
924            if let Ok(json) = serde_json::to_string(&ev) {
925                let _ = tx.send(json);
926            }
927        }));
928    }
929    // Bridge edge-prop DbEvents to /watch (row 155).
930    // `subscribe_edge_props()` is called here, synchronously, so the
931    // subscription is registered before this function returns.  Any write that
932    // fires after `start_state` completes is guaranteed to reach the bridge.
933    // The narrower `EdgePropsOnly` filter means only edge-property commits wake
934    // the bridge loop, keeping the subscription queue and lag risk low.  The
935    // polling loop runs in a persistent blocking thread so the async runtime is
936    // not blocked on `recv_timeout`.
937    //
938    // `Lagged` events from the subscription (dropped events due to a full
939    // queue) are forwarded as `{"lagged": N}` — the same frame shape that
940    // `ws::watch_text` emits when the broadcast channel itself lags.
941    //
942    // Lifetime: the bridge holds a `Weak` reference to `bridge_keepalive`.
943    // When every `AppState` clone is dropped (server shutdown), the `Arc` is
944    // released and the `Weak` upgrade fails → bridge exits its loop.  This
945    // prevents the tokio blocking-thread pool from hanging on shutdown.
946    let bridge_keepalive = std::sync::Arc::new(());
947    {
948        let tx = tx.clone();
949        let alive_weak = std::sync::Arc::downgrade(&bridge_keepalive);
950        match db.write().subscribe_edge_props() {
951            Err(e) => {
952                tracing::warn!("watch edge-prop bridge: subscribe_edge_props failed: {e}");
953            }
954            Ok(sub) => {
955                // `spawn_blocking` requires a tokio runtime.  When
956                // `build_app_with_cors` is called from a sync context (e.g.
957                // unit tests that build a Router without a running runtime),
958                // skip the bridge; edge-prop events won't appear on /watch
959                // in that context, which is acceptable since no server is up.
960                if tokio::runtime::Handle::try_current().is_ok() {
961                    tokio::task::spawn_blocking(move || {
962                        let idle = std::time::Duration::from_millis(100);
963                        loop {
964                            match sub.recv_timeout(idle) {
965                                Some(DbEvent::EdgePropSet {
966                                    edge_type,
967                                    src,
968                                    dst,
969                                    field,
970                                    ..
971                                }) => {
972                                    let frame = serde_json::json!({
973                                        "edge_prop_set": {
974                                            "edge_type": edge_type,
975                                            "src": src,
976                                            "dst": dst,
977                                            "field": field
978                                        }
979                                    });
980                                    let _ = tx.send(frame.to_string());
981                                }
982                                Some(DbEvent::EdgePropRemoved {
983                                    edge_type,
984                                    src,
985                                    dst,
986                                    field,
987                                    ..
988                                }) => {
989                                    let frame = serde_json::json!({
990                                        "edge_prop_removed": {
991                                            "edge_type": edge_type,
992                                            "src": src,
993                                            "dst": dst,
994                                            "field": field
995                                        }
996                                    });
997                                    let _ = tx.send(frame.to_string());
998                                }
999                                Some(DbEvent::Lagged { missed }) => {
1000                                    // Forward the bridge's own queue overflow as
1001                                    // the same {"lagged": N} frame that ws.rs
1002                                    // emits for broadcast-channel lag.
1003                                    let _ =
1004                                        tx.send(serde_json::json!({"lagged": missed}).to_string());
1005                                }
1006                                _ => {
1007                                    // On each timeout (None), check if the server
1008                                    // has shut down (all AppState clones dropped).
1009                                    if alive_weak.upgrade().is_none() {
1010                                        break;
1011                                    }
1012                                }
1013                            }
1014                        }
1015                    });
1016                } // end if tokio runtime available
1017            }
1018        }
1019    }
1020    let state = AppState {
1021        db,
1022        watch: tx,
1023        token,
1024        role_tokens,
1025        addr,
1026        tls_active,
1027        started_at: std::time::Instant::now(),
1028        started_unix_s: unix_seconds_now(),
1029        ready,
1030        access_keys: keys,
1031        txs,
1032        _bridge_keepalive: bridge_keepalive,
1033    };
1034    let app = Router::new()
1035        .route("/health", get(health))
1036        .route("/ready", get(ready_handler))
1037        .route("/query", post(query))
1038        .route("/schema/indexes", get(schema_indexes_list))
1039        .route("/schema/indexes", post(schema_index_create))
1040        .route(
1041            "/schema/indexes/{name}",
1042            axum::routing::delete(schema_index_drop),
1043        )
1044        .route("/schema/constraints", get(schema_constraints_list))
1045        .route("/schema/constraints", post(schema_constraint_create))
1046        .route(
1047            "/schema/constraints/{name}",
1048            axum::routing::delete(schema_constraint_drop),
1049        )
1050        .route("/stats", get(stats))
1051        .route("/metrics", get(metrics))
1052        .route("/ingest", post(ingest))
1053        .route("/rules", post(create_rule))
1054        .route("/suggest", get(suggest))
1055        .route("/explain", get(explain))
1056        .route("/find_similar", post(find_similar))
1057        .route("/node/{key}", get(node_info))
1058        .route("/node/{key}", axum::routing::delete(delete_node))
1059        .route("/node/{key}/edges", get(node_edges))
1060        .route("/node/{key}/neighborhood", get(neighborhood))
1061        .route("/node/{key}/history", get(node_history_handler))
1062        .route("/history/edge", get(edge_history_handler))
1063        .route("/history/was_linked", get(was_linked_handler))
1064        .route(
1065            "/node/{key}/prop/{field}",
1066            axum::routing::put(set_node_prop),
1067        )
1068        .route(
1069            "/node/{key}/prop/{field}",
1070            axum::routing::delete(remove_node_prop),
1071        )
1072        // Simple BatchOp-mapped endpoints — routed through the group-commit
1073        // queue (submit_batch) so concurrent node/edge CRUD does not hold
1074        // the write lock during fsync.
1075        .route("/nodes", post(create_node))
1076        .route("/nodes/{key}/rename", post(rename_node))
1077        .route("/edges", post(create_edge))
1078        .route("/edges/upsert", post(upsert_edge))
1079        .route(
1080            "/edges/{etype}/{src}/{dst}",
1081            axum::routing::delete(delete_edge),
1082        )
1083        .route("/algo/pagerank", post(algo_pagerank))
1084        .route("/algo/wcc", post(algo_wcc))
1085        .route("/algo/degree", post(algo_degree))
1086        .route("/backup", post(backup))
1087        .merge(tx::routes())
1088        .with_state(state.clone());
1089
1090    // The two WebSocket routes are built separately and merged unlayered: a
1091    // request timeout applied to them would sever a healthy subscription at
1092    // the deadline, and a subscription is meant to outlive any request.
1093    let streaming = Router::new()
1094        .route("/watch", get(crate::ws::watch))
1095        .route("/subscribe", get(crate::subscribe::subscribe))
1096        .with_state(state.clone());
1097
1098    // Everything else gets a deadline and a ceiling.
1099    //
1100    // `RequestBodyTimeoutLayer` is deliberately not what is wanted here: the
1101    // risk is a handler that never finishes, not only a client that never
1102    // finishes sending. `TimeoutLayer` bounds the whole exchange and answers
1103    // 408 when it expires.
1104    //
1105    // The concurrency limit sheds rather than queues. An unbounded queue in
1106    // front of a single-writer engine converts overload into latency that
1107    // never recovers; 503 is the honest answer, and the one a load balancer
1108    // can act on.
1109    let app = app
1110        .layer(tower_http::timeout::TimeoutLayer::with_status_code(
1111            StatusCode::REQUEST_TIMEOUT,
1112            REQUEST_TIMEOUT,
1113        ))
1114        .layer(
1115            tower::ServiceBuilder::new()
1116                // Load shedding surfaces as a `BoxError`, and a Router's stack
1117                // must be infallible, so the overload is mapped to a response
1118                // here rather than escaping as an error.
1119                .layer(axum::error_handling::HandleErrorLayer::new(
1120                    |_: tower::BoxError| async {
1121                        (
1122                            StatusCode::SERVICE_UNAVAILABLE,
1123                            Json(json!({
1124                                "error": "server at its concurrency limit; retry"
1125                            })),
1126                        )
1127                    },
1128                ))
1129                .load_shed()
1130                .concurrency_limit(MAX_CONCURRENT_REQUESTS),
1131        );
1132    let app = app.merge(streaming);
1133    let app = match ui {
1134        UiFallback::None => app,
1135        UiFallback::Dir(dir) => app.fallback_service(ServeDir::new(dir)),
1136        #[cfg(feature = "embed-ui")]
1137        UiFallback::Embedded => app.fallback(embedded_fallback),
1138    };
1139    let app = app
1140        .layer(middleware::from_fn_with_state(state, auth_middleware))
1141        // Outside auth, so a 401 is logged too — an unauthenticated flood is
1142        // exactly the thing an operator needs to see.
1143        .layer(middleware::from_fn(log_middleware))
1144        // Cap request bodies (default axum limit is only 2 MiB; we allow larger
1145        // ingest batches but reject multi-GB bodies that would OOM the collector
1146        // before any handler runs).
1147        .layer(axum::extract::DefaultBodyLimit::max(MAX_REQUEST_BODY_BYTES));
1148    // Outermost: a preflight is answered here, before auth (it carries no
1149    // credentials), and a 401 still gets CORS headers so a page can read it.
1150    match cors {
1151        Some(cors) => app
1152            .layer(cors)
1153            .layer(middleware::from_fn(crate::cors::strip_unless_allowed)),
1154        None => app,
1155    }
1156}
1157
1158/// Maximum accepted HTTP request body size (64 MiB). Large enough for batched
1159/// `/ingest` payloads, small enough to prevent a single request from
1160/// exhausting memory. Clients with larger imports should chunk.
1161/// Per-request logging, with a policy chosen so the log stays readable.
1162///
1163/// A line per request at one level is what makes a server log unreadable: the
1164/// 200s bury the 500s, and nobody reads it after the first week. So the level
1165/// follows what happened:
1166///
1167/// | Outcome | Level | Visible by default |
1168/// |---|---|---|
1169/// | 5xx | `error` | yes — always worth a look |
1170/// | slower than [`SLOW_REQUEST`] | `warn` | yes — the request that will time out next |
1171/// | 4xx | `info` | yes — one line, the caller's mistake or a probe |
1172/// | everything else | `debug` | no |
1173///
1174/// A healthy server is therefore quiet, and turning `debug` on for one module
1175/// gives the full trace without recompiling.
1176///
1177/// Every response carries `x-request-id`, echoed from the caller's header when
1178/// it sent one and generated otherwise, so an error someone reports can be
1179/// found in the log. Before 0.6.12 there was no `tracing`, no access log and no
1180/// request id anywhere in the tree: a 500 in production could not be tied to
1181/// the request that caused it.
1182async fn log_middleware(req: Request, next: Next) -> Response {
1183    let request_id = req
1184        .headers()
1185        .get("x-request-id")
1186        .and_then(|v| v.to_str().ok())
1187        .filter(|v| !v.is_empty() && v.len() <= 128 && v.is_ascii())
1188        .map(str::to_owned)
1189        .unwrap_or_else(new_request_id);
1190
1191    let method = req.method().clone();
1192    // The path only. A query string can carry `?token=`, and a log is exactly
1193    // the wrong place for a credential.
1194    let path = req.uri().path().to_owned();
1195    let started = std::time::Instant::now();
1196
1197    let mut res = next.run(req).await;
1198    let status = res.status();
1199    let ms = started.elapsed().as_millis();
1200
1201    if status.is_server_error() {
1202        tracing::error!(%request_id, %method, %path, status = status.as_u16(), ms, "request failed");
1203    } else if started.elapsed() >= SLOW_REQUEST {
1204        tracing::warn!(%request_id, %method, %path, status = status.as_u16(), ms, "slow request");
1205    } else if status.is_client_error() {
1206        tracing::info!(%request_id, %method, %path, status = status.as_u16(), ms, "request refused");
1207    } else {
1208        tracing::debug!(%request_id, %method, %path, status = status.as_u16(), ms, "request");
1209    }
1210
1211    if let Ok(v) = axum::http::HeaderValue::from_str(&request_id) {
1212        res.headers_mut().insert("x-request-id", v);
1213    }
1214    res
1215}
1216
1217/// A request id with no dependency on a uuid crate: the process id, a
1218/// monotonic counter, and the low bits of a nanosecond clock. Unique enough to
1219/// correlate a report with a log line, which is all it is for.
1220fn new_request_id() -> String {
1221    use std::sync::atomic::{AtomicU64, Ordering};
1222    static N: AtomicU64 = AtomicU64::new(0);
1223    let n = N.fetch_add(1, Ordering::Relaxed);
1224    let nanos = std::time::SystemTime::now()
1225        .duration_since(std::time::UNIX_EPOCH)
1226        .map(|d| d.subsec_nanos() as u64)
1227        .unwrap_or(0);
1228    format!("{:x}-{:x}-{:x}", std::process::id(), n, nanos)
1229}
1230
1231/// A request slower than this is logged at `warn` whatever its status. Well
1232/// under [`REQUEST_TIMEOUT`], so the log names the requests heading for a
1233/// timeout before one happens.
1234const SLOW_REQUEST: std::time::Duration = std::time::Duration::from_secs(5);
1235
1236const MAX_REQUEST_BODY_BYTES: usize = 64 * 1024 * 1024;
1237
1238/// How long any one non-streaming request may take before the server answers
1239/// `408 Request Timeout` and drops it.
1240///
1241/// Sized for the slowest legitimate handler rather than the typical one: a
1242/// `/query` against a large store, a `/backup`, or a `/suggest` that profiles
1243/// the graph can all take tens of seconds. The point is not to make slow
1244/// requests fast — it is that a handler which will never finish stops holding
1245/// a connection, a permit and a read guard forever.
1246///
1247/// The WebSocket routes are exempt; see `build_app`.
1248const REQUEST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(120);
1249
1250/// How many non-streaming requests may be in flight at once before the server
1251/// sheds load with `503 Service Unavailable`.
1252///
1253/// The engine is single-writer and many-reader, so the ceiling exists to bound
1254/// memory and the blocking pool, not to serialise work. Chosen well above any
1255/// plausible operator workload and well below what would exhaust the process.
1256const MAX_CONCURRENT_REQUESTS: usize = 256;
1257
1258/// Readiness for a container platform. Needs no token and says nothing about
1259/// the graph.
1260async fn ready_handler(State(state): State<AppState>) -> Response {
1261    if state.ready.load(Ordering::Acquire) {
1262        json_ok(json!({ "ok": true }))
1263    } else {
1264        let mut response = json_ok(json!({ "ok": false, "reason": "not ready" }));
1265        *response.status_mut() = StatusCode::SERVICE_UNAVAILABLE;
1266        response
1267    }
1268}
1269
1270async fn health(
1271    State(state): State<AppState>,
1272    Extension(identity): Extension<AuthIdentity>,
1273) -> Response {
1274    // Liveness is for everyone; the graph's shape is not. `/stats` and
1275    // `/metrics` already refuse a role-bound token because counters leak graph
1276    // size, and an endpoint that hands the same counters to a caller with no
1277    // token at all makes that refusal decorative.
1278    if !matches!(identity, AuthIdentity::Full) {
1279        return json_ok(json!({ "ok": true }));
1280    }
1281    let (nodes, _, edges) = state.db.reader().counts();
1282    json_ok(json!({
1283        "ok": true,
1284        "nodes": nodes,
1285        "edges": edges,
1286        "addr": state.addr.to_string(),
1287    }))
1288}
1289
1290/// Run a GraphDb write on the blocking pool. The write guard lives only
1291/// inside `f` and is dropped before this future awaits.
1292async fn blocking_write<T, F>(f: F) -> std::result::Result<T, Response>
1293where
1294    T: Send + 'static,
1295    F: FnOnce() -> mushroomdb::Result<T> + Send + 'static,
1296{
1297    blocking_write_with(f, graph_err).await
1298}
1299
1300/// [`blocking_write`] with the engine error answered by `on_err`.
1301async fn blocking_write_with<T, F>(
1302    f: F,
1303    on_err: fn(GraphError) -> Response,
1304) -> std::result::Result<T, Response>
1305where
1306    T: Send + 'static,
1307    F: FnOnce() -> mushroomdb::Result<T> + Send + 'static,
1308{
1309    match tokio::task::spawn_blocking(f).await {
1310        Ok(Ok(v)) => Ok(v),
1311        Ok(Err(e)) => Err(on_err(e)),
1312        Err(_) => Err(err_response("write task panicked")),
1313    }
1314}
1315
1316const TOKEN_COOKIE: &str = "mushroomdb_token";
1317
1318async fn auth_middleware(State(state): State<AppState>, mut req: Request, next: Next) -> Response {
1319    // No auth configured at all: every request is Full, no restriction.
1320    let keys_enforce = state
1321        .access_keys
1322        .as_ref()
1323        .is_some_and(AccessKeys::enforcing);
1324    if state.token.is_none() && state.role_tokens.is_empty() && !keys_enforce {
1325        req.extensions_mut().insert(AuthIdentity::Full);
1326        return next.run(req).await;
1327    }
1328
1329    let presented = request_token(&req);
1330
1331    // Health stays reachable without a credential — a load balancer has none —
1332    // but reaching it is not the same as being trusted by it. A caller whose
1333    // token does not resolve to full access is `Anonymous` here, and the
1334    // handler discloses liveness only.
1335    //
1336    // Before 0.6.12 this inserted `Full` unconditionally, so an unauthenticated
1337    // caller read `nodes_live` and `edges` from `/health` while a role-bound
1338    // token was refused the same counters by `/metrics` and `/stats` on the
1339    // stated grounds that counters leak graph size. Two endpoints, one
1340    // disclosure, opposite policies.
1341    if req.method() == Method::GET && req.uri().path() == "/ready" {
1342        req.extensions_mut().insert(AuthIdentity::Anonymous);
1343        return next.run(req).await;
1344    }
1345    if req.method() == Method::GET && req.uri().path() == "/health" {
1346        let full = state
1347            .token
1348            .clone()
1349            .filter(|s| !s.is_empty())
1350            .zip(presented.as_deref())
1351            .is_some_and(|(tok, p)| constant_time_eq(p.as_bytes(), tok.as_bytes()))
1352            || presented
1353                .as_deref()
1354                .zip(state.access_keys.as_ref())
1355                .is_some_and(|(p, keys)| keys.resolve(p) == Some(KeyTier::Admin));
1356        req.extensions_mut().insert(if full {
1357            AuthIdentity::Full
1358        } else {
1359            AuthIdentity::Anonymous
1360        });
1361        return next.run(req).await;
1362    }
1363
1364    // Check full-access token first.
1365    if let Some(ref full_tok) = state.token.clone().filter(|s| !s.is_empty()) {
1366        if presented
1367            .as_deref()
1368            .is_some_and(|p| constant_time_eq(p.as_bytes(), full_tok.as_bytes()))
1369        {
1370            let set_cookie = presented_bearer_or_query(&req).as_deref() == Some(full_tok.as_str());
1371            req.extensions_mut().insert(AuthIdentity::Full);
1372            let mut res = next.run(req).await;
1373            if set_cookie && is_html_response(&res) {
1374                attach_token_cookie(&mut res, full_tok, state.tls_active);
1375            }
1376            return res;
1377        }
1378    }
1379
1380    // Check role-bound tokens.
1381    if let Some(tok) = presented.as_deref() {
1382        if let Some(role_name) = role_for_token(&state.role_tokens, tok) {
1383            // Early-deny paths whose handlers use WebSocket upgrade: the WS
1384            // extraction consumes the request before the handler body runs, so
1385            // the handler-level identity check is unreachable in those cases.
1386            // Returning 403 here also removes the need for the handler to
1387            // hold a read lock just to enforce this.
1388            let path = req.uri().path();
1389            if path == "/subscribe" || path == "/watch" {
1390                return forbidden("role-bound token: this endpoint is not permitted");
1391            }
1392            req.extensions_mut()
1393                .insert(AuthIdentity::Role(role_name.clone()));
1394            return next.run(req).await;
1395        }
1396    }
1397
1398    // Check the keys.json set. `resolve` compares every key in constant time.
1399    if let Some(tier) = presented
1400        .as_deref()
1401        .zip(state.access_keys.as_ref())
1402        .and_then(|(tok, keys)| keys.resolve(tok))
1403    {
1404        let path = req.uri().path();
1405        let ws_path = path == "/subscribe" || path == "/watch";
1406        match tier {
1407            KeyTier::Admin => {
1408                let set_cookie = presented_bearer_or_query(&req);
1409                req.extensions_mut().insert(AuthIdentity::Full);
1410                let mut res = next.run(req).await;
1411                if let Some(token) = set_cookie.filter(|_| is_html_response(&res)) {
1412                    attach_token_cookie(&mut res, &token, state.tls_active);
1413                }
1414                return res;
1415            }
1416            KeyTier::Write => {
1417                // The WS extractor runs before the handler body, so these
1418                // are refused here as well as in the handlers.
1419                if ws_path {
1420                    return forbidden(WRITE_TIER_REFUSAL);
1421                }
1422                req.extensions_mut().insert(AuthIdentity::Write);
1423                return next.run(req).await;
1424            }
1425            KeyTier::Read(role_name) => {
1426                if ws_path {
1427                    return forbidden("role-bound token: this endpoint is not permitted");
1428                }
1429                // A role may carry a write scope; a read key never writes.
1430                if !read_key_may(req.method(), path) {
1431                    return forbidden(READ_TIER_REFUSAL);
1432                }
1433                req.extensions_mut().insert(AuthIdentity::Role(role_name));
1434                req.extensions_mut().insert(ReadOnlyKey);
1435                return next.run(req).await;
1436            }
1437        }
1438    }
1439
1440    // Nothing matched → 401.
1441    unauthorized()
1442}
1443
1444/// The role bound to `presented`, comparing it with every role token in
1445/// constant time (a hash-map lookup would stop at the first difference).
1446fn role_for_token<'a>(
1447    role_tokens: &'a HashMap<String, String>,
1448    presented: &str,
1449) -> Option<&'a String> {
1450    let mut found = None;
1451    for (token, role) in role_tokens {
1452        if constant_time_eq(presented.as_bytes(), token.as_bytes()) {
1453            found = Some(role);
1454        }
1455    }
1456    found
1457}
1458
1459/// Whether a read-tier key may make this request at all. An allowlist, so a
1460/// route added later is refused to read keys until someone decides otherwise:
1461/// reads (`GET`), the two `POST` routes that carry a read (`/query`, whose
1462/// handler refuses a write statement, and `/find_similar`), and the `/tx/*`
1463/// routes: a read key may hold a transaction, whose handler refuses a write.
1464fn read_key_may(method: &Method, path: &str) -> bool {
1465    match *method {
1466        Method::GET | Method::HEAD => true,
1467        Method::POST => path == "/query" || path == "/find_similar" || path.starts_with("/tx/"),
1468        Method::DELETE => path.starts_with("/tx/"),
1469        _ => false,
1470    }
1471}
1472
1473/// The 403 detail a read-tier key gets on a write.
1474pub(crate) const READ_TIER_REFUSAL: &str = "read-tier key: writes are not permitted";
1475
1476fn request_token(req: &Request) -> Option<String> {
1477    presented_bearer_or_query(req).or_else(|| presented_cookie(req))
1478}
1479
1480fn presented_bearer_or_query(req: &Request) -> Option<String> {
1481    if let Some(header) = req
1482        .headers()
1483        .get(header::AUTHORIZATION)
1484        .and_then(|v| v.to_str().ok())
1485    {
1486        if let Some(value) = bearer_token(header) {
1487            return Some(value.to_string());
1488        }
1489    }
1490    query_param(req.uri().query().unwrap_or(""), "token")
1491}
1492
1493fn presented_cookie(req: &Request) -> Option<String> {
1494    let header = req.headers().get(header::COOKIE)?.to_str().ok()?;
1495    cookie_named(header, TOKEN_COOKIE).map(str::to_string)
1496}
1497
1498fn cookie_named<'a>(header: &'a str, name: &str) -> Option<&'a str> {
1499    for part in header.split(';') {
1500        let part = part.trim();
1501        let Some((k, v)) = part.split_once('=') else {
1502            continue;
1503        };
1504        if k.trim() == name {
1505            return Some(v.trim());
1506        }
1507    }
1508    None
1509}
1510
1511fn is_html_response(res: &Response) -> bool {
1512    res.headers()
1513        .get(header::CONTENT_TYPE)
1514        .and_then(|v| v.to_str().ok())
1515        .is_some_and(|ct| {
1516            ct.split(';')
1517                .next()
1518                .unwrap_or("")
1519                .trim()
1520                .eq_ignore_ascii_case("text/html")
1521        })
1522}
1523
1524fn attach_token_cookie(res: &mut Response, token: &str, secure: bool) {
1525    let secure_attr = if secure { "; Secure" } else { "" };
1526    let value = format!("{TOKEN_COOKIE}={token}; Path=/; SameSite=Lax; HttpOnly{secure_attr}");
1527    if let Ok(hv) = HeaderValue::from_str(&value) {
1528        res.headers_mut().insert(header::SET_COOKIE, hv);
1529    }
1530}
1531
1532fn bearer_token(header: &str) -> Option<&str> {
1533    let (scheme, value) = header.split_once(' ')?;
1534    if scheme.eq_ignore_ascii_case("Bearer") {
1535        Some(value.trim())
1536    } else {
1537        None
1538    }
1539}
1540
1541fn query_param(query: &str, key: &str) -> Option<String> {
1542    for pair in query.split('&') {
1543        if pair.is_empty() {
1544            continue;
1545        }
1546        match pair.split_once('=') {
1547            Some((k, v)) if k == key => return percent_decode_plus(v),
1548            None if pair == key => return Some(String::new()),
1549            _ => {}
1550        }
1551    }
1552    None
1553}
1554
1555/// `application/x-www-form-urlencoded`: `+` is space, `%HH` is a byte.
1556fn percent_decode_plus(s: &str) -> Option<String> {
1557    let bytes = s.as_bytes();
1558    let mut out = Vec::with_capacity(bytes.len());
1559    let mut i = 0;
1560    while i < bytes.len() {
1561        match bytes[i] {
1562            b'+' => {
1563                out.push(b' ');
1564                i += 1;
1565            }
1566            b'%' => {
1567                if i + 2 >= bytes.len() {
1568                    return None;
1569                }
1570                let hi = from_hex(bytes[i + 1])?;
1571                let lo = from_hex(bytes[i + 2])?;
1572                out.push((hi << 4) | lo);
1573                i += 3;
1574            }
1575            c => {
1576                out.push(c);
1577                i += 1;
1578            }
1579        }
1580    }
1581    String::from_utf8(out).ok()
1582}
1583
1584fn from_hex(b: u8) -> Option<u8> {
1585    match b {
1586        b'0'..=b'9' => Some(b - b'0'),
1587        b'a'..=b'f' => Some(b - b'a' + 10),
1588        b'A'..=b'F' => Some(b - b'A' + 10),
1589        _ => None,
1590    }
1591}
1592
1593fn unauthorized() -> Response {
1594    (
1595        StatusCode::UNAUTHORIZED,
1596        Json(json!({"error": "unauthorized"})),
1597    )
1598        .into_response()
1599}
1600
1601/// The 403 detail a write-tier key gets on an operator route.
1602pub(crate) const WRITE_TIER_REFUSAL: &str = "write-tier token: route not allowed";
1603
1604/// 403 for a write-tier key on a route that is for operators.
1605pub(crate) fn refuse_write_tier(identity: &AuthIdentity) -> Option<Response> {
1606    matches!(identity, AuthIdentity::Write).then(|| forbidden(WRITE_TIER_REFUSAL))
1607}
1608
1609/// 403 response for role-bound token operations that are denied in v1.
1610fn forbidden(detail: &str) -> Response {
1611    (StatusCode::FORBIDDEN, Json(json!({"error": detail}))).into_response()
1612}
1613
1614/// Convert a `mask_for_role` error into an HTTP response.
1615///
1616/// - `GraphError::Corrupt` → 500: the roles sidecar is poisoned; the server
1617///   refuses all role-token requests until the file is fixed and the DB is
1618///   re-opened.  Full-access tokens are unaffected.
1619/// - `GraphError::KeyNotFound` with a `role:` prefix → 401: the token is bound
1620///   to a role that does not exist in the DB at this request time.
1621fn role_mask_err(e: GraphError) -> Response {
1622    match e {
1623        GraphError::Corrupt { detail } => (
1624            StatusCode::INTERNAL_SERVER_ERROR,
1625            Json(json!({"error": format!("roles misconfigured: {detail}")})),
1626        )
1627            .into_response(),
1628        GraphError::KeyNotFound { key } if key.starts_with("role:") => unauthorized(),
1629        other => graph_err(other),
1630    }
1631}
1632
1633/// Convert a role-scoped write's error into an HTTP response: a role the
1634/// store no longer defines is 401, as it is for a read (`role_mask_err`) and
1635/// over Bolt; anything else is [`graph_err`]. Unlike `role_mask_err`, a
1636/// `Corrupt` keeps `graph_err`'s text, since on the write path it need not be
1637/// about roles.
1638fn role_write_err(e: GraphError) -> Response {
1639    match e {
1640        GraphError::KeyNotFound { key } if key.starts_with("role:") => unauthorized(),
1641        other => graph_err(other),
1642    }
1643}
1644
1645fn err_response(detail: impl Into<String>) -> Response {
1646    (
1647        StatusCode::BAD_REQUEST,
1648        Json(json!({"error": detail.into()})),
1649    )
1650        .into_response()
1651}
1652
1653/// 503 response for a store held by another process's write lock.
1654///
1655/// `Busy` is transient and nothing was written, so retrying is always safe.
1656/// 503 plus `Retry-After` is what standard client retry middleware keys on;
1657/// a 4xx would tell the client the request itself was malformed.
1658fn busy_response(detail: String) -> Response {
1659    (
1660        StatusCode::SERVICE_UNAVAILABLE,
1661        [(header::RETRY_AFTER, HeaderValue::from_static("1"))],
1662        Json(json!({ "error": detail })),
1663    )
1664        .into_response()
1665}
1666
1667fn graph_err(e: GraphError) -> Response {
1668    match e {
1669        // §4.3: role-scoped write denials map to 403 with the verbatim reason
1670        // string (Display delegates to reason, so .to_string() == reason).
1671        GraphError::RoleWriteDenied { reason } => forbidden(&reason),
1672        // Ledger row 12. `role_mask_err` already answered 500 for `Corrupt`
1673        // while this catch-all answered 400, so one damaged store answered
1674        // differently depending on whether the role-mask memo was warm. A
1675        // corrupt store is a server-side condition: 400 tells a caller to change
1676        // an input, and no input they can send will help. The message is
1677        // deliberately `to_string()` — identical to what the catch-all produced
1678        // — so only the status moves.
1679        corrupt @ GraphError::Corrupt { .. } => (
1680            StatusCode::INTERNAL_SERVER_ERROR,
1681            Json(json!({ "error": corrupt.to_string() })),
1682        )
1683            .into_response(),
1684        GraphError::QueryError { detail } | GraphError::IngestError { detail } => {
1685            err_response(detail)
1686        }
1687        // Transient: another process holds the cross-process write lock.
1688        busy @ GraphError::Busy { .. } => busy_response(busy.to_string()),
1689        // The data directory is not writable by the server's user. That is the
1690        // operator's to fix and no request can change it, so it is a 500, not
1691        // the 400 the catch-all would give. The full message (paths, uids) goes
1692        // to the log; the client gets a fixed one.
1693        GraphError::Unwritable { path, detail, .. } => {
1694            tracing::error!(path = %path, "{detail}");
1695            (
1696                StatusCode::INTERNAL_SERVER_ERROR,
1697                Json(json!({ "error": GraphError::UNWRITABLE_CLIENT_MESSAGE })),
1698            )
1699                .into_response()
1700        }
1701        // A unique-constraint clash is a conflict with stored state, like a
1702        // duplicate key. The text names the constraint and field only.
1703        clash @ GraphError::ConstraintViolation { .. } => (
1704            StatusCode::CONFLICT,
1705            Json(json!({ "error": clash.to_string(), "code": "constraint" })),
1706        )
1707            .into_response(),
1708        // A value over the store's size limit: the request is too large for
1709        // what it stores, not malformed.
1710        big @ GraphError::ValueTooLarge { .. } => (
1711            StatusCode::PAYLOAD_TOO_LARGE,
1712            Json(json!({ "error": big.to_string() })),
1713        )
1714            .into_response(),
1715        other => err_response(other.to_string()),
1716    }
1717}
1718
1719fn key_not_found(key: String) -> Response {
1720    (
1721        StatusCode::NOT_FOUND,
1722        Json(json!({"error": GraphError::KeyNotFound { key }.to_string()})),
1723    )
1724        .into_response()
1725}
1726
1727fn conflict_response(key: String) -> Response {
1728    (
1729        StatusCode::CONFLICT,
1730        Json(json!({"error": GraphError::DuplicateKey { key }.to_string()})),
1731    )
1732        .into_response()
1733}
1734
1735fn json_ok(value: Js) -> Response {
1736    (StatusCode::OK, Json(value)).into_response()
1737}
1738
1739fn ingest_options(v: Option<&Js>) -> Result<IngestOptions, String> {
1740    let Some(v) = v else {
1741        return Ok(IngestOptions::default());
1742    };
1743    if v.is_null() {
1744        return Ok(IngestOptions::default());
1745    }
1746    let obj = v
1747        .as_object()
1748        .ok_or_else(|| "options must be an object".to_string())?;
1749    let mut opts = IngestOptions::default();
1750    if let Some(kf) = obj.get("key_field") {
1751        opts.key_field = kf
1752            .as_str()
1753            .ok_or_else(|| "options.key_field must be a string".to_string())?
1754            .to_string();
1755    }
1756    if let Some(fk) = obj.get("auto_fk") {
1757        if fk == &Js::Bool(false) || fk.as_str() == Some("off") {
1758            opts.auto_fk = AutoFk::Off;
1759        } else if let Some(m) = fk.as_object() {
1760            let suf = m
1761                .get("suffix")
1762                .and_then(Js::as_str)
1763                .ok_or_else(|| "options.auto_fk.suffix must be a string".to_string())?;
1764            opts.auto_fk = AutoFk::Auto {
1765                suffix: suf.to_string(),
1766            };
1767        } else {
1768            return Err("options.auto_fk must be false, \"off\", or {suffix}".into());
1769        }
1770    }
1771    Ok(opts)
1772}
1773
1774/// Format a `ResultSet` as Arrow IPC or JSON depending on `format`.
1775fn format_query_result(rs: ResultSet, format: &str) -> Response {
1776    match format {
1777        "" => match to_ipc_bytes(&rs) {
1778            Ok(bytes) => (
1779                StatusCode::OK,
1780                [(header::CONTENT_TYPE, "application/vnd.apache.arrow.stream")],
1781                bytes,
1782            )
1783                .into_response(),
1784            Err(e) => err_response(e),
1785        },
1786        "json" => json_ok(result_set_json(&rs)),
1787        other => err_response(format!("unknown format: {other}")),
1788    }
1789}
1790
1791async fn query(
1792    State(state): State<AppState>,
1793    Extension(identity): Extension<AuthIdentity>,
1794    read_only_key: Option<Extension<ReadOnlyKey>>,
1795    Query(qs): Query<BTreeMap<String, String>>,
1796    Json(body): Json<Js>,
1797) -> Response {
1798    let cypher = match body.get("cypher").and_then(Js::as_str) {
1799        Some(s) => s.to_string(),
1800        None => return err_response("missing cypher"),
1801    };
1802    let params = match params_from_json(body.get("params")) {
1803        Ok(p) => p,
1804        Err(e) => return err_response(e),
1805    };
1806    let format = qs.get("format").map(String::as_str).unwrap_or("");
1807
1808    // Optional time-travel: `as_of` is a 0-based WAL commit index. The query
1809    // runs against the graph as it existed at that commit (read-only).
1810    let as_of = match body.get("as_of") {
1811        None | Some(Js::Null) => None,
1812        Some(v) => match v.as_u64() {
1813            Some(n) => Some(n),
1814            None => return err_response("as_of must be a non-negative integer commit index"),
1815        },
1816    };
1817
1818    // Parse client-supplied mask (optional array of node keys).
1819    let mask_keys: Option<Vec<String>> = match body.get("mask") {
1820        None | Some(Js::Null) => None,
1821        Some(Js::Array(arr)) => {
1822            let mut keys = Vec::with_capacity(arr.len());
1823            for v in arr {
1824                match v.as_str() {
1825                    Some(s) => keys.push(s.to_string()),
1826                    None => return err_response("mask must be an array of strings"),
1827                }
1828            }
1829            Some(keys)
1830        }
1831        Some(_) => return err_response("mask must be an array of strings"),
1832    };
1833
1834    // The second visibility axis. `namespace` is a leg that **intersects**
1835    // whatever restriction the identity and the body already impose — a role
1836    // token's binding, a client mask, both — so it can only narrow. A role bound
1837    // to namespaces honours them with no `namespace` in the body; one outside
1838    // the binding is the empty intersection, never the union.
1839    let namespace = match namespace_arg(body.get("namespace")) {
1840        Ok(n) => n,
1841        Err(e) => return err_response(e),
1842    };
1843
1844    // Stub mode discloses node existence, which is exactly the question an
1845    // as-of read is asking. The two do not compose.
1846    if as_of.is_some()
1847        && body
1848            .get("stub_hidden")
1849            .and_then(|v| v.as_bool())
1850            .unwrap_or(false)
1851    {
1852        return err_response("as_of (time-travel) does not compose with stub_hidden");
1853    }
1854
1855    // Role token: write Cypher routes to query_write_authz (scope + mask
1856    // resolved under the write lock, §5 discipline).  Read Cypher uses the
1857    // lock-free epoch snapshot path unchanged.
1858    if let AuthIdentity::Role(ref role_name) = identity {
1859        let is_write = match is_write_query(&cypher) {
1860            Ok(b) => b,
1861            Err(e) => return err_response(e),
1862        };
1863        if is_write && read_only_key.is_some() {
1864            return forbidden(READ_TIER_REFUSAL);
1865        }
1866        if as_of.is_some() && is_write {
1867            return err_response("as_of (time-travel) queries are read-only");
1868        }
1869        // `mask` and `namespace` are read guards: both documented as making the
1870        // request a read, and the full-token branch below enforces that by
1871        // routing anything carrying either to `query_masked`, which refuses a
1872        // write. The role branch used to check `is_write` first and never look
1873        // at them, so a client sending a write *with* a namespace as its guard
1874        // got the write executed instead of the 400 the docs promise. Not a
1875        // widening — `check_single_op_authz` still gates the write — but the
1876        // guard the caller asked for was not applied.
1877        if is_write && (mask_keys.is_some() || namespace.is_some()) {
1878            return (
1879                StatusCode::BAD_REQUEST,
1880                Json(json!({"error": "masked queries are read-only"})),
1881            )
1882                .into_response();
1883        }
1884        if is_write {
1885            let role = role_name.clone();
1886            let cypher_c = cypher.clone();
1887            let params_c = params.clone();
1888            let db = state.db.clone();
1889            return match blocking_write_with(
1890                move || db.write().query_write_authz(&role, &cypher_c, &params_c),
1891                role_write_err,
1892            )
1893            .await
1894            {
1895                Ok(rs) => format_query_result(rs, format),
1896                Err(resp) => resp,
1897            };
1898        }
1899        // Time travel: the role's keys and labels are resolved against the
1900        // graph as it was at `commit`, and a client mask intersects them
1901        // there.  The role *definition* is the current one — `roles.json` is a
1902        // sidecar and is never a WAL record, so it has no past version.
1903        if let Some(commit) = as_of {
1904            let scope = match mask_keys {
1905                Some(ref keys) => AsOfScope::RoleAndKeys(role_name, keys),
1906                None => AsOfScope::Role(role_name),
1907            };
1908            let g = state.db.read();
1909            let out = match namespace.as_deref() {
1910                Some(ns) => g.query_at_scoped_in_namespace(commit, &cypher, &params, scope, ns),
1911                None => g.query_at_scoped(commit, &cypher, &params, scope),
1912            };
1913            return match out {
1914                Ok(rs) => format_query_result(rs, format),
1915                Err(e) => role_mask_err(e),
1916            };
1917        }
1918        let snap = state.db.reader();
1919        let role_mask = match snap.mask_for_role(role_name) {
1920            Ok(m) => m,
1921            Err(e) => return role_mask_err(e),
1922        };
1923        let effective_mask = if let Some(ref keys) = mask_keys {
1924            // Client mask intersects role mask — never widens visibility.
1925            let client_mask = NodeMask::from_ids(keys.iter().filter_map(|k| snap.resolve_key(k)));
1926            role_mask.intersect(&client_mask)
1927        } else {
1928            role_mask
1929        };
1930        // And the namespace leg intersects whatever that came to, on the same
1931        // snapshot the query runs against.
1932        let effective_mask = match namespace.as_deref() {
1933            Some(ns) => match snap.mask_for_namespace(ns) {
1934                Ok(ns_mask) => effective_mask.intersect(&ns_mask),
1935                Err(e) => return graph_err(e),
1936            },
1937            None => effective_mask,
1938        };
1939        return match snap.query_masked(&cypher, &params, &effective_mask) {
1940            Ok(rs) => format_query_result(rs, format),
1941            Err(GraphError::MaskedReadOnly) => (
1942                StatusCode::BAD_REQUEST,
1943                Json(json!({"error": "masked queries are read-only"})),
1944            )
1945                .into_response(),
1946            Err(e) => graph_err(e),
1947        };
1948    }
1949
1950    // Full token: existing paths below.
1951
1952    // When a client-supplied mask is present, route to query_masked (read-only).
1953    // The mask and query_masked resolve against one snapshot so the mask and
1954    // the query see the same graph.
1955    //
1956    // `stub_hidden: true` opts into MaskMode::Stub for the mask; Cypher query
1957    // behaviour is identical in both modes (hidden nodes are excluded from
1958    // query results regardless of mode).
1959    if mask_keys.is_some() || namespace.is_some() {
1960        // Time travel: the allow-list resolves against the as-of graph, so a
1961        // key that did not exist at `commit` resolves to nothing.
1962        if let Some(commit) = as_of {
1963            let g = state.db.read();
1964            let out = match (&mask_keys, namespace.as_deref()) {
1965                (Some(keys), Some(ns)) => g.query_at_scoped_in_namespace(
1966                    commit,
1967                    &cypher,
1968                    &params,
1969                    AsOfScope::Keys(keys),
1970                    ns,
1971                ),
1972                (Some(keys), None) => {
1973                    g.query_at_scoped(commit, &cypher, &params, AsOfScope::Keys(keys))
1974                }
1975                (None, Some(ns)) => {
1976                    g.query_at_scoped(commit, &cypher, &params, AsOfScope::Namespace(ns))
1977                }
1978                (None, None) => unreachable!("one of the two is Some in this branch"),
1979            };
1980            return match out {
1981                Ok(rs) => format_query_result(rs, format),
1982                Err(e) => graph_err(e),
1983            };
1984        }
1985        let stub_hidden = body
1986            .get("stub_hidden")
1987            .and_then(|v| v.as_bool())
1988            .unwrap_or(false);
1989        let with_mode = |m: NodeMask| {
1990            if stub_hidden {
1991                m.with_mode(MaskMode::Stub)
1992            } else {
1993                m
1994            }
1995        };
1996        // The mask and the query resolve against one lock-free snapshot, so
1997        // they see the same graph without holding the read lock for the query.
1998        let snap = state.db.reader();
1999        let key_mask = mask_keys
2000            .as_ref()
2001            .map(|keys| snapshot_mask_from_keys(&snap, keys.iter().map(String::as_str)));
2002        let mask = match (key_mask, namespace.as_deref()) {
2003            (Some(keys), Some(ns)) => match snap.mask_for_namespace(ns) {
2004                Ok(ns_mask) => keys.intersect(&ns_mask),
2005                Err(e) => return graph_err(e),
2006            },
2007            (Some(keys), None) => keys,
2008            (None, Some(ns)) => match snap.mask_for_namespace(ns) {
2009                Ok(ns_mask) => ns_mask,
2010                Err(e) => return graph_err(e),
2011            },
2012            (None, None) => unreachable!("one of the two is Some in this branch"),
2013        };
2014        return masked_query_response(
2015            snap.query_masked(&cypher, &params, &with_mode(mask)),
2016            format,
2017        );
2018    }
2019
2020    // Detect write statements to dispatch to the correct lock.
2021    // Write statements (CREATE / MATCH…SET / MATCH…DELETE / MERGE) need the
2022    // write lock so mutations flow through WAL + rule engine with fsync before
2023    // the response is sent.  Read queries (MATCH … RETURN …) use the
2024    // lock-free reader (see `read_query`).
2025    let is_write = match is_write_query(&cypher) {
2026        Ok(b) => b,
2027        Err(e) => return err_response(e),
2028    };
2029
2030    if as_of.is_some() && is_write {
2031        return err_response("as_of (time-travel) queries are read-only");
2032    }
2033
2034    let rs = if is_write {
2035        let db = state.db.clone();
2036        match blocking_write(move || db.write().query_write(&cypher, &params)).await {
2037            Ok(rs) => rs,
2038            Err(resp) => return resp,
2039        }
2040    } else if let Some(commit) = as_of {
2041        match state.db.read().query_at(commit, &cypher, &params) {
2042            Ok(rs) => rs,
2043            Err(e) => return graph_err(e),
2044        }
2045    } else {
2046        match read_query(&state.db, &cypher, &params) {
2047            Ok(rs) => rs,
2048            Err(e) => return graph_err(e),
2049        }
2050    };
2051
2052    format_query_result(rs, format)
2053}
2054
2055// ── /schema/indexes and /schema/constraints (TY-11) ─────────────────────────
2056//
2057// Each route runs what the matching Cypher DDL statement runs
2058// (`GraphDb::schema_*`, `SHOW …`), so HTTP, Cypher and the CLI cannot
2059// disagree: a row has the fields and values `SHOW` gives it, a role-bound
2060// caller sees what its `SHOW` shows, and a write is refused to the callers a
2061// DDL write is refused to.
2062
2063/// The rows of `SHOW INDEXES` / `SHOW CONSTRAINTS` as objects, narrowed to
2064/// what `identity` can see exactly as the same statement through `/query` is.
2065fn schema_listing_response(state: &AppState, identity: &AuthIdentity, statement: &str) -> Response {
2066    let params = BTreeMap::new();
2067    let rs = match identity {
2068        AuthIdentity::Role(role_name) => {
2069            let snap = state.db.reader();
2070            let role_mask = match snap.mask_for_role(role_name) {
2071                Ok(m) => m,
2072                Err(e) => return role_mask_err(e),
2073            };
2074            snap.query_masked(statement, &params, &role_mask)
2075        }
2076        AuthIdentity::Full | AuthIdentity::Write => state.db.reader().query(statement, &params),
2077        AuthIdentity::Anonymous => return forbidden("this endpoint requires a token"),
2078    };
2079    let rs = match rs {
2080        Ok(rs) => rs,
2081        Err(e) => return graph_err(e),
2082    };
2083    let columns: Vec<&String> = rs.columns().iter().collect();
2084    let rows: Vec<Js> = (0..rs.len())
2085        .map(|i| {
2086            let obj: serde_json::Map<String, Js> = columns
2087                .iter()
2088                .zip(rs.row(i))
2089                .map(|(col, cell)| {
2090                    let v = cell.as_ref().map_or(Js::Null, crate::json::value_to_json);
2091                    ((*col).clone(), v)
2092                })
2093                .collect();
2094            Js::Object(obj)
2095        })
2096        .collect();
2097    json_ok(Js::Array(rows))
2098}
2099
2100async fn schema_indexes_list(
2101    State(state): State<AppState>,
2102    Extension(identity): Extension<AuthIdentity>,
2103) -> Response {
2104    schema_listing_response(&state, &identity, "SHOW INDEXES")
2105}
2106
2107async fn schema_constraints_list(
2108    State(state): State<AppState>,
2109    Extension(identity): Extension<AuthIdentity>,
2110) -> Response {
2111    schema_listing_response(&state, &identity, "SHOW CONSTRAINTS")
2112}
2113
2114/// 403 for a role-bound token on a schema write, as a DDL statement gives it.
2115const SCHEMA_ROLE_REFUSAL: &str =
2116    "role-bound token: schema statements (CREATE/DROP INDEX, CREATE/DROP \
2117     CONSTRAINT) are not permitted";
2118
2119/// The refusal a schema write gets for `identity`, if any. A read-tier key never
2120/// reaches here (the auth middleware refuses it first), but carries a role.
2121fn refuse_schema_write(identity: &AuthIdentity) -> Option<Response> {
2122    match identity {
2123        AuthIdentity::Role(_) => Some(forbidden(SCHEMA_ROLE_REFUSAL)),
2124        AuthIdentity::Anonymous => Some(forbidden("this endpoint requires a token")),
2125        AuthIdentity::Full | AuthIdentity::Write => None,
2126    }
2127}
2128
2129/// A schema write's error as HTTP, by the kind the engine decided from the
2130/// state it found: a name nothing answers to is 404, a collision with what
2131/// exists (a taken name or pair, an index a constraint backs, existing
2132/// duplicates, a view-owned property) is 409, and a malformed request is 400.
2133fn schema_err(e: SchemaError) -> Response {
2134    let status = match e.kind {
2135        SchemaErrorKind::NotFound => StatusCode::NOT_FOUND,
2136        SchemaErrorKind::Conflict => StatusCode::CONFLICT,
2137        SchemaErrorKind::Invalid => StatusCode::BAD_REQUEST,
2138        SchemaErrorKind::Other => return graph_err(e.error),
2139    };
2140    let detail = match e.error {
2141        GraphError::QueryError { detail } | GraphError::RuleInvalid { detail } => detail,
2142        other => other.to_string(),
2143    };
2144    (status, Json(json!({ "error": detail }))).into_response()
2145}
2146
2147/// The refusal for a schema body that is not an object, or carries a field
2148/// outside `allowed`, naming it.
2149fn schema_body_check(body: &Js, allowed: &[&str]) -> std::result::Result<(), String> {
2150    let obj = body
2151        .as_object()
2152        .ok_or_else(|| "the body must be a JSON object".to_string())?;
2153    match obj.keys().find(|k| !allowed.contains(&k.as_str())) {
2154        Some(unknown) => Err(format!(
2155            "unknown field '{unknown}'; this route takes: {}",
2156            allowed.join(", ")
2157        )),
2158        None => Ok(()),
2159    }
2160}
2161
2162/// A required, non-empty string field of a schema request body.
2163fn schema_text<'a>(body: &'a Js, field: &str) -> std::result::Result<&'a str, String> {
2164    match body.get(field).and_then(Js::as_str) {
2165        Some(s) if !s.is_empty() => Ok(s),
2166        _ => Err(format!(
2167            "{field} is required and must be a non-empty string"
2168        )),
2169    }
2170}
2171
2172/// An optional `name`: absent or null is `None`; otherwise it must be an
2173/// identifier, so every surface (HTTP path, Cypher `DROP INDEX name`) can
2174/// address it.
2175fn schema_name(body: &Js) -> std::result::Result<Option<&str>, String> {
2176    match body.get("name") {
2177        None | Some(Js::Null) => Ok(None),
2178        Some(v) => match v.as_str() {
2179            Some(s) if is_schema_identifier(s) => Ok(Some(s)),
2180            _ => Err(
2181                "name must be a string of letters, digits and underscores, not starting with a digit"
2182                    .to_string(),
2183            ),
2184        },
2185    }
2186}
2187
2188fn is_schema_identifier(s: &str) -> bool {
2189    let mut chars = s.chars();
2190    chars
2191        .next()
2192        .is_some_and(|c| c.is_ascii_alphabetic() || c == '_')
2193        && chars.all(|c| c.is_ascii_alphanumeric() || c == '_')
2194}
2195
2196async fn schema_index_create(
2197    State(state): State<AppState>,
2198    Extension(identity): Extension<AuthIdentity>,
2199    Json(body): Json<Js>,
2200) -> Response {
2201    if let Some(refusal) = refuse_schema_write(&identity) {
2202        return refusal;
2203    }
2204    if let Err(e) = schema_body_check(&body, &["name", "label", "property", "kind"]) {
2205        return err_response(e);
2206    }
2207    let (label, property) = match (schema_text(&body, "label"), schema_text(&body, "property")) {
2208        (Ok(l), Ok(p)) => (l.to_string(), p.to_string()),
2209        (Err(e), _) | (_, Err(e)) => return err_response(e),
2210    };
2211    let name = match schema_name(&body) {
2212        Ok(n) => n.map(str::to_string),
2213        Err(e) => return err_response(e),
2214    };
2215    let kind =
2216        match body.get("kind") {
2217            None | Some(Js::Null) => "range",
2218            Some(v) => match v.as_str() {
2219                Some(k @ ("range" | "equality")) => k,
2220                _ => return err_response(
2221                    "kind must be \"range\" or \"equality\"; full-text indexes are not created \
2222                     over this route",
2223                ),
2224            },
2225        };
2226    let kind = kind.to_string();
2227    let db = state.db.clone();
2228    let created = tokio::task::spawn_blocking(move || {
2229        let mut g = db.write();
2230        match kind.as_str() {
2231            "equality" => g.schema_create_equality_index(name.as_deref(), &label, &property),
2232            _ => g.schema_create_range_index(name.as_deref(), &label, &property),
2233        }
2234        .map(|listed| (listed, kind, label, property))
2235    })
2236    .await;
2237    match created {
2238        Ok(Ok((listed, kind, label, property))) => (
2239            StatusCode::CREATED,
2240            Json(json!({
2241                "name": listed, "type": kind, "label": label,
2242                "property": property, "state": "online"
2243            })),
2244        )
2245            .into_response(),
2246        Ok(Err(e)) => schema_err(e),
2247        Err(_) => err_response("write task panicked"),
2248    }
2249}
2250
2251async fn schema_index_drop(
2252    State(state): State<AppState>,
2253    Extension(identity): Extension<AuthIdentity>,
2254    Path(name): Path<String>,
2255) -> Response {
2256    if let Some(refusal) = refuse_schema_write(&identity) {
2257        return refusal;
2258    }
2259    let db = state.db.clone();
2260    match tokio::task::spawn_blocking(move || db.write().schema_drop_index(&name)).await {
2261        Ok(Ok(())) => StatusCode::NO_CONTENT.into_response(),
2262        Ok(Err(e)) => schema_err(e),
2263        Err(_) => err_response("write task panicked"),
2264    }
2265}
2266
2267async fn schema_constraint_create(
2268    State(state): State<AppState>,
2269    Extension(identity): Extension<AuthIdentity>,
2270    Json(body): Json<Js>,
2271) -> Response {
2272    if let Some(refusal) = refuse_schema_write(&identity) {
2273        return refusal;
2274    }
2275    if let Err(e) = schema_body_check(&body, &["name", "label", "property", "type"]) {
2276        return err_response(e);
2277    }
2278    let (label, property) = match (schema_text(&body, "label"), schema_text(&body, "property")) {
2279        (Ok(l), Ok(p)) => (l.to_string(), p.to_string()),
2280        (Err(e), _) | (_, Err(e)) => return err_response(e),
2281    };
2282    // A constraint needs a name (Cypher DDL requires one), so there is none to invent.
2283    let name = match schema_name(&body) {
2284        Ok(Some(n)) => n.to_string(),
2285        Ok(None) => return err_response("name is required for a constraint"),
2286        Err(e) => return err_response(e),
2287    };
2288    match body.get("type") {
2289        None | Some(Js::Null) => {}
2290        Some(v) if v.as_str() == Some("unique") => {}
2291        Some(_) => return err_response("type must be \"unique\", the only constraint type"),
2292    }
2293    let db = state.db.clone();
2294    let reply = json!({
2295        "name": name, "type": "unique", "label": label, "property": property
2296    });
2297    match tokio::task::spawn_blocking(move || {
2298        db.write()
2299            .schema_create_constraint(&name, &label, &property)
2300    })
2301    .await
2302    {
2303        Ok(Ok(())) => (StatusCode::CREATED, Json(reply)).into_response(),
2304        Ok(Err(e)) => schema_err(e),
2305        Err(_) => err_response("write task panicked"),
2306    }
2307}
2308
2309async fn schema_constraint_drop(
2310    State(state): State<AppState>,
2311    Extension(identity): Extension<AuthIdentity>,
2312    Path(name): Path<String>,
2313) -> Response {
2314    if let Some(refusal) = refuse_schema_write(&identity) {
2315        return refusal;
2316    }
2317    let db = state.db.clone();
2318    match tokio::task::spawn_blocking(move || db.write().schema_drop_constraint(&name)).await {
2319        Ok(Ok(())) => StatusCode::NO_CONTENT.into_response(),
2320        Ok(Err(e)) => schema_err(e),
2321        Err(_) => err_response("write task panicked"),
2322    }
2323}
2324
2325/// An unmasked, live (non-`as_of`) read query, on the lock-free reader. An
2326/// equality lookup on a declared index is served from the snapshot's own
2327/// index, so this holds for indexed stores too.
2328fn read_query(
2329    db: &SharedDb,
2330    cypher: &str,
2331    params: &BTreeMap<String, Value>,
2332) -> mushroomdb::Result<ResultSet> {
2333    db.reader().query(cypher, params)
2334}
2335
2336/// The response for a full-token masked `/query`, from either path.
2337fn masked_query_response(out: mushroomdb::Result<ResultSet>, format: &str) -> Response {
2338    match out {
2339        Ok(rs) => format_query_result(rs, format),
2340        Err(GraphError::MaskedReadOnly) => (
2341            StatusCode::BAD_REQUEST,
2342            Json(json!({"error": "masked queries are read-only"})),
2343        )
2344            .into_response(),
2345        Err(e) => graph_err(e),
2346    }
2347}
2348
2349/// [`NodeMask::from_keys`] against a snapshot: keys it does not hold are
2350/// ignored, and the mode is `Omit`.
2351fn snapshot_mask_from_keys<'a>(
2352    snap: &ReaderSnapshot,
2353    keys: impl IntoIterator<Item = &'a str>,
2354) -> NodeMask {
2355    NodeMask::from_ids(keys.into_iter().filter_map(|k| snap.resolve_key(k)))
2356}
2357
2358/// The client mask a full-token `/node/*` route reads from `mask=` and
2359/// `stub_hidden=`, resolved against `snap`.
2360fn snapshot_client_mask(snap: &ReaderSnapshot, mask_param: &str, stub_hidden: bool) -> NodeMask {
2361    let keys = mask_param
2362        .split(',')
2363        .map(str::trim)
2364        .filter(|s| !s.is_empty());
2365    let m = snapshot_mask_from_keys(snap, keys);
2366    if stub_hidden {
2367        m.with_mode(MaskMode::Stub)
2368    } else {
2369        m
2370    }
2371}
2372
2373/// `GET /stats` — store-wide counts, including the `namespaces` roster.
2374///
2375/// Role tokens are denied, as they always were, and that is also what settles
2376/// the namespace roster on this surface: the roster names every namespace and
2377/// its live count, which is a list of other tenants, so narrowing it for a role
2378/// token would be a second answer to a question this route never answers for one
2379/// at all. A full-access token sees the whole roster; a tenant-scoped client gets
2380/// 403 here and asks MCP `stats` with its `role`, which narrows.
2381async fn stats(
2382    State(state): State<AppState>,
2383    Extension(identity): Extension<AuthIdentity>,
2384) -> Response {
2385    // v1: deny role tokens — raw counts leak graph size beyond the role's
2386    // subgraph, and `namespaces` would name every other tenant.
2387    if let AuthIdentity::Role(_) = identity {
2388        return forbidden("role-bound token: /stats requires a full-access token");
2389    }
2390    if let Some(refusal) = refuse_write_tier(&identity) {
2391        return refusal;
2392    }
2393    let snap = {
2394        let g = state.db.read();
2395        g.stats()
2396    };
2397    match serde_json::to_value(&snap) {
2398        Ok(v) => json_ok(v),
2399        Err(e) => err_response(e.to_string()),
2400    }
2401}
2402
2403/// The continuous-backup counters `GET /metrics` reports as
2404/// `mushroomdb_backup_*`. The serve loop registers them with
2405/// [`register_backup_metrics`] when backup is configured; without that, no
2406/// backup series is emitted at all (absent, not zero, so an alert on
2407/// `mushroomdb_backup_lag_seconds` cannot be satisfied by a store that was
2408/// never backed up).
2409#[derive(Debug, Clone)]
2410pub struct BackupMetrics {
2411    /// Failed backup steps since start.
2412    pub error_count: Arc<AtomicU64>,
2413    /// Unix seconds of the last moment everything durable was shipped; 0
2414    /// before the first.
2415    pub last_success_ts: Arc<AtomicU64>,
2416    /// Seconds the oldest durable, unshipped WAL byte has waited.
2417    pub lag_s: Arc<AtomicU64>,
2418    /// Whether the worker reports itself healthy.
2419    pub healthy: Arc<AtomicBool>,
2420    /// Whether the worker holds the bucket lease right now.
2421    pub lease_held: Arc<AtomicBool>,
2422}
2423
2424/// The Bolt listener's counters `GET /metrics` reports as
2425/// `mushroomdb_bolt_connections_*`; absent when Bolt is off.
2426#[derive(Debug, Clone)]
2427pub struct BoltMetrics {
2428    /// Connections being served now.
2429    pub open: Arc<AtomicU64>,
2430    /// Connections admitted since the listener started.
2431    pub accepted: Arc<AtomicU64>,
2432}
2433
2434static BOLT_METRICS: std::sync::Mutex<Option<BoltMetrics>> = std::sync::Mutex::new(None);
2435
2436/// Report `metrics` on `GET /metrics` from now on. Like
2437/// [`register_backup_metrics`] this is per process: one process serves one
2438/// store's metrics.
2439pub fn register_bolt_metrics(metrics: BoltMetrics) {
2440    *BOLT_METRICS.lock().unwrap_or_else(PoisonError::into_inner) = Some(metrics);
2441}
2442
2443fn registered_bolt_metrics() -> Option<BoltSample> {
2444    let guard = BOLT_METRICS.lock().unwrap_or_else(PoisonError::into_inner);
2445    guard.as_ref().map(|m| BoltSample {
2446        open: m.open.load(Ordering::Acquire),
2447        accepted_total: m.accepted.load(Ordering::Acquire),
2448    })
2449}
2450
2451/// One reading of the Bolt counters.
2452struct BoltSample {
2453    open: u64,
2454    accepted_total: u64,
2455}
2456
2457static BACKUP_METRICS: std::sync::Mutex<Option<BackupMetrics>> = std::sync::Mutex::new(None);
2458
2459/// Report `metrics` on `GET /metrics` from now on. Replaces an earlier call.
2460///
2461/// The registration is process-wide: one process serves one store's metrics,
2462/// as `mushroomdb serve` does. Serving several stores from one process would
2463/// need these threaded through `AppState` instead.
2464pub fn register_backup_metrics(metrics: BackupMetrics) {
2465    *BACKUP_METRICS
2466        .lock()
2467        .unwrap_or_else(PoisonError::into_inner) = Some(metrics);
2468}
2469
2470fn registered_backup_metrics() -> Option<BackupSample> {
2471    let guard = BACKUP_METRICS
2472        .lock()
2473        .unwrap_or_else(PoisonError::into_inner);
2474    guard.as_ref().map(|m| BackupSample {
2475        upload_errors_total: m.error_count.load(Ordering::Acquire),
2476        last_success_ts: m.last_success_ts.load(Ordering::Acquire),
2477        lag_s: m.lag_s.load(Ordering::Acquire),
2478        healthy: m.healthy.load(Ordering::Acquire),
2479        lease_held: m.lease_held.load(Ordering::Acquire),
2480    })
2481}
2482
2483fn unix_seconds_now() -> u64 {
2484    std::time::SystemTime::now()
2485        .duration_since(std::time::UNIX_EPOCH)
2486        .map_or(0, |d| d.as_secs())
2487}
2488
2489/// One reading of the backup counters.
2490struct BackupSample {
2491    upload_errors_total: u64,
2492    last_success_ts: u64,
2493    lag_s: u64,
2494    healthy: bool,
2495    lease_held: bool,
2496}
2497
2498/// Everything the text exposition reports, read once.
2499struct PrometheusSample {
2500    nodes_live: u64,
2501    nodes_tombstoned: u64,
2502    edges_live: u64,
2503    commits_total: u64,
2504    wal_size_bytes: Option<u64>,
2505    rss_bytes: Option<u64>,
2506    process_start_time_seconds: u64,
2507    slow_queries_total: u64,
2508    transactions_open: u64,
2509    bolt: Option<BoltSample>,
2510    backup: Option<BackupSample>,
2511}
2512
2513/// Whether `Accept` asks for the Prometheus text format rather than JSON.
2514///
2515/// JSON is the default: a caller with no `Accept`, `*/*`, a browser's
2516/// `text/html,...` or a client that lists `application/json` among others
2517/// (`application/json, text/plain, */*`) gets JSON. Only a caller that names
2518/// `text/plain` or `application/openmetrics-text` and not JSON gets text, which
2519/// is what a Prometheus scraper sends. Quality values are not weighed.
2520fn wants_prometheus_text(headers: &HeaderMap) -> bool {
2521    let mut text = false;
2522    for value in headers.get_all(header::ACCEPT) {
2523        let Ok(value) = value.to_str() else { continue };
2524        for range in value.split(',') {
2525            let media = range
2526                .split(';')
2527                .next()
2528                .unwrap_or("")
2529                .trim()
2530                .to_ascii_lowercase();
2531            match media.as_str() {
2532                "application/json" => return false,
2533                "text/plain" | "application/openmetrics-text" => text = true,
2534                _ => {}
2535            }
2536        }
2537    }
2538    text
2539}
2540
2541/// One metric family in the text exposition format, version 0.0.4.
2542fn write_family(out: &mut String, name: &str, kind: &str, help: &str, value: impl Display) {
2543    let _ = write!(
2544        out,
2545        "# HELP {name} {help}\n# TYPE {name} {kind}\n{name} {value}\n"
2546    );
2547}
2548
2549/// The Prometheus text exposition (version 0.0.4), written by hand: no
2550/// labels, so no cardinality to manage, and a gauge whose source is
2551/// unavailable on this platform (`rss_bytes`) is left out rather than
2552/// reported as 0.
2553fn format_prometheus(s: &PrometheusSample) -> String {
2554    let mut out = String::with_capacity(2048);
2555    write_family(
2556        &mut out,
2557        "mushroomdb_nodes_live",
2558        "gauge",
2559        "Live (non-tombstoned) nodes.",
2560        s.nodes_live,
2561    );
2562    write_family(
2563        &mut out,
2564        "mushroomdb_nodes_tombstoned",
2565        "gauge",
2566        "Soft-deleted nodes not yet compacted away.",
2567        s.nodes_tombstoned,
2568    );
2569    write_family(
2570        &mut out,
2571        "mushroomdb_edges_live",
2572        "gauge",
2573        "Edges in the store.",
2574        s.edges_live,
2575    );
2576    write_family(
2577        &mut out,
2578        "mushroomdb_commits_total",
2579        "counter",
2580        "Commits since the store was created (the commit sequence number).",
2581        s.commits_total,
2582    );
2583    if let Some(bytes) = s.wal_size_bytes {
2584        write_family(
2585            &mut out,
2586            "mushroomdb_wal_size_bytes",
2587            "gauge",
2588            "On-disk size of the write-ahead log in bytes.",
2589            bytes,
2590        );
2591    }
2592    if let Some(bytes) = s.rss_bytes {
2593        write_family(
2594            &mut out,
2595            "mushroomdb_rss_bytes",
2596            "gauge",
2597            "Resident set size of the server process in bytes.",
2598            bytes,
2599        );
2600    }
2601    write_family(
2602        &mut out,
2603        "mushroomdb_process_start_time_seconds",
2604        "gauge",
2605        "Start time of the server as seconds since the Unix epoch.",
2606        s.process_start_time_seconds,
2607    );
2608    write_family(
2609        &mut out,
2610        "mushroomdb_slow_queries_total",
2611        "counter",
2612        "Read queries that met the slow-query threshold since start.",
2613        s.slow_queries_total,
2614    );
2615    write_family(
2616        &mut out,
2617        "mushroomdb_http_transactions_open",
2618        "gauge",
2619        "Open interactive HTTP transactions.",
2620        s.transactions_open,
2621    );
2622    if let Some(b) = &s.bolt {
2623        write_family(
2624            &mut out,
2625            "mushroomdb_bolt_connections_open",
2626            "gauge",
2627            "Bolt connections being served now.",
2628            b.open,
2629        );
2630        write_family(
2631            &mut out,
2632            "mushroomdb_bolt_connections_accepted_total",
2633            "counter",
2634            "Bolt connections admitted since the listener started.",
2635            b.accepted_total,
2636        );
2637    }
2638    if let Some(b) = &s.backup {
2639        write_family(
2640            &mut out,
2641            "mushroomdb_backup_upload_errors_total",
2642            "counter",
2643            "Failed continuous-backup steps since start.",
2644            b.upload_errors_total,
2645        );
2646        write_family(
2647            &mut out,
2648            "mushroomdb_backup_last_success_timestamp_seconds",
2649            "gauge",
2650            "Unix time of the last moment all durable data was shipped; 0 before the first.",
2651            b.last_success_ts,
2652        );
2653        write_family(
2654            &mut out,
2655            "mushroomdb_backup_lag_seconds",
2656            "gauge",
2657            "Seconds the oldest durable, unshipped WAL byte has waited; 0 when all is shipped.",
2658            b.lag_s,
2659        );
2660        write_family(
2661            &mut out,
2662            "mushroomdb_backup_healthy",
2663            "gauge",
2664            "1 while the backup worker reports itself healthy, else 0.",
2665            u8::from(b.healthy),
2666        );
2667        write_family(
2668            &mut out,
2669            "mushroomdb_backup_lease_held",
2670            "gauge",
2671            "1 while this instance holds the backup bucket lease, else 0.",
2672            u8::from(b.lease_held),
2673        );
2674    }
2675    out
2676}
2677
2678async fn metrics(
2679    State(state): State<AppState>,
2680    Extension(identity): Extension<AuthIdentity>,
2681    headers: HeaderMap,
2682) -> Response {
2683    // Mirror /stats auth: deny role tokens — counters leak graph size.
2684    if let AuthIdentity::Role(_) = identity {
2685        return forbidden("role-bound token: /metrics requires a full-access token");
2686    }
2687    if let Some(refusal) = refuse_write_tier(&identity) {
2688        return refusal;
2689    }
2690    let snap = state.db.reader();
2691    let (nodes_live, nodes_tombstoned, edges) = snap.counts();
2692    let commit_seq = snap.commit_seq();
2693    let wal_size_bytes = snap.wal_size_bytes().ok();
2694    let slow_snap = snap.slow_query_snapshot();
2695    if wants_prometheus_text(&headers) {
2696        let text = format_prometheus(&PrometheusSample {
2697            nodes_live: nodes_live as u64,
2698            nodes_tombstoned: nodes_tombstoned as u64,
2699            edges_live: edges,
2700            commits_total: commit_seq,
2701            wal_size_bytes,
2702            rss_bytes: rss_bytes(),
2703            process_start_time_seconds: state.started_unix_s,
2704            slow_queries_total: slow_snap.count,
2705            transactions_open: state.txs.open_count() as u64,
2706            bolt: registered_bolt_metrics(),
2707            backup: registered_backup_metrics(),
2708        });
2709        return (
2710            StatusCode::OK,
2711            [
2712                (header::CONTENT_TYPE, PROMETHEUS_CONTENT_TYPE),
2713                (header::VARY, "Accept"),
2714            ],
2715            text,
2716        )
2717            .into_response();
2718    }
2719    let uptime_s = state.started_at.elapsed().as_secs();
2720    let slow_entries: Vec<Js> = slow_snap
2721        .last
2722        .iter()
2723        .map(|e| {
2724            json!({
2725                "ms": e.ms,
2726                "query": e.query,
2727                "at_commit": e.at_commit,
2728            })
2729        })
2730        .collect();
2731    let mut response = json_ok(json!({
2732        "nodes_live": nodes_live,
2733        "nodes_tombstoned": nodes_tombstoned,
2734        "edges": edges,
2735        "commit_seq": commit_seq,
2736        "wal_size_bytes": wal_size_bytes,
2737        "rss_bytes": rss_bytes(),
2738        "uptime_s": uptime_s,
2739        "slow_queries": {
2740            "threshold_ms": slow_snap.threshold_ms,
2741            "count": slow_snap.count,
2742            "last": slow_entries,
2743        },
2744    }));
2745    response
2746        .headers_mut()
2747        .insert(header::VARY, HeaderValue::from_static("Accept"));
2748    response
2749}
2750
2751/// `Content-Type` of the Prometheus text exposition.
2752const PROMETHEUS_CONTENT_TYPE: &str = "text/plain; version=0.0.4; charset=utf-8";
2753
2754/// Resident set size of the current process in bytes.
2755///
2756/// macOS: mach task_info (MACH_TASK_BASIC_INFO).
2757/// Linux: /proc/self/statm RSS pages × page size.
2758/// Other: always returns None.
2759///
2760/// Never panics; returns None on any failure.
2761fn rss_bytes() -> Option<u64> {
2762    #[cfg(target_os = "macos")]
2763    {
2764        // MACH_TASK_BASIC_INFO flavor = 20.
2765        // struct mach_task_basic_info { u64 virtual, u64 resident,
2766        //   u64 resident_max, [u32;2] user_time, [u32;2] sys_time,
2767        //   i32 policy, i32 suspend_count } = 48 bytes = 12 natural_t(u32)s.
2768        const MACH_TASK_BASIC_INFO: u32 = 20;
2769        const MACH_TASK_BASIC_INFO_COUNT: u32 = 12;
2770
2771        #[repr(C)]
2772        struct MachTaskBasicInfo {
2773            virtual_size: u64,
2774            resident_size: u64,
2775            resident_size_max: u64,
2776            user_time: [u32; 2],
2777            system_time: [u32; 2],
2778            policy: i32,
2779            suspend_count: i32,
2780        }
2781
2782        extern "C" {
2783            fn mach_task_self() -> u32;
2784            fn task_info(
2785                target_task: u32,
2786                flavor: u32,
2787                task_info_out: *mut std::ffi::c_void,
2788                task_info_cnt: *mut u32,
2789            ) -> i32;
2790        }
2791
2792        let mut info: MachTaskBasicInfo = unsafe { std::mem::zeroed() };
2793        let mut count = MACH_TASK_BASIC_INFO_COUNT;
2794        let ret = unsafe {
2795            task_info(
2796                mach_task_self(),
2797                MACH_TASK_BASIC_INFO,
2798                &mut info as *mut _ as *mut _,
2799                &mut count,
2800            )
2801        };
2802        if ret != 0 {
2803            return None; // KERN_SUCCESS = 0
2804        }
2805        return Some(info.resident_size);
2806    }
2807
2808    #[cfg(target_os = "linux")]
2809    {
2810        // /proc/self/statm: "vsize rss ..." in pages.
2811        let content = std::fs::read_to_string("/proc/self/statm").ok()?;
2812        let mut parts = content.split_whitespace();
2813        let _vsize = parts.next()?;
2814        let rss_pages: u64 = parts.next()?.parse().ok()?;
2815        let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) };
2816        if page_size <= 0 {
2817            return None;
2818        }
2819        return Some(rss_pages * page_size as u64);
2820    }
2821
2822    #[allow(unreachable_code)]
2823    None
2824}
2825
2826async fn ingest(
2827    State(state): State<AppState>,
2828    Extension(identity): Extension<AuthIdentity>,
2829    Json(body): Json<Js>,
2830) -> Response {
2831    let label = match body.get("label").and_then(Js::as_str) {
2832        Some(s) => s.to_string(),
2833        None => return err_response("missing label"),
2834    };
2835    let rows = match body.get("rows") {
2836        Some(r) => r,
2837        None => return err_response("missing rows"),
2838    };
2839    let mut converted = match json_to_rows(rows) {
2840        Ok(c) => c,
2841        Err(e) => return graph_err(e),
2842    };
2843    let opts = match ingest_options(body.get("options")) {
2844        Ok(o) => o,
2845        Err(e) => return err_response(e),
2846    };
2847    // `namespace` applies to every node this call creates.
2848    let namespace = match namespace_arg(body.get("namespace")) {
2849        Ok(n) => n,
2850        Err(e) => return err_response(e),
2851    };
2852    if let Err(e) = stamp_namespace(&mut converted.rows, namespace.as_deref()) {
2853        return err_response(e);
2854    }
2855    let taken = std::mem::take(&mut converted.rows);
2856    let edges = match body.get("edges") {
2857        None | Some(Js::Null) => Vec::new(),
2858        Some(raw) => match parse_ingest_edges(raw) {
2859            Ok(e) => e,
2860            Err(e) => return err_response(e),
2861        },
2862    };
2863    let db = state.db.clone();
2864
2865    // Role token: route to ingest_with_edges_authz (scope + mask resolved under
2866    // the write lock, §5 discipline; §7.3 — create_labels required, enforced by
2867    // the engine's per-op authz check inside commit_logged_batch).
2868    if let AuthIdentity::Role(role_name) = &identity {
2869        let role = role_name.clone();
2870        return match blocking_write(move || {
2871            db.write()
2872                .ingest_with_edges_authz(&role, &label, taken, &opts, &edges)
2873        })
2874        .await
2875        {
2876            Ok(r) => {
2877                let report = converted.into_report(r);
2878                match serde_json::to_value(&report) {
2879                    Ok(v) => json_ok(v),
2880                    Err(e) => err_response(e.to_string()),
2881                }
2882            }
2883            Err(resp) => resp,
2884        };
2885    }
2886
2887    let report =
2888        match blocking_write(move || db.write().ingest_with_edges(&label, taken, &opts, &edges))
2889            .await
2890        {
2891            Ok(r) => converted.into_report(r),
2892            Err(resp) => return resp,
2893        };
2894    match serde_json::to_value(&report) {
2895        Ok(v) => json_ok(v),
2896        Err(e) => err_response(e.to_string()),
2897    }
2898}
2899
2900/// `GET /suggest` — profile the database and return rule suggestions.
2901///
2902/// # Locking and blocking strategy
2903///
2904/// `suggest_rules_with_config` is CPU-intensive and synchronous. Running it on a
2905/// Tokio worker thread would starve the executor. This handler offloads the work to
2906/// `tokio::task::spawn_blocking`, which uses the blocking thread-pool. The
2907/// `std::sync::RwLock` read guard is acquired and held inside the blocking task —
2908/// reads don't block other reads; writes wait for the guard to drop. The global
2909/// budget (`SuggestConfig::global_budget_ms`, default 5 s) caps lock-hold time.
2910async fn suggest(
2911    State(state): State<AppState>,
2912    Extension(identity): Extension<AuthIdentity>,
2913) -> Response {
2914    // v1: deny role tokens — suggest scans the full graph and would reveal
2915    // existence of nodes outside the role's mask.
2916    if let AuthIdentity::Role(_) = identity {
2917        return forbidden("role-bound token: /suggest requires a full-access token");
2918    }
2919    let db = state.db.clone();
2920    match tokio::task::spawn_blocking(move || {
2921        let config = SuggestConfig::default();
2922        db.read()
2923            .suggest_rules_with_config(&config, SUGGEST_DEFAULT_SEED)
2924    })
2925    .await
2926    {
2927        Ok(report) => json_ok(serde_json::to_value(&report).unwrap_or_else(|_| json!({}))),
2928        Err(_) => err_response("suggest task panicked"),
2929    }
2930}
2931
2932async fn create_rule(
2933    State(state): State<AppState>,
2934    Extension(identity): Extension<AuthIdentity>,
2935    Json(body): Json<Js>,
2936) -> Response {
2937    if let AuthIdentity::Role(_) = identity {
2938        return forbidden("role-bound token: writes are not permitted");
2939    }
2940    let def = match rule_def_from_json(body) {
2941        Ok(d) => d,
2942        Err(e) => return err_response(e),
2943    };
2944    let name = def.name.clone();
2945    let db = state.db.clone();
2946    if let Err(resp) = blocking_write(move || db.write().create_rule(def)).await {
2947        return resp;
2948    }
2949    // A corpus too large to index in one commit leaves the rule installed but
2950    // deriving nothing, so the route says "accepted", not "done", and hands
2951    // back the progress the caller can poll on `GET /stats`.
2952    //
2953    // Read under a *separate* lock from the write above, so a concurrent pump
2954    // can finish the build in between and this route then answers 200 for a
2955    // create that really did defer. Benign — 200 means "the edges are there",
2956    // which by then they are — and the caller's contract is to poll `GET /stats`
2957    // either way.
2958    let building = state
2959        .db
2960        .read()
2961        .builds_in_progress()
2962        .into_iter()
2963        .find(|b| b.rule == name);
2964    match building {
2965        Some(b) => (
2966            StatusCode::ACCEPTED,
2967            Json(json!({"rule": name, "building": {"indexed": b.indexed, "total": b.total}})),
2968        )
2969            .into_response(),
2970        None => json_ok(json!({"ok": true, "name": name})),
2971    }
2972}
2973
2974async fn explain(
2975    State(state): State<AppState>,
2976    Extension(identity): Extension<AuthIdentity>,
2977    Query(qs): Query<BTreeMap<String, String>>,
2978) -> Response {
2979    // v1: deny role tokens — explain reveals hidden-node linkage through rules.
2980    if let AuthIdentity::Role(_) = identity {
2981        return forbidden(
2982            "role-bound token: /explain requires a full-access token \
2983             (v1: explain may reveal hidden-node linkage; revisit when stubs land)",
2984        );
2985    }
2986    let a = match qs.get("a") {
2987        Some(s) if !s.is_empty() => s.clone(),
2988        _ => return err_response("missing query param a"),
2989    };
2990    let b = match qs.get("b") {
2991        Some(s) if !s.is_empty() => s.clone(),
2992        _ => return err_response("missing query param b"),
2993    };
2994    let out = {
2995        let g = state.db.read();
2996        g.explain(&a, &b)
2997    };
2998    match out {
2999        Ok(v) => match serde_json::to_value(&v) {
3000            Ok(j) => json_ok(j),
3001            Err(e) => err_response(e.to_string()),
3002        },
3003        Err(e) => graph_err(e),
3004    }
3005}
3006
3007async fn node_info(
3008    State(state): State<AppState>,
3009    Extension(identity): Extension<AuthIdentity>,
3010    Path(key): Path<String>,
3011    Query(qs): Query<BTreeMap<String, String>>,
3012) -> Response {
3013    if let AuthIdentity::Role(ref role_name) = identity {
3014        // Role-token path: hard-coded Omit mode; hidden keys are indistinguishable
3015        // from absent keys.  `stub_hidden` query param is silently ignored here —
3016        // role paths must NEVER produce stubs (RBAC invariant).
3017        let snap = state.db.reader();
3018        let role_mask = match snap.mask_for_role(role_name) {
3019            Ok(m) => m,
3020            Err(e) => return role_mask_err(e),
3021        };
3022        if !snap
3023            .resolve_key(&key)
3024            .is_some_and(|id| role_mask.contains_id(id))
3025        {
3026            return key_not_found(key);
3027        }
3028        return match snap.node_info(&key) {
3029            Some(info) => json_ok(node_info_json(&info)),
3030            None => key_not_found(key),
3031        };
3032    }
3033
3034    // Full-token path: optional client mask + stub_hidden via query params.
3035    // `mask=key1,key2` — comma-separated visible keys (empty string = no mask).
3036    // `stub_hidden=true` — opt into MaskMode::Stub for this request.
3037    let mask_param = qs.get("mask").map(String::as_str).unwrap_or("").trim();
3038    if !mask_param.is_empty() {
3039        let stub_hidden = qs
3040            .get("stub_hidden")
3041            .map(|v| v == "true" || v == "1")
3042            .unwrap_or(false);
3043        let snap = state.db.reader();
3044        if let Err(e) = snap.ensure_readable() {
3045            return graph_err(e);
3046        }
3047        let mask = snapshot_client_mask(&snap, mask_param, stub_hidden);
3048        return match snap.node_info_masked(&key, &mask) {
3049            Some(mushroomdb::MaskedNodeResult::Visible(info)) => json_ok(node_info_json(&info)),
3050            Some(mushroomdb::MaskedNodeResult::Restricted) => {
3051                json_ok(crate::json::stub_node_json(&key))
3052            }
3053            None => key_not_found(key),
3054        };
3055    }
3056
3057    let snap = state.db.reader();
3058    if let Err(e) = snap.ensure_readable() {
3059        return graph_err(e);
3060    }
3061    match snap.node_info(&key) {
3062        Some(info) => json_ok(node_info_json(&info)),
3063        None => key_not_found(key),
3064    }
3065}
3066
3067async fn node_edges(
3068    State(state): State<AppState>,
3069    Extension(identity): Extension<AuthIdentity>,
3070    Path(key): Path<String>,
3071    Query(qs): Query<BTreeMap<String, String>>,
3072) -> Response {
3073    if let AuthIdentity::Role(ref role_name) = identity {
3074        // Role-token path: hard-coded Omit mode.  `stub_hidden` query param is
3075        // silently ignored — role paths must NEVER produce stubs (RBAC invariant).
3076        //
3077        // The subject check and the hidden-endpoint filter live in
3078        // `ReaderSnapshot::node_edges_scoped`, so every scoped caller gets them,
3079        // not only this route.
3080        let snap = state.db.reader();
3081        let role_mask = match snap.mask_for_role(role_name) {
3082            Ok(m) => m,
3083            Err(e) => return role_mask_err(e),
3084        };
3085        return match snap.node_edges_scoped(&key, &role_mask) {
3086            Ok(edges) => json_ok(node_edges_json(&edges)),
3087            Err(GraphError::KeyNotFound { key }) => key_not_found(key),
3088            Err(e) => graph_err(e),
3089        };
3090    }
3091
3092    // Full-token path: optional client mask + stub_hidden via query params.
3093    let mask_param = qs.get("mask").map(String::as_str).unwrap_or("").trim();
3094    if !mask_param.is_empty() {
3095        let stub_hidden = qs
3096            .get("stub_hidden")
3097            .map(|v| v == "true" || v == "1")
3098            .unwrap_or(false);
3099        let snap = state.db.reader();
3100        if let Err(e) = snap.ensure_readable() {
3101            return graph_err(e);
3102        }
3103        let mask = snapshot_client_mask(&snap, mask_param, stub_hidden);
3104        return match snap.node_edges_masked(&key, &mask) {
3105            Ok(edges) => json_ok(crate::json::masked_edges_json(&edges)),
3106            Err(GraphError::KeyNotFound { key }) => key_not_found(key),
3107            Err(e) => graph_err(e),
3108        };
3109    }
3110
3111    match state.db.reader().node_edges(&key) {
3112        Ok(edges) => json_ok(node_edges_json(&edges)),
3113        Err(GraphError::KeyNotFound { key }) => key_not_found(key),
3114        Err(e) => graph_err(e),
3115    }
3116}
3117
3118async fn neighborhood(
3119    State(state): State<AppState>,
3120    Extension(identity): Extension<AuthIdentity>,
3121    Path(key): Path<String>,
3122    Query(qs): Query<BTreeMap<String, String>>,
3123) -> Response {
3124    let depth = match resolve_neighborhood_depth(qs.get("depth").map(String::as_str)) {
3125        Ok(d) => d,
3126        Err(e) => return err_response(e),
3127    };
3128    let dir = match qs.get("dir").map(String::as_str).unwrap_or("both") {
3129        s if s.eq_ignore_ascii_case("out") => Dir::Out,
3130        s if s.eq_ignore_ascii_case("in") => Dir::In,
3131        s if s.eq_ignore_ascii_case("both") => Dir::Both,
3132        other => return err_response(format!("unknown dir: {other}")),
3133    };
3134    let edge_type_names: Option<Vec<String>> = qs.get("edge_types").map(|s| {
3135        s.split(',')
3136            .map(str::trim)
3137            .filter(|t| !t.is_empty())
3138            .map(str::to_string)
3139            .collect()
3140    });
3141    let etype_refs: Option<Vec<&str>> = edge_type_names
3142        .as_ref()
3143        .map(|v| v.iter().map(String::as_str).collect());
3144    if let AuthIdentity::Role(ref role_name) = identity {
3145        // Lock-free epoch snapshot: mask and neighborhood BFS on same frozen state
3146        // (constraint 2 — RBAC mask coherence).
3147        let snap = state.db.reader();
3148        let role_mask = match snap.mask_for_role(role_name) {
3149            Ok(m) => m,
3150            Err(e) => return role_mask_err(e),
3151        };
3152        // The scoped BFS: the subject is checked first (a hidden key answers as
3153        // an absent one), and hidden nodes are excluded from results AND cannot
3154        // be used as traversal intermediaries (never-leak invariant).
3155        return match snap.neighborhood_scoped(&key, depth, etype_refs.as_deref(), dir, &role_mask) {
3156            Ok(rs) => json_ok(result_set_json(&rs)),
3157            Err(GraphError::KeyNotFound { key }) => key_not_found(key),
3158            Err(e) => graph_err(e),
3159        };
3160    }
3161    // Full-token path: optional client mask + stub_hidden via query params.
3162    // `mask=key1,key2` — comma-separated visible keys.
3163    // `stub_hidden=true` — opt into MaskMode::Stub; hidden direct neighbours
3164    // appear as stub rows (label: null) in the result; BFS does not expand
3165    // through them in either mode.
3166    let mask_param = qs.get("mask").map(String::as_str).unwrap_or("").trim();
3167    if !mask_param.is_empty() {
3168        let stub_hidden = qs
3169            .get("stub_hidden")
3170            .map(|v| v == "true" || v == "1")
3171            .unwrap_or(false);
3172        let snap = state.db.reader();
3173        if let Err(e) = snap.ensure_readable() {
3174            return graph_err(e);
3175        }
3176        let mask = snapshot_client_mask(&snap, mask_param, stub_hidden);
3177        return match snap.neighborhood_masked(&key, depth, etype_refs.as_deref(), dir, &mask) {
3178            Some(rs) => json_ok(result_set_json(&rs)),
3179            None => key_not_found(key),
3180        };
3181    }
3182
3183    // Unmasked full-token path (no mask param).
3184    let snap = state.db.reader();
3185    if let Err(e) = snap.ensure_readable() {
3186        return graph_err(e);
3187    }
3188    match snap.neighborhood(&key, depth, etype_refs.as_deref(), dir) {
3189        Some(rs) => json_ok(result_set_json(&rs)),
3190        None => key_not_found(key),
3191    }
3192}
3193
3194/// `POST /find_similar` — vector kNN. First-class read, not `/algo/*`, so a
3195/// role token is allowed and a client `mask` intersects the role (never widens).
3196/// Default `min` is `mushroomdb::FIND_SIMILAR_DEFAULT_MIN` (0.8), as on MCP and
3197/// in the Python binding. It was 0.0 here through 0.6.
3198async fn find_similar(
3199    State(state): State<AppState>,
3200    Extension(identity): Extension<AuthIdentity>,
3201    Json(body): Json<Js>,
3202) -> Response {
3203    let field = match body.get("field").and_then(Js::as_str) {
3204        Some(s) if !s.is_empty() => s.to_string(),
3205        _ => return err_response("missing field"),
3206    };
3207    let vector = match parse_find_similar_vector(body.get("vector")) {
3208        Ok(v) => v,
3209        Err(e) => return err_response(e),
3210    };
3211    let k = match body.get("k") {
3212        None | Some(Js::Null) => 10usize,
3213        Some(v) => match v.as_u64() {
3214            Some(n) => n as usize,
3215            None => return err_response("k must be a non-negative integer"),
3216        },
3217    };
3218    let min = match body.get("min") {
3219        None | Some(Js::Null) => mushroomdb::FIND_SIMILAR_DEFAULT_MIN,
3220        Some(v) => match v.as_f64() {
3221            Some(n) => n,
3222            None => return err_response("min must be a number"),
3223        },
3224    };
3225    let label = match body.get("label").and_then(Js::as_str) {
3226        None | Some("") => None,
3227        Some(s) => Some(s.to_string()),
3228    };
3229    let mask_keys = match parse_mask_keys(body.get("mask")) {
3230        Ok(m) => m,
3231        Err(e) => return err_response(e),
3232    };
3233    let where_pred = match parse_where_body(body.get("where")) {
3234        Ok(p) => p,
3235        Err(detail) => return graph_err(GraphError::QueryError { detail }),
3236    };
3237    let exact = match body.get("exact") {
3238        None | Some(Js::Null) => false,
3239        Some(v) => match v.as_bool() {
3240            Some(b) => b,
3241            None => return err_response("exact must be a boolean"),
3242        },
3243    };
3244    let exact = exact || where_pred.is_some();
3245    let role_name = match identity {
3246        AuthIdentity::Role(r) => Some(r),
3247        AuthIdentity::Full | AuthIdentity::Write => None,
3248        // Unreachable on this route — `Anonymous` is only ever inserted for
3249        // `GET /health` — but treated as the narrowest identity rather than
3250        // the widest, so a future open endpoint cannot inherit full access by
3251        // omission.
3252        AuthIdentity::Anonymous => {
3253            return forbidden("this endpoint requires a token");
3254        }
3255    };
3256    let db = state.db.clone();
3257    match tokio::task::spawn_blocking(move || {
3258        let g = db.read();
3259        let role_mask = match role_name.as_deref() {
3260            Some(role) => Some(g.mask_for_role(role)?),
3261            None => None,
3262        };
3263        let client_mask = mask_keys
3264            .as_ref()
3265            .map(|keys| NodeMask::from_keys(&*g, keys.iter().map(String::as_str)));
3266        let effective = match (role_mask, client_mask) {
3267            (Some(role), Some(client)) => Some(role.intersect(&client)),
3268            (Some(role), None) => Some(role),
3269            (None, Some(client)) => Some(client),
3270            (None, None) => None,
3271        };
3272        g.find_similar_vector_filtered(
3273            &field,
3274            label.as_deref(),
3275            &vector,
3276            k,
3277            min,
3278            effective.as_ref(),
3279            where_pred.as_ref(),
3280            exact,
3281        )
3282    })
3283    .await
3284    {
3285        Ok(Ok(hits)) => json_ok(json!({ "hits": hits })),
3286        Ok(Err(e)) => role_mask_err(e),
3287        Err(_) => err_response("find_similar task panicked"),
3288    }
3289}
3290
3291fn refuse_nonobject_algorithm_config(body: &serde_json::Value) -> Option<Response> {
3292    (!body.is_object()).then(|| err_response("algorithm configuration must be a JSON object"))
3293}
3294
3295/// `POST /algo/pagerank` — run PageRank over the unified topology.
3296///
3297/// # Locking and blocking strategy
3298///
3299/// PageRank is CPU-intensive and synchronous. This handler offloads the work
3300/// to `tokio::task::spawn_blocking` (blocking thread-pool), over a lock-free
3301/// [`ReaderSnapshot`], so it holds no guard and never holds a writer up. The
3302/// `budget_ms` field in [`PageRankConfig`] caps its running time.
3303async fn algo_pagerank(
3304    State(state): State<AppState>,
3305    Extension(identity): Extension<AuthIdentity>,
3306    Json(body): Json<serde_json::Value>,
3307) -> Response {
3308    // v1: deny role tokens — algo endpoints scan the full graph.
3309    if let AuthIdentity::Role(_) = identity {
3310        return forbidden("role-bound token: /algo/* requires a full-access token");
3311    }
3312    if let Some(response) = refuse_nonobject_algorithm_config(&body) {
3313        return response;
3314    }
3315    let config: PageRankConfig = match serde_json::from_value(body) {
3316        Ok(c) => c,
3317        Err(e) => return err_response(format!("invalid pagerank config: {e}")),
3318    };
3319    if let Err(e) = config.validate() {
3320        return err_response(format!("invalid pagerank config: {e}"));
3321    }
3322    let db = state.db.clone();
3323    match tokio::task::spawn_blocking(move || db.reader().pagerank(&config)).await {
3324        Ok(Ok(report)) => json_ok(serde_json::to_value(&report).unwrap_or_else(|_| json!({}))),
3325        Ok(Err(e)) => graph_err(e),
3326        Err(_) => err_response("pagerank task panicked"),
3327    }
3328}
3329
3330/// `POST /algo/wcc` — weakly-connected components over the unified topology.
3331///
3332/// Mirrors `algo_pagerank`: spawn_blocking over a lock-free snapshot,
3333/// `budget_ms` in config caps running time.
3334async fn algo_wcc(
3335    State(state): State<AppState>,
3336    Extension(identity): Extension<AuthIdentity>,
3337    Json(body): Json<serde_json::Value>,
3338) -> Response {
3339    if let AuthIdentity::Role(_) = identity {
3340        return forbidden("role-bound token: /algo/* requires a full-access token");
3341    }
3342    if let Some(response) = refuse_nonobject_algorithm_config(&body) {
3343        return response;
3344    }
3345    let config: WccConfig = match serde_json::from_value(body) {
3346        Ok(c) => c,
3347        Err(e) => return err_response(format!("invalid wcc config: {e}")),
3348    };
3349    let db = state.db.clone();
3350    match tokio::task::spawn_blocking(move || db.reader().connected_components(&config)).await {
3351        Ok(Ok(report)) => json_ok(serde_json::to_value(&report).unwrap_or_else(|_| json!({}))),
3352        Ok(Err(e)) => graph_err(e),
3353        Err(_) => err_response("wcc task panicked"),
3354    }
3355}
3356
3357/// `POST /algo/degree` — degree centrality over the unified topology.
3358///
3359/// Mirrors `algo_pagerank`.
3360async fn algo_degree(
3361    State(state): State<AppState>,
3362    Extension(identity): Extension<AuthIdentity>,
3363    Json(body): Json<serde_json::Value>,
3364) -> Response {
3365    if let AuthIdentity::Role(_) = identity {
3366        return forbidden("role-bound token: /algo/* requires a full-access token");
3367    }
3368    if let Some(response) = refuse_nonobject_algorithm_config(&body) {
3369        return response;
3370    }
3371    let config: DegreeConfig = match serde_json::from_value(body) {
3372        Ok(c) => c,
3373        Err(e) => return err_response(format!("invalid degree config: {e}")),
3374    };
3375    let db = state.db.clone();
3376    match tokio::task::spawn_blocking(move || db.reader().degree_centrality(&config)).await {
3377        Ok(Ok(report)) => json_ok(serde_json::to_value(&report).unwrap_or_else(|_| json!({}))),
3378        Ok(Err(e)) => graph_err(e),
3379        Err(_) => err_response("degree task panicked"),
3380    }
3381}
3382
3383// ── Simple BatchOp-mapped endpoints ──────────────────────────────────────────
3384//
3385// These endpoints map 1:1 to BatchOp variants and route through submit_batch
3386// so concurrent writes share one WAL fsync per drain group, keeping reader
3387// p95 latency low under write bursts.  Auth checks (role tokens → 403) run
3388// before enqueue so RBAC enforcement is unchanged.
3389//
3390// Complex paths (/query Cypher writes, /ingest bulk JSON) stay on db.write()
3391// because they are multi-step operations that cannot be pre-expressed as a
3392// Vec<BatchOp> without redesigning the query executor.
3393
3394/// Parse a JSON object into a `Vec<(String, Value)>` prop list.
3395fn props_from_json_obj(v: &serde_json::Value) -> Result<Vec<(String, Value)>, String> {
3396    let obj = match v.as_object() {
3397        Some(o) => o,
3398        None => return Err("props must be a JSON object".into()),
3399    };
3400    let mut out = Vec::with_capacity(obj.len());
3401    for (k, val) in obj {
3402        if let Some(v) = json_to_value(val.clone()).map_err(|e| format!("props {k:?}: {e}"))? {
3403            out.push((k.clone(), v));
3404        }
3405    }
3406    Ok(out)
3407}
3408
3409/// `POST /nodes` — create or upsert a node via the group-commit queue.
3410///
3411/// Body: `{"label": "Person", "key": "alice", "props": {"age": 30}}`
3412async fn create_node(
3413    State(state): State<AppState>,
3414    Extension(identity): Extension<AuthIdentity>,
3415    Json(body): Json<Js>,
3416) -> Response {
3417    let label = match body.get("label").and_then(Js::as_str) {
3418        Some(s) => s.to_string(),
3419        None => return err_response("missing label"),
3420    };
3421    let key = match body.get("key").and_then(Js::as_str) {
3422        Some(s) => s.to_string(),
3423        None => return err_response("missing key"),
3424    };
3425    let mut props = match body.get("props") {
3426        None | Some(Js::Null) => vec![],
3427        Some(v) => match props_from_json_obj(v) {
3428            Ok(p) => p,
3429            Err(e) => return err_response(e),
3430        },
3431    };
3432    // `namespace` is the namespace the node is created in — the same write-once
3433    // `ns` property, named on the call instead of buried in `props`.
3434    match namespace_arg(body.get("namespace")) {
3435        Ok(None) => {}
3436        Ok(Some(ns)) => {
3437            let mut row: BTreeMap<String, Value> = props.into_iter().collect();
3438            if let Err(e) = stamp_namespace_row(&mut row, &ns) {
3439                return err_response(e);
3440            }
3441            props = row.into_iter().collect();
3442        }
3443        Err(e) => return err_response(e),
3444    }
3445    let db = state.db.clone();
3446    if let AuthIdentity::Role(role_name) = &identity {
3447        let role = role_name.clone();
3448        return match blocking_write(move || {
3449            db.submit_batch_authz(role, vec![BatchOp::InsertNode { label, key, props }])
3450        })
3451        .await
3452        {
3453            Ok((nodes, edges)) => json_ok(json!({"ok": true, "nodes": nodes, "edges": edges})),
3454            Err(resp) => resp,
3455        };
3456    }
3457    match blocking_write(move || db.submit_batch(vec![BatchOp::InsertNode { label, key, props }]))
3458        .await
3459    {
3460        Ok((nodes, edges)) => json_ok(json!({"ok": true, "nodes": nodes, "edges": edges})),
3461        Err(resp) => resp,
3462    }
3463}
3464
3465/// `DELETE /node/{key}` — delete a node via the group-commit queue.
3466async fn delete_node(
3467    State(state): State<AppState>,
3468    Extension(identity): Extension<AuthIdentity>,
3469    Path(key): Path<String>,
3470) -> Response {
3471    let db = state.db.clone();
3472    if let AuthIdentity::Role(role_name) = &identity {
3473        let role = role_name.clone();
3474        return match blocking_write(move || {
3475            db.submit_batch_authz(role, vec![BatchOp::DeleteNode { key }])
3476        })
3477        .await
3478        {
3479            Ok(_) => json_ok(json!({"ok": true})),
3480            Err(resp) => resp,
3481        };
3482    }
3483    match blocking_write(move || db.submit_batch(vec![BatchOp::DeleteNode { key }])).await {
3484        Ok(_) => json_ok(json!({"ok": true})),
3485        Err(resp) => resp,
3486    }
3487}
3488
3489/// `POST /edges` — create an edge via the group-commit queue.
3490///
3491/// Body: `{"type": "KNOWS", "src": "alice", "dst": "bob"}`
3492async fn create_edge(
3493    State(state): State<AppState>,
3494    Extension(identity): Extension<AuthIdentity>,
3495    Json(body): Json<Js>,
3496) -> Response {
3497    let edge_type = match body.get("type").and_then(Js::as_str) {
3498        Some(s) => s.to_string(),
3499        None => return err_response("missing type"),
3500    };
3501    let src = match body.get("src").and_then(Js::as_str) {
3502        Some(s) => s.to_string(),
3503        None => return err_response("missing src"),
3504    };
3505    let dst = match body.get("dst").and_then(Js::as_str) {
3506        Some(s) => s.to_string(),
3507        None => return err_response("missing dst"),
3508    };
3509    let db = state.db.clone();
3510    if let AuthIdentity::Role(role_name) = &identity {
3511        let role = role_name.clone();
3512        return match blocking_write(move || {
3513            db.submit_batch_authz(
3514                role,
3515                vec![BatchOp::InsertEdge {
3516                    edge_type,
3517                    src_key: src,
3518                    dst_key: dst,
3519                }],
3520            )
3521        })
3522        .await
3523        {
3524            Ok(_) => json_ok(json!({"ok": true})),
3525            Err(resp) => resp,
3526        };
3527    }
3528    match blocking_write(move || {
3529        db.submit_batch(vec![BatchOp::InsertEdge {
3530            edge_type,
3531            src_key: src,
3532            dst_key: dst,
3533        }])
3534    })
3535    .await
3536    {
3537        Ok(_) => json_ok(json!({"ok": true})),
3538        Err(resp) => resp,
3539    }
3540}
3541
3542/// `DELETE /edges/{etype}/{src}/{dst}` — delete an edge via the group-commit queue.
3543async fn delete_edge(
3544    State(state): State<AppState>,
3545    Extension(identity): Extension<AuthIdentity>,
3546    Path((etype, src, dst)): Path<(String, String, String)>,
3547) -> Response {
3548    let db = state.db.clone();
3549    if let AuthIdentity::Role(role_name) = &identity {
3550        let role = role_name.clone();
3551        return match blocking_write(move || {
3552            db.submit_batch_authz(
3553                role,
3554                vec![BatchOp::DeleteEdge {
3555                    edge_type: etype,
3556                    src_key: src,
3557                    dst_key: dst,
3558                }],
3559            )
3560        })
3561        .await
3562        {
3563            Ok(_) => json_ok(json!({"ok": true})),
3564            Err(resp) => resp,
3565        };
3566    }
3567    match blocking_write(move || {
3568        db.submit_batch(vec![BatchOp::DeleteEdge {
3569            edge_type: etype,
3570            src_key: src,
3571            dst_key: dst,
3572        }])
3573    })
3574    .await
3575    {
3576        Ok(_) => json_ok(json!({"ok": true})),
3577        Err(resp) => resp,
3578    }
3579}
3580
3581/// `POST /nodes/{key}/rename` — rename a node's key via the group-commit queue.
3582///
3583/// Body: `{"new_key": "alice2"}`
3584/// Returns 404 on KeyNotFound, 409 on DuplicateKey, 200 on success.
3585async fn rename_node(
3586    State(state): State<AppState>,
3587    Extension(identity): Extension<AuthIdentity>,
3588    Path(key): Path<String>,
3589    Json(body): Json<Js>,
3590) -> Response {
3591    if let AuthIdentity::Role(_) = identity {
3592        return forbidden("role-bound token: writes are not permitted");
3593    }
3594    let new_key = match body.get("new_key").and_then(Js::as_str) {
3595        Some(s) => s.to_string(),
3596        None => return err_response("missing new_key"),
3597    };
3598    let db = state.db.clone();
3599    match tokio::task::spawn_blocking(move || {
3600        db.submit_batch(vec![BatchOp::RenameNode {
3601            old_key: key,
3602            new_key,
3603        }])
3604    })
3605    .await
3606    {
3607        Ok(Ok(_)) => json_ok(json!({"ok": true})),
3608        Ok(Err(GraphError::KeyNotFound { key })) => key_not_found(key),
3609        Ok(Err(GraphError::DuplicateKey { key })) => conflict_response(key),
3610        Ok(Err(e)) => graph_err(e),
3611        Err(_) => err_response("write task panicked"),
3612    }
3613}
3614
3615/// `POST /edges/upsert` — insert an edge, auto-creating missing endpoints.
3616///
3617/// Body: `{"edge_type":"KNOWS","src_key":"alice","dst_key":"bob","placeholder_label":"Person"}`
3618/// Returns `{"nodes_created": N, "edge_inserted": bool}`.
3619async fn upsert_edge(
3620    State(state): State<AppState>,
3621    Extension(identity): Extension<AuthIdentity>,
3622    Json(body): Json<Js>,
3623) -> Response {
3624    let edge_type = match body.get("edge_type").and_then(Js::as_str) {
3625        Some(s) => s.to_string(),
3626        None => return err_response("missing edge_type"),
3627    };
3628    let src_key = match body.get("src_key").and_then(Js::as_str) {
3629        Some(s) => s.to_string(),
3630        None => return err_response("missing src_key"),
3631    };
3632    let dst_key = match body.get("dst_key").and_then(Js::as_str) {
3633        Some(s) => s.to_string(),
3634        None => return err_response("missing dst_key"),
3635    };
3636    let placeholder_label = match body.get("placeholder_label").and_then(Js::as_str) {
3637        Some(s) => s.to_string(),
3638        None => return err_response("missing placeholder_label"),
3639    };
3640    let db = state.db.clone();
3641    if let AuthIdentity::Role(role_name) = &identity {
3642        let role = role_name.clone();
3643        return match blocking_write(move || {
3644            db.submit_batch_authz(
3645                role,
3646                vec![BatchOp::InsertEdgeUpsert {
3647                    edge_type,
3648                    src_key,
3649                    dst_key,
3650                    placeholder_label,
3651                }],
3652            )
3653        })
3654        .await
3655        {
3656            Ok((nodes, edges)) => json_ok(json!({
3657                "nodes_created": nodes,
3658                "edge_inserted": edges > 0,
3659            })),
3660            Err(resp) => resp,
3661        };
3662    }
3663    match blocking_write(move || {
3664        db.submit_batch(vec![BatchOp::InsertEdgeUpsert {
3665            edge_type,
3666            src_key,
3667            dst_key,
3668            placeholder_label,
3669        }])
3670    })
3671    .await
3672    {
3673        Ok((nodes, edges)) => json_ok(json!({
3674            "nodes_created": nodes,
3675            "edge_inserted": edges > 0,
3676        })),
3677        Err(resp) => resp,
3678    }
3679}
3680
3681/// `PUT /node/{key}/prop/{field}` — set a property via the group-commit queue.
3682///
3683/// Body: `{"value": 42}`
3684async fn set_node_prop(
3685    State(state): State<AppState>,
3686    Extension(identity): Extension<AuthIdentity>,
3687    Path((key, field)): Path<(String, String)>,
3688    Json(body): Json<Js>,
3689) -> Response {
3690    let parsed = body.get("value").map(|v| json_to_value(v.clone()));
3691    let value = match parsed {
3692        Some(Err(e)) => return err_response(format!("value: {e}")),
3693        Some(Ok(Some(v))) => v,
3694        Some(Ok(None)) | None => {
3695            return err_response(
3696                "request body must be a JSON object with a \"value\" field, \
3697                 e.g. {\"value\": \"SanFrancisco\"} or {\"value\": [\"a\", \"b\"]}",
3698            )
3699        }
3700    };
3701    let db = state.db.clone();
3702    if let AuthIdentity::Role(role_name) = &identity {
3703        let role = role_name.clone();
3704        return match blocking_write(move || {
3705            db.submit_batch_authz(role, vec![BatchOp::SetProp { key, field, value }])
3706        })
3707        .await
3708        {
3709            Ok(_) => json_ok(json!({"ok": true})),
3710            Err(resp) => resp,
3711        };
3712    }
3713    match blocking_write(move || db.submit_batch(vec![BatchOp::SetProp { key, field, value }]))
3714        .await
3715    {
3716        Ok(_) => json_ok(json!({"ok": true})),
3717        Err(resp) => resp,
3718    }
3719}
3720
3721// ── History endpoints ─────────────────────────────────────────────────────────
3722//
3723// These are cold-path diagnostic endpoints. They scan the on-disk WAL and
3724// must NOT extend ReaderSnapshot — they use db.read() directly per the
3725// controller ruling (see module doc and task-2-brief.md).
3726//
3727// Role masking: node visibility is checked under the SAME read guard as the
3728// history call (coherent snapshot). A node outside the role's mask responds
3729// identically to an absent node — no existence oracle.
3730
3731/// `GET /node/{key}/history` — return the WAL change history for `key`.
3732///
3733/// Response: `{ key, history: [{commit, change}], total_commits, horizon }`.
3734/// `horizon` is the oldest commit history still reaches; events before it were
3735/// pruned and are not in `history`.
3736/// Role tokens: if `key` is hidden by the role mask, responds with 404
3737/// (same shape as querying an absent key — no existence oracle).
3738async fn node_history_handler(
3739    State(state): State<AppState>,
3740    Extension(identity): Extension<AuthIdentity>,
3741    Path(key): Path<String>,
3742) -> Response {
3743    if let AuthIdentity::Role(ref role_name) = identity {
3744        let g = state.db.read();
3745        let role_mask = match g.mask_for_role(role_name) {
3746            Ok(m) => m,
3747            Err(e) => return role_mask_err(e),
3748        };
3749        // Hidden keys must respond identically to absent keys (no oracle).
3750        if !role_mask.contains_node(&*g, &key) {
3751            return key_not_found(key);
3752        }
3753        let result = match g.node_history(&key) {
3754            Ok(e) => e,
3755            Err(e) => return graph_err(e),
3756        };
3757        // Filter EdgeAdded/EdgeRemoved entries whose `other` endpoint is hidden.
3758        // A role token must not learn about hidden nodes via edge history events —
3759        // mirrors the same protection in `node_edges` (http.rs ~978-989).
3760        use mushroomdb::HistoryChange;
3761        let visible = mushroomdb::HistoryResult {
3762            total_commits: result.total_commits,
3763            horizon: result.horizon,
3764            items: result
3765                .items
3766                .into_iter()
3767                .filter(|entry| match &entry.change {
3768                    HistoryChange::EdgeAdded { other, .. }
3769                    | HistoryChange::EdgeRemoved { other, .. } => {
3770                        role_mask.contains_node(&*g, other)
3771                    }
3772                    _ => true,
3773                })
3774                .collect(),
3775        };
3776        return json_ok(node_history_json(&key, &visible));
3777    }
3778    // Full identity: no masking. Return 404 for absent keys (consistent with
3779    // GET /node/{key} and the Role branch above).
3780    let g = state.db.read();
3781    if !g.has_node(&key) {
3782        return key_not_found(key);
3783    }
3784    let result = match g.node_history(&key) {
3785        Ok(e) => e,
3786        Err(e) => return graph_err(e),
3787    };
3788    json_ok(node_history_json(&key, &result))
3789}
3790
3791/// `GET /history/edge?a=&b=` — return the edge lifecycle between two nodes.
3792///
3793/// Response: `{ a, b, events: [{edge_type, commit, event, rule}], total_commits, horizon }`.
3794/// `horizon` is the oldest commit history still reaches; events before it were
3795/// pruned and are not in `events`.
3796/// Role tokens: BOTH `a` AND `b` must be visible in the role mask, otherwise
3797/// responds with 404 for the first invisible key (no existence oracle).
3798async fn edge_history_handler(
3799    State(state): State<AppState>,
3800    Extension(identity): Extension<AuthIdentity>,
3801    Query(qs): Query<BTreeMap<String, String>>,
3802) -> Response {
3803    let a = match qs.get("a").filter(|s| !s.is_empty()) {
3804        Some(s) => s.clone(),
3805        None => return err_response("missing query param a"),
3806    };
3807    let b = match qs.get("b").filter(|s| !s.is_empty()) {
3808        Some(s) => s.clone(),
3809        None => return err_response("missing query param b"),
3810    };
3811    if let AuthIdentity::Role(ref role_name) = identity {
3812        let g = state.db.read();
3813        let role_mask = match g.mask_for_role(role_name) {
3814            Ok(m) => m,
3815            Err(e) => return role_mask_err(e),
3816        };
3817        // BOTH endpoints must be visible (no oracle for either).
3818        if !role_mask.contains_node(&*g, &a) {
3819            return key_not_found(a);
3820        }
3821        if !role_mask.contains_node(&*g, &b) {
3822            return key_not_found(b);
3823        }
3824        let result = match g.edge_history(&a, &b) {
3825            Ok(r) => r,
3826            Err(e) => return graph_err(e),
3827        };
3828        return json_ok(edge_history_result_json(&a, &b, &result));
3829    }
3830    // Full identity: no masking.
3831    let g = state.db.read();
3832    let result = match g.edge_history(&a, &b) {
3833        Ok(r) => r,
3834        Err(e) => return graph_err(e),
3835    };
3836    json_ok(edge_history_result_json(&a, &b, &result))
3837}
3838
3839/// `GET /history/was_linked?a=&b=&edge_type=&at_commit=` — point-in-time edge check.
3840///
3841/// Response: `{ a, b, edge_type, at_commit, linked }`.
3842/// Returns 400 (not 500) when `at_commit` is outside the visible horizon.
3843/// Role tokens: BOTH `a` AND `b` must be visible, same-as-absent otherwise.
3844async fn was_linked_handler(
3845    State(state): State<AppState>,
3846    Extension(identity): Extension<AuthIdentity>,
3847    Query(qs): Query<BTreeMap<String, String>>,
3848) -> Response {
3849    let a = match qs.get("a").filter(|s| !s.is_empty()) {
3850        Some(s) => s.clone(),
3851        None => return err_response("missing query param a"),
3852    };
3853    let b = match qs.get("b").filter(|s| !s.is_empty()) {
3854        Some(s) => s.clone(),
3855        None => return err_response("missing query param b"),
3856    };
3857    let edge_type = match qs.get("edge_type").filter(|s| !s.is_empty()) {
3858        Some(s) => s.clone(),
3859        None => return err_response("missing query param edge_type"),
3860    };
3861    // `at_commit` takes a 0-based frame index or an RFC 3339 date. A caller
3862    // asking "were they linked on 2026-06-19" should not have to find the frame
3863    // themselves — and a guessed index is how a plausible wrong answer is
3864    // returned.
3865    let at_commit: u64 = match qs.get("at_commit") {
3866        Some(s) => match s.parse::<u64>() {
3867            Ok(n) => n,
3868            Err(_) => match state.db.read().resolve_date(s) {
3869                Ok(n) => n,
3870                Err(e) => return graph_err(e),
3871            },
3872        },
3873        None => return err_response("missing query param at_commit"),
3874    };
3875
3876    if let AuthIdentity::Role(ref role_name) = identity {
3877        let g = state.db.read();
3878        let role_mask = match g.mask_for_role(role_name) {
3879            Ok(m) => m,
3880            Err(e) => return role_mask_err(e),
3881        };
3882        if !role_mask.contains_node(&*g, &a) {
3883            return key_not_found(a);
3884        }
3885        if !role_mask.contains_node(&*g, &b) {
3886            return key_not_found(b);
3887        }
3888        return match g.was_linked(&a, &b, &edge_type, at_commit) {
3889            Ok(linked) => json_ok(json!({
3890                "a": a, "b": b, "edge_type": edge_type,
3891                "at_commit": at_commit, "linked": linked,
3892            })),
3893            Err(e @ GraphError::CommitOutOfRange { .. }) => (
3894                StatusCode::BAD_REQUEST,
3895                Json(json!({"error": e.to_string()})),
3896            )
3897                .into_response(),
3898            Err(e) => graph_err(e),
3899        };
3900    }
3901
3902    // Full identity: no masking.
3903    let g = state.db.read();
3904    match g.was_linked(&a, &b, &edge_type, at_commit) {
3905        Ok(linked) => json_ok(json!({
3906            "a": a, "b": b, "edge_type": edge_type,
3907            "at_commit": at_commit, "linked": linked,
3908        })),
3909        Err(e @ GraphError::CommitOutOfRange { .. }) => (
3910            StatusCode::BAD_REQUEST,
3911            Json(json!({"error": e.to_string()})),
3912        )
3913            .into_response(),
3914        Err(e) => graph_err(e),
3915    }
3916}
3917
3918/// `DELETE /node/{key}/prop/{field}` — remove a property via the group-commit queue.
3919async fn remove_node_prop(
3920    State(state): State<AppState>,
3921    Extension(identity): Extension<AuthIdentity>,
3922    Path((key, field)): Path<(String, String)>,
3923) -> Response {
3924    let db = state.db.clone();
3925    if let AuthIdentity::Role(role_name) = &identity {
3926        let role = role_name.clone();
3927        return match blocking_write(move || {
3928            db.submit_batch_authz(role, vec![BatchOp::RemoveProp { key, field }])
3929        })
3930        .await
3931        {
3932            Ok(_) => json_ok(json!({"ok": true})),
3933            Err(resp) => resp,
3934        };
3935    }
3936    match blocking_write(move || db.submit_batch(vec![BatchOp::RemoveProp { key, field }])).await {
3937        Ok(_) => json_ok(json!({"ok": true})),
3938        Err(resp) => resp,
3939    }
3940}
3941
3942/// Constant-time byte-string equality, used for secret (token) comparison so
3943/// the match does not short-circuit on the first differing byte and leak the
3944/// token through response-timing. Length is compared up front (token length is
3945/// not treated as a secret); the byte loop is branch-free over equal lengths.
3946fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
3947    if a.len() != b.len() {
3948        return false;
3949    }
3950    let mut diff = 0u8;
3951    for (x, y) in a.iter().zip(b.iter()) {
3952        diff |= x ^ y;
3953    }
3954    diff == 0
3955}
3956
3957/// Maximum `depth` accepted by the neighborhood endpoint. Bounds the read-lock
3958/// hold time of a single BFS request on a dense graph (a hostile
3959/// `depth=4294967295` would otherwise traverse the whole graph under the read
3960/// guard, starving writers).
3961const MAX_NEIGHBORHOOD_DEPTH: u32 = 64;
3962
3963/// Parse `POST /find_similar` `vector` — a non-empty array of numbers.
3964fn parse_find_similar_vector(v: Option<&Js>) -> Result<Vec<f64>, String> {
3965    let Some(arr) = v.and_then(Js::as_array) else {
3966        return Err("missing vector".into());
3967    };
3968    let mut out = Vec::with_capacity(arr.len());
3969    for item in arr {
3970        match item.as_f64() {
3971            Some(n) => out.push(n),
3972            None => return Err("vector must be an array of numbers".into()),
3973        }
3974    }
3975    if out.is_empty() {
3976        return Err("vector must be a non-empty array of numbers".into());
3977    }
3978    Ok(out)
3979}
3980
3981/// Parse an optional JSON `mask` array of node keys.
3982fn parse_mask_keys(v: Option<&Js>) -> Result<Option<Vec<String>>, String> {
3983    match v {
3984        None | Some(Js::Null) => Ok(None),
3985        Some(Js::Array(arr)) => {
3986            let mut keys = Vec::with_capacity(arr.len());
3987            for item in arr {
3988                match item.as_str() {
3989                    Some(s) => keys.push(s.to_string()),
3990                    None => return Err("mask must be an array of strings".into()),
3991                }
3992            }
3993            Ok(Some(keys))
3994        }
3995        Some(_) => Err("mask must be an array of strings".into()),
3996    }
3997}
3998
3999/// Parse optional `where` as a `PropPredicate`. Invalid predicates are QueryError.
4000fn parse_where_body(v: Option<&Js>) -> Result<Option<PropPredicate>, String> {
4001    match v {
4002        None | Some(Js::Null) => Ok(None),
4003        Some(w) => {
4004            let pred: PropPredicate =
4005                serde_json::from_value(w.clone()).map_err(|e| format!("where: {e}"))?;
4006            pred.validate_named("where")?;
4007            Ok(Some(pred))
4008        }
4009    }
4010}
4011
4012/// Parse and bound the neighborhood `depth` query parameter.
4013fn resolve_neighborhood_depth(raw: Option<&str>) -> Result<u32, String> {
4014    match raw {
4015        None => Ok(1),
4016        Some(s) => {
4017            let d: u32 = s
4018                .parse()
4019                .map_err(|_| "depth must be an integer".to_string())?;
4020            if d > MAX_NEIGHBORHOOD_DEPTH {
4021                return Err(format!("depth must be ≤ {MAX_NEIGHBORHOOD_DEPTH}"));
4022            }
4023            Ok(d)
4024        }
4025    }
4026}
4027
4028/// Confine a client-supplied backup `dest` to `root`, closing the arbitrary
4029/// filesystem-write vector (a full-access token could otherwise direct the
4030/// backup writer at `/etc/...`, `/root/.ssh/...`, etc.).
4031///
4032/// Rules: reject empty; reject any `..` segment; relative paths resolve under
4033/// `root`; absolute paths must already fall within `root`. The check is lexical
4034/// (no filesystem access) and safe because `..` is rejected before joining.
4035fn confine_backup_dest(dest: &str, root: &std::path::Path) -> Result<PathBuf, String> {
4036    if dest.is_empty() {
4037        return Err("missing or empty \"dest\" field".into());
4038    }
4039    let dest_path = std::path::Path::new(dest);
4040    if dest_path
4041        .components()
4042        .any(|c| matches!(c, std::path::Component::ParentDir))
4043    {
4044        return Err("backup \"dest\" may not contain \"..\" path segments".into());
4045    }
4046    let joined = if dest_path.is_absolute() {
4047        dest_path.to_path_buf()
4048    } else {
4049        root.join(dest_path)
4050    };
4051    if !joined.starts_with(root) {
4052        return Err(format!(
4053            "backup \"dest\" must be within the backup root ({}); \
4054             set MUSHROOMDB_BACKUP_DIR to change it",
4055            root.display()
4056        ));
4057    }
4058    Ok(joined)
4059}
4060
4061/// Directory backups are confined to: `MUSHROOMDB_BACKUP_DIR` if set, else the
4062/// server's current working directory.
4063fn backup_root() -> PathBuf {
4064    std::env::var_os("MUSHROOMDB_BACKUP_DIR")
4065        .map(PathBuf::from)
4066        .or_else(|| std::env::current_dir().ok())
4067        .unwrap_or_else(|| PathBuf::from("."))
4068}
4069
4070/// `POST /backup` — take a consistent backup of the database to `dest`.
4071///
4072/// The read guard is held for the duration of the file copies, which is the
4073/// correct cross-process synchronisation point: the server is the single
4074/// process touching the files, so holding the read lock excludes concurrent
4075/// in-process writers.  This is the safe alternative to running the
4076/// `mushroomdb backup` CLI against a live-served store.
4077///
4078/// Request body: `{"dest": "backup-dir"}`. `dest` is confined to the backup
4079/// root (`MUSHROOMDB_BACKUP_DIR`, else the server's working directory):
4080/// relative paths resolve under it, absolute paths must fall within it, and
4081/// `..` segments are rejected. This prevents a full-access token from writing
4082/// to arbitrary filesystem paths.
4083///
4084/// Responses:
4085/// - `200 OK` — backup completed; body is a `BackupReport` JSON object.
4086/// - `500 Internal Server Error` — backup succeeded but verification failed;
4087///   body is the `BackupReport` JSON object (examine `files` and `bytes`).
4088/// - `400 Bad Request` — missing or invalid `dest`.
4089/// - `403 Forbidden` — role-bound token; this endpoint requires a full-access token.
4090async fn backup(
4091    State(state): State<AppState>,
4092    Extension(identity): Extension<AuthIdentity>,
4093    Json(body): Json<Js>,
4094) -> Response {
4095    if let AuthIdentity::Role(_) = identity {
4096        return forbidden("role-bound token: /backup requires a full-access token");
4097    }
4098    if let Some(refusal) = refuse_write_tier(&identity) {
4099        return refusal;
4100    }
4101    let root = backup_root();
4102    let dest = match body.get("dest").and_then(Js::as_str) {
4103        Some(s) => match confine_backup_dest(s, &root) {
4104            Ok(p) => p,
4105            Err(e) => return err_response(e),
4106        },
4107        None => return err_response("missing or empty \"dest\" field"),
4108    };
4109    let db = state.db.clone();
4110    let report: BackupReport = match tokio::task::spawn_blocking(move || {
4111        // The read guard is held inside spawn_blocking so file copies happen
4112        // with the write lock excluded.  The guard drops at end of closure.
4113        let g = db.read();
4114        g.backup_to(&dest)
4115    })
4116    .await
4117    {
4118        Ok(Ok(r)) => r,
4119        Ok(Err(e)) => return graph_err(e),
4120        Err(_) => return err_response("backup task panicked"),
4121    };
4122
4123    let body = match serde_json::to_value(BackupReportJson::from(&report)) {
4124        Ok(v) => v,
4125        Err(e) => return err_response(e.to_string()),
4126    };
4127
4128    if report.verified {
4129        json_ok(body)
4130    } else {
4131        (StatusCode::INTERNAL_SERVER_ERROR, Json(body)).into_response()
4132    }
4133}
4134
4135/// JSON-serialisable projection of [`BackupReport`].
4136#[derive(serde::Serialize)]
4137struct BackupReportJson<'a> {
4138    files: &'a [String],
4139    bytes: u64,
4140    verified: bool,
4141}
4142
4143impl<'a> From<&'a BackupReport> for BackupReportJson<'a> {
4144    fn from(r: &'a BackupReport) -> Self {
4145        Self {
4146            files: &r.files,
4147            bytes: r.bytes,
4148            verified: r.verified,
4149        }
4150    }
4151}
4152
4153#[cfg(test)]
4154mod tests {
4155    fn prom_sample(
4156        backup: Option<BackupSample>,
4157        bolt: Option<BoltSample>,
4158        rss: Option<u64>,
4159    ) -> PrometheusSample {
4160        PrometheusSample {
4161            nodes_live: 3,
4162            nodes_tombstoned: 1,
4163            edges_live: 2,
4164            commits_total: 9,
4165            wal_size_bytes: Some(4096),
4166            rss_bytes: rss,
4167            process_start_time_seconds: 1_700_000_000,
4168            slow_queries_total: 4,
4169            transactions_open: 2,
4170            bolt,
4171            backup,
4172        }
4173    }
4174
4175    #[test]
4176    fn prometheus_text_has_help_and_type_for_every_sample() {
4177        let text = format_prometheus(&prom_sample(None, None, Some(1024)));
4178        let samples: Vec<&str> = text.lines().filter(|l| !l.starts_with('#')).collect();
4179        assert!(samples.contains(&"mushroomdb_nodes_live 3"));
4180        assert!(samples.contains(&"mushroomdb_commits_total 9"));
4181        assert!(samples.contains(&"mushroomdb_http_transactions_open 2"));
4182        for line in &samples {
4183            let name = line.split(' ').next().unwrap();
4184            assert!(text.contains(&format!("# HELP {name} ")), "{name} HELP");
4185            assert!(text.contains(&format!("# TYPE {name} ")), "{name} TYPE");
4186        }
4187        assert!(!text.contains("backup"));
4188        assert!(!text.contains("bolt"), "no Bolt series with Bolt off");
4189    }
4190
4191    #[test]
4192    fn prometheus_text_reports_bolt_connections_when_bolt_is_on() {
4193        let text = format_prometheus(&prom_sample(
4194            None,
4195            Some(BoltSample {
4196                open: 3,
4197                accepted_total: 41,
4198            }),
4199            None,
4200        ));
4201        assert!(text.contains("# TYPE mushroomdb_bolt_connections_open gauge\n"));
4202        assert!(text.contains("\nmushroomdb_bolt_connections_open 3\n"));
4203        assert!(text.contains("# TYPE mushroomdb_bolt_connections_accepted_total counter\n"));
4204        assert!(text.contains("\nmushroomdb_bolt_connections_accepted_total 41\n"));
4205    }
4206
4207    #[test]
4208    fn prometheus_text_omits_an_unavailable_gauge_and_adds_backup_series() {
4209        let text = format_prometheus(&prom_sample(
4210            Some(BackupSample {
4211                upload_errors_total: 5,
4212                last_success_ts: 1_700_000_123,
4213                lag_s: 7,
4214                healthy: false,
4215                lease_held: true,
4216            }),
4217            None,
4218            None,
4219        ));
4220        assert!(!text.contains("mushroomdb_rss_bytes"));
4221        assert!(text.contains("# TYPE mushroomdb_backup_upload_errors_total counter\n"));
4222        assert!(text.contains("\nmushroomdb_backup_upload_errors_total 5\n"));
4223        assert!(text.contains("\nmushroomdb_backup_last_success_timestamp_seconds 1700000123\n"));
4224        assert!(text.contains("\nmushroomdb_backup_lag_seconds 7\n"));
4225        assert!(text.contains("\nmushroomdb_backup_healthy 0\n"));
4226        assert!(text.contains("# TYPE mushroomdb_backup_lease_held gauge\n"));
4227        assert!(text.contains("\nmushroomdb_backup_lease_held 1\n"));
4228    }
4229
4230    #[test]
4231    fn accept_negotiation_defaults_to_json() {
4232        let accept = |v: &str| {
4233            let mut h = HeaderMap::new();
4234            h.insert(header::ACCEPT, HeaderValue::from_str(v).unwrap());
4235            wants_prometheus_text(&h)
4236        };
4237        assert!(!wants_prometheus_text(&HeaderMap::new()));
4238        assert!(!accept("*/*"));
4239        assert!(!accept("application/json, text/plain, */*"));
4240        assert!(!accept("text/html,application/xhtml+xml"));
4241        assert!(accept("text/plain"));
4242        assert!(accept("TEXT/PLAIN;version=0.0.4;q=0.5"));
4243        assert!(accept(
4244            "application/openmetrics-text;version=1.0.0,*/*;q=0.1"
4245        ));
4246    }
4247
4248    use super::*;
4249    use crate::json::result_set_json;
4250    use mushroomdb::{DegreeConfig, PageRankConfig, ResultSet, Value, WccConfig};
4251
4252    #[tokio::test]
4253    async fn an_unwritable_directory_is_a_500_with_a_short_client_message() {
4254        let resp = graph_err(GraphError::Unwritable {
4255            path: "/secret/dir/LOCK".into(),
4256            detail: "uid 65532 owner 0 /secret/dir".into(),
4257            read_only_filesystem: false,
4258        });
4259        assert_eq!(resp.status(), StatusCode::INTERNAL_SERVER_ERROR);
4260        let body = axum::body::to_bytes(resp.into_body(), 4096).await.unwrap();
4261        let text = String::from_utf8(body.to_vec()).unwrap();
4262        assert!(
4263            text.contains(GraphError::UNWRITABLE_CLIENT_MESSAGE),
4264            "{text}"
4265        );
4266        assert!(
4267            !text.contains("secret") && !text.contains("65532"),
4268            "{text}"
4269        );
4270    }
4271
4272    #[test]
4273    fn nan_float_cell_serializes_as_null() {
4274        let mut rs = ResultSet::new(vec!["n".into()]);
4275        rs.push_row(vec![Some(Value::Float(f64::NAN))]);
4276        let j = result_set_json(&rs);
4277        assert_eq!(j["rows"][0][0], Js::Null);
4278    }
4279
4280    /// Verify that `POST /algo/pagerank` accepts an empty JSON body `{}` and
4281    /// applies server defaults (regression guard for `#[serde(default)]`).
4282    #[test]
4283    fn pagerank_config_empty_body_uses_defaults() {
4284        let config: PageRankConfig = serde_json::from_str("{}").unwrap();
4285        let default = PageRankConfig::default();
4286        assert_eq!(config.damping, default.damping);
4287        assert_eq!(config.max_iters, default.max_iters);
4288        assert_eq!(config.tol, default.tol);
4289        assert_eq!(config.budget_ms, default.budget_ms);
4290        assert_eq!(config.edge_type, default.edge_type);
4291    }
4292
4293    /// Same guard for `POST /algo/wcc`.
4294    #[test]
4295    fn wcc_config_empty_body_uses_defaults() {
4296        let config: WccConfig = serde_json::from_str("{}").unwrap();
4297        let default = WccConfig::default();
4298        assert_eq!(config.budget_ms, default.budget_ms);
4299        assert_eq!(config.edge_type, default.edge_type);
4300    }
4301
4302    /// Same guard for `POST /algo/degree`.
4303    #[test]
4304    fn degree_config_empty_body_uses_defaults() {
4305        let config: DegreeConfig = serde_json::from_str("{}").unwrap();
4306        let default = DegreeConfig::default();
4307        assert_eq!(config.budget_ms, default.budget_ms);
4308        assert_eq!(config.edge_type, default.edge_type);
4309    }
4310
4311    #[test]
4312    fn backup_dest_rejects_empty() {
4313        assert!(confine_backup_dest("", std::path::Path::new("/srv/backups")).is_err());
4314    }
4315
4316    #[test]
4317    fn constant_time_eq_matches_equality() {
4318        assert!(constant_time_eq(b"secret-token", b"secret-token"));
4319        assert!(constant_time_eq(b"", b""));
4320        assert!(!constant_time_eq(b"secret-token", b"secret-toke")); // shorter
4321        assert!(!constant_time_eq(b"secret-token", b"secret-tokex")); // last byte differs
4322        assert!(!constant_time_eq(b"secret-token", b"Xecret-token")); // first byte differs
4323        assert!(!constant_time_eq(b"", b"x"));
4324    }
4325
4326    #[test]
4327    fn depth_defaults_to_one_when_absent() {
4328        assert_eq!(resolve_neighborhood_depth(None).unwrap(), 1);
4329    }
4330
4331    #[test]
4332    fn depth_within_cap_is_accepted() {
4333        assert_eq!(resolve_neighborhood_depth(Some("10")).unwrap(), 10);
4334        assert_eq!(
4335            resolve_neighborhood_depth(Some(&MAX_NEIGHBORHOOD_DEPTH.to_string())).unwrap(),
4336            MAX_NEIGHBORHOOD_DEPTH
4337        );
4338    }
4339
4340    #[test]
4341    fn depth_over_cap_is_rejected() {
4342        assert!(resolve_neighborhood_depth(Some("65")).is_err());
4343        assert!(resolve_neighborhood_depth(Some("4294967295")).is_err());
4344    }
4345
4346    #[test]
4347    fn depth_non_integer_is_rejected() {
4348        assert!(resolve_neighborhood_depth(Some("abc")).is_err());
4349    }
4350
4351    #[test]
4352    fn backup_dest_rejects_parent_traversal() {
4353        let root = std::path::Path::new("/srv/backups");
4354        assert!(confine_backup_dest("../../etc/cron.d", root).is_err());
4355        assert!(confine_backup_dest("ok/../../../etc", root).is_err());
4356    }
4357
4358    #[test]
4359    fn backup_dest_rejects_absolute_outside_root() {
4360        let root = std::path::Path::new("/srv/backups");
4361        assert!(confine_backup_dest("/etc/cron.d", root).is_err());
4362        assert!(confine_backup_dest("/root/.ssh/authorized_keys", root).is_err());
4363    }
4364
4365    #[test]
4366    fn backup_dest_allows_relative_within_root() {
4367        let root = std::path::Path::new("/srv/backups");
4368        assert_eq!(
4369            confine_backup_dest("nightly", root).unwrap(),
4370            PathBuf::from("/srv/backups/nightly")
4371        );
4372        assert_eq!(
4373            confine_backup_dest("2026/aug", root).unwrap(),
4374            PathBuf::from("/srv/backups/2026/aug")
4375        );
4376    }
4377
4378    #[test]
4379    fn backup_dest_allows_absolute_within_root() {
4380        let root = std::path::Path::new("/srv/backups");
4381        assert_eq!(
4382            confine_backup_dest("/srv/backups/x", root).unwrap(),
4383            PathBuf::from("/srv/backups/x")
4384        );
4385    }
4386}
4387
4388#[cfg(test)]
4389mod read_parity_tests {
4390    //! TX-5 differential: every HTTP read route that moved from the read lock to
4391    //! the lock-free reader answers byte-for-byte what the read-lock path answered.
4392    //!
4393    //! The `legacy_*` functions are the pre-TX-5 handler bodies, kept here as the
4394    //! oracle: each runs under `db.read()` against the `GraphDb` handle, exactly as
4395    //! the route did before it moved. Every case sends the same request through the
4396    //! live router and compares status, content type and body bytes.
4397    //!
4398    //! The matrix runs over the house-rule axes: one batch vs separate commits, a
4399    //! never-closed handle with a delta tail, a snapshotted store with an overlay
4400    //! on top, and a reopened store (WAL replay).
4401
4402    use super::*;
4403    use crate::json::{masked_edges_json, node_edges_json, node_info_json, result_set_json};
4404    use axum::body::{to_bytes, Body};
4405    use mushroomdb::{Predicate, RoleDef, RuleDef};
4406    use tower::ServiceExt;
4407
4408    // ── The oracle: the read-lock bodies as they were before TX-5 ────────────────
4409
4410    fn legacy_client_mask(
4411        db: &mushroomdb::GraphDb<mushroomdb::RealFs>,
4412        mask_param: &str,
4413        stub: bool,
4414    ) -> NodeMask {
4415        let keys = mask_param
4416            .split(',')
4417            .map(str::trim)
4418            .filter(|s| !s.is_empty());
4419        let m = NodeMask::from_keys(db, keys);
4420        if stub {
4421            m.with_mode(MaskMode::Stub)
4422        } else {
4423            m
4424        }
4425    }
4426
4427    fn legacy_node_info(db: &SharedDb, key: &str, mask: &str, stub: bool) -> Response {
4428        let g = db.read();
4429        if !mask.is_empty() {
4430            let mask = legacy_client_mask(&g, mask, stub);
4431            return match g.node_info_masked(key, &mask) {
4432                Some(mushroomdb::MaskedNodeResult::Visible(info)) => json_ok(node_info_json(&info)),
4433                Some(mushroomdb::MaskedNodeResult::Restricted) => {
4434                    json_ok(crate::json::stub_node_json(key))
4435                }
4436                None => key_not_found(key.to_string()),
4437            };
4438        }
4439        match g.node_info(key) {
4440            Some(info) => json_ok(node_info_json(&info)),
4441            None => key_not_found(key.to_string()),
4442        }
4443    }
4444
4445    fn legacy_node_edges(db: &SharedDb, key: &str, mask: &str, stub: bool) -> Response {
4446        let g = db.read();
4447        if !mask.is_empty() {
4448            let mask = legacy_client_mask(&g, mask, stub);
4449            return match g.node_edges_masked(key, &mask) {
4450                Ok(edges) => json_ok(masked_edges_json(&edges)),
4451                Err(GraphError::KeyNotFound { key }) => key_not_found(key),
4452                Err(e) => graph_err(e),
4453            };
4454        }
4455        match g.node_edges(key) {
4456            Ok(edges) => json_ok(node_edges_json(&edges)),
4457            Err(GraphError::KeyNotFound { key }) => key_not_found(key),
4458            Err(e) => graph_err(e),
4459        }
4460    }
4461
4462    #[allow(clippy::too_many_arguments)]
4463    fn legacy_neighborhood(
4464        db: &SharedDb,
4465        key: &str,
4466        depth: u32,
4467        edge_types: Option<&[&str]>,
4468        dir: Dir,
4469        mask: &str,
4470        stub: bool,
4471    ) -> Response {
4472        let g = db.read();
4473        if !mask.is_empty() {
4474            let mask = legacy_client_mask(&g, mask, stub);
4475            return match g.neighborhood_masked(key, depth, edge_types, dir, &mask) {
4476                Some(rs) => json_ok(result_set_json(&rs)),
4477                None => key_not_found(key.to_string()),
4478            };
4479        }
4480        match g.node_ref(key) {
4481            Some(n) => json_ok(result_set_json(&n.neighborhood(depth, edge_types, dir))),
4482            None => key_not_found(key.to_string()),
4483        }
4484    }
4485
4486    /// The full-token, live `/query` read paths (unmasked, and client mask and/or
4487    /// namespace), after the request has been parsed.
4488    fn legacy_query(
4489        db: &SharedDb,
4490        cypher: &str,
4491        mask_keys: Option<&[String]>,
4492        namespace: Option<&str>,
4493        stub: bool,
4494        format: &str,
4495    ) -> Response {
4496        let params = BTreeMap::new();
4497        if mask_keys.is_some() || namespace.is_some() {
4498            let db = db.read();
4499            let m = match (mask_keys, namespace) {
4500                (Some(keys), Some(ns)) => {
4501                    NodeMask::from_keys(&*db, keys.iter().map(String::as_str))
4502                        .intersect(&db.mask_for_namespace(ns))
4503                }
4504                (Some(keys), None) => NodeMask::from_keys(&*db, keys.iter().map(String::as_str)),
4505                (None, Some(ns)) => db.mask_for_namespace(ns),
4506                (None, None) => unreachable!(),
4507            };
4508            let mask = if stub { m.with_mode(MaskMode::Stub) } else { m };
4509            return match db.query_masked(cypher, &params, &mask) {
4510                Ok(rs) => format_query_result(rs, format),
4511                Err(GraphError::MaskedReadOnly) => (
4512                    StatusCode::BAD_REQUEST,
4513                    Json(json!({"error": "masked queries are read-only"})),
4514                )
4515                    .into_response(),
4516                Err(e) => graph_err(e),
4517            };
4518        }
4519        // The route classifies the statement before it picks a lock; a statement
4520        // that does not lex is refused there, on either path.
4521        match is_write_query(cypher) {
4522            Ok(false) => {}
4523            Ok(true) => unreachable!("writes are not read routes"),
4524            Err(e) => return err_response(e),
4525        }
4526        match db.read().query(cypher, &params) {
4527            Ok(rs) => format_query_result(rs, format),
4528            Err(e) => graph_err(e),
4529        }
4530    }
4531
4532    /// The role-token read of `/query`, resolved the read-lock way: the role's mask
4533    /// from the live handle, intersected as the route intersects it, and the query
4534    /// run by `GraphDb::query_masked`. The route itself has been on the reader since
4535    /// 0.6; this pins that its answer, `UNION` included, is the write handle's.
4536    fn legacy_role_query(
4537        db: &SharedDb,
4538        role: &str,
4539        cypher: &str,
4540        mask_keys: Option<&[String]>,
4541        namespace: Option<&str>,
4542    ) -> Response {
4543        // The role branch's own checks, which run before any lock on either path.
4544        match is_write_query(cypher) {
4545            Ok(false) => {}
4546            Ok(true) => unreachable!("writes are not read routes"),
4547            Err(e) => return err_response(e),
4548        }
4549        let g = db.read();
4550        let role_mask = match g.mask_for_role(role) {
4551            Ok(m) => m,
4552            Err(e) => return role_mask_err(e),
4553        };
4554        let mask = match mask_keys {
4555            Some(keys) => {
4556                role_mask.intersect(&NodeMask::from_keys(&*g, keys.iter().map(String::as_str)))
4557            }
4558            None => role_mask,
4559        };
4560        let mask = match namespace {
4561            Some(ns) => mask.intersect(&g.mask_for_namespace(ns)),
4562            None => mask,
4563        };
4564        match g.query_masked(cypher, &BTreeMap::new(), &mask) {
4565            Ok(rs) => format_query_result(rs, "json"),
4566            Err(GraphError::MaskedReadOnly) => (
4567                StatusCode::BAD_REQUEST,
4568                Json(json!({"error": "masked queries are read-only"})),
4569            )
4570                .into_response(),
4571            Err(e) => graph_err(e),
4572        }
4573    }
4574
4575    // ── Plumbing ──────────────────────────────────────────────────────────────────
4576
4577    #[derive(Debug, PartialEq)]
4578    struct Answer {
4579        status: StatusCode,
4580        content_type: Option<String>,
4581        body: Vec<u8>,
4582    }
4583
4584    async fn answer(res: Response) -> Answer {
4585        let status = res.status();
4586        let content_type = res
4587            .headers()
4588            .get(header::CONTENT_TYPE)
4589            .and_then(|v| v.to_str().ok())
4590            .map(str::to_string);
4591        let body = to_bytes(res.into_body(), usize::MAX)
4592            .await
4593            .unwrap()
4594            .to_vec();
4595        Answer {
4596            status,
4597            content_type,
4598            body,
4599        }
4600    }
4601
4602    async fn routed(app: &Router, req: axum::http::Request<Body>) -> Answer {
4603        answer(app.clone().oneshot(req).await.unwrap()).await
4604    }
4605
4606    fn get(uri: &str, token: Option<&str>) -> axum::http::Request<Body> {
4607        let mut b = axum::http::Request::builder().method("GET").uri(uri);
4608        if let Some(t) = token {
4609            b = b.header(header::AUTHORIZATION, format!("Bearer {t}"));
4610        }
4611        b.body(Body::empty()).unwrap()
4612    }
4613
4614    fn post(uri: &str, body: Js, token: Option<&str>) -> axum::http::Request<Body> {
4615        let mut b = axum::http::Request::builder()
4616            .method("POST")
4617            .uri(uri)
4618            .header(header::CONTENT_TYPE, "application/json");
4619        if let Some(t) = token {
4620            b = b.header(header::AUTHORIZATION, format!("Bearer {t}"));
4621        }
4622        b.body(Body::from(body.to_string())).unwrap()
4623    }
4624
4625    fn tmp(name: &str) -> PathBuf {
4626        let nanos = std::time::SystemTime::now()
4627            .duration_since(std::time::UNIX_EPOCH)
4628            .unwrap()
4629            .as_nanos();
4630        let d =
4631            std::env::temp_dir().join(format!("tx5-parity-{name}-{}-{nanos}", std::process::id()));
4632        let _ = std::fs::remove_dir_all(&d);
4633        d
4634    }
4635
4636    const FULL: &str = "admin";
4637    const ROLE: &str = "role-tok";
4638
4639    /// How a fixture reaches its final state.
4640    #[derive(Clone, Copy, Debug)]
4641    enum Build {
4642        /// Every write its own commit, on a handle that never closed: a long
4643        /// delta tail over the open-time fold.
4644        SeparateCommits,
4645        /// The same graph written as one batch.
4646        OneBatch,
4647        /// Snapshotted part-way, then more writes: an mmap base with an overlay
4648        /// and a tail on top.
4649        Snapshotted,
4650        /// Written, closed, and reopened: the state comes from WAL replay.
4651        Reopened,
4652        /// Equality indexes declared and folded, then the second half on top:
4653        /// indexed lookups answered from the fold's index plus the tail.
4654        Indexed,
4655    }
4656
4657    fn rule() -> RuleDef {
4658        RuleDef {
4659            name: "works_at".into(),
4660            src_label: "Person".into(),
4661            dst_label: "Org".into(),
4662            predicate: Predicate::KeyMatch {
4663                field: "org_id".into(),
4664            },
4665            edge_type: "WORKS_AT".into(),
4666            weight_prop: None,
4667            max_edges: None,
4668            approximate: false,
4669            via_label: None,
4670            via_edge: None,
4671            via_dir: None,
4672            namespace: None,
4673        }
4674    }
4675
4676    /// A weighted rule: its `score` is written on each edge by the rule engine,
4677    /// with no WAL record, and refreshed in place when an endpoint changes.
4678    fn weighted_rule() -> RuleDef {
4679        RuleDef {
4680            name: "skill_fit".into(),
4681            src_label: "Person".into(),
4682            dst_label: "Org".into(),
4683            predicate: Predicate::Overlap {
4684                field: "skills".into(),
4685                min: 0.3,
4686            },
4687            edge_type: "FITS".into(),
4688            weight_prop: Some("score".into()),
4689            ..rule()
4690        }
4691    }
4692
4693    fn skills(list: &[&str]) -> Value {
4694        Value::List(list.iter().map(|s| Value::Str(s.to_string())).collect())
4695    }
4696
4697    fn person(key: &str, org: &str, ns: &str) -> BatchOp {
4698        let mine: &[&str] = match key.len() % 3 {
4699            0 => &["rust", "sql"],
4700            1 => &["rust", "go", "sql"],
4701            _ => &["go"],
4702        };
4703        BatchOp::InsertNode {
4704            label: "Person".into(),
4705            key: key.into(),
4706            props: vec![
4707                ("name".into(), Value::Str(key.to_uppercase())),
4708                ("org_id".into(), Value::Str(org.into())),
4709                ("age".into(), Value::Int(key.len() as i64 * 7)),
4710                ("ns".into(), Value::Str(ns.into())),
4711                ("skills".into(), skills(mine)),
4712            ],
4713        }
4714    }
4715
4716    fn edge(t: &str, s: &str, d: &str) -> BatchOp {
4717        BatchOp::InsertEdge {
4718            edge_type: t.into(),
4719            src_key: s.into(),
4720            dst_key: d.into(),
4721        }
4722    }
4723
4724    /// The graph every fixture holds: two orgs, people in two namespaces, a
4725    /// rule-derived `WORKS_AT` per person, asserted `KNOWS` edges (each inside one
4726    /// namespace, as the store requires) including a self-loop and a 2-cycle, a
4727    /// property removed, and a node deleted.
4728    fn first_half() -> Vec<BatchOp> {
4729        vec![
4730            BatchOp::InsertNode {
4731                label: "Org".into(),
4732                key: "o1".into(),
4733                props: vec![
4734                    ("name".into(), Value::Str("Acme".into())),
4735                    ("ns".into(), Value::Str("tenant-a".into())),
4736                    ("skills".into(), skills(&["rust", "sql", "go"])),
4737                ],
4738            },
4739            BatchOp::InsertNode {
4740                label: "Org".into(),
4741                key: "o2".into(),
4742                props: vec![
4743                    ("ns".into(), Value::Str("tenant-b".into())),
4744                    ("skills".into(), skills(&["go", "sql"])),
4745                ],
4746            },
4747            person("alice", "o1", "tenant-a"),
4748            person("bob", "o1", "tenant-a"),
4749            person("carol", "o2", "tenant-b"),
4750        ]
4751    }
4752
4753    fn second_half() -> Vec<BatchOp> {
4754        vec![
4755            person("dave", "o2", "tenant-b"),
4756            person("erin", "o1", "tenant-a"),
4757            edge("KNOWS", "alice", "bob"),
4758            edge("KNOWS", "erin", "bob"),
4759            edge("KNOWS", "bob", "alice"),
4760            edge("KNOWS", "carol", "dave"),
4761            edge("KNOWS", "dave", "dave"),
4762            edge("LIKES", "erin", "alice"),
4763            edge("LIKES", "alice", "o1"),
4764            edge("LIKES", "dave", "o2"),
4765            BatchOp::SetProp {
4766                key: "bob".into(),
4767                field: "age".into(),
4768                value: Value::Int(41),
4769            },
4770            // Moves bob's `FITS` score in place: no edge is derived or retracted.
4771            BatchOp::SetProp {
4772                key: "bob".into(),
4773                field: "skills".into(),
4774                value: skills(&["rust", "sql", "go"]),
4775            },
4776            BatchOp::RemoveProp {
4777                key: "carol".into(),
4778                field: "age".into(),
4779            },
4780            person("zed", "o1", "tenant-a"),
4781            edge("KNOWS", "zed", "alice"),
4782            BatchOp::DeleteNode { key: "zed".into() },
4783            // Moves an indexed value: the fold's index still says CAROL.
4784            BatchOp::SetProp {
4785                key: "carol".into(),
4786                field: "name".into(),
4787                value: Value::Str("CAROLINE".into()),
4788            },
4789        ]
4790    }
4791
4792    fn write_ops(db: &SharedDb, ops: Vec<BatchOp>, one_batch: bool) {
4793        if one_batch {
4794            db.submit_batch(ops).unwrap();
4795        } else {
4796            for op in ops {
4797                db.submit_batch(vec![op]).unwrap();
4798            }
4799        }
4800    }
4801
4802    fn roles() -> mushroomdb::schema::Schema {
4803        mushroomdb::schema::Schema {
4804            roles: vec![RoleDef {
4805                name: "analyst".into(),
4806                labels: vec!["Person".into()],
4807                keys: vec!["o1".into()],
4808                visible_where: None,
4809                namespaces: None,
4810                write: None,
4811            }],
4812            ..Default::default()
4813        }
4814    }
4815
4816    fn fixture(name: &str, build: Build) -> (Router, SharedDb) {
4817        let dir = tmp(name);
4818        let db = SharedDb::open(&dir).unwrap();
4819        db.write().apply_schema(&roles()).unwrap();
4820        db.write().create_rule(rule()).unwrap();
4821        db.write().create_rule(weighted_rule()).unwrap();
4822        match build {
4823            Build::SeparateCommits => {
4824                write_ops(&db, first_half(), false);
4825                write_ops(&db, second_half(), false);
4826            }
4827            Build::OneBatch => {
4828                let mut ops = first_half();
4829                ops.extend(second_half());
4830                write_ops(&db, ops, true);
4831            }
4832            Build::Snapshotted => {
4833                write_ops(&db, first_half(), false);
4834                db.write().snapshot().unwrap();
4835                write_ops(&db, second_half(), false);
4836            }
4837            Build::Reopened => {
4838                write_ops(&db, first_half(), false);
4839                write_ops(&db, second_half(), false);
4840            }
4841            Build::Indexed => {
4842                db.write().enable_index("Person", "name").unwrap();
4843                db.write().enable_index("Org", "name").unwrap();
4844                write_ops(&db, first_half(), false);
4845                // Enough commits to fold, so the indexes and the first half are
4846                // in the fold and the second half is the tail.
4847                for i in 0..mushroomdb::FOLD_EVERY_K {
4848                    db.submit_batch(vec![BatchOp::InsertNode {
4849                        label: "Pad".into(),
4850                        key: format!("pad{i}"),
4851                        props: vec![],
4852                    }])
4853                    .unwrap();
4854                }
4855                write_ops(&db, second_half(), false);
4856                assert!(db.reader().queries_use_index(), "{name}: index unused");
4857            }
4858        }
4859        let db = match build {
4860            Build::Reopened => {
4861                drop(db);
4862                SharedDb::open(&dir).unwrap()
4863            }
4864            _ => db,
4865        };
4866        let app = router_with_role_tokens(
4867            db.clone(),
4868            Some(FULL.into()),
4869            HashMap::from([(ROLE.to_string(), "analyst".to_string())]),
4870        );
4871        (app, db)
4872    }
4873
4874    const KEYS: &[&str] = &["alice", "bob", "carol", "dave", "o1", "o2", "zed", "nobody"];
4875    const MASKS: &[(&str, bool)] = &[
4876        ("", false),
4877        ("alice,carol,o1", false),
4878        ("alice,carol,o1", true),
4879        ("dave", true),
4880        ("nobody", false),
4881    ];
4882
4883    async fn node_routes_match(app: &Router, db: &SharedDb, label: &str) -> usize {
4884        let mut cases = 0;
4885        for key in KEYS {
4886            for (mask, stub) in MASKS {
4887                let qs = if mask.is_empty() {
4888                    String::new()
4889                } else {
4890                    format!("?mask={mask}&stub_hidden={stub}")
4891                };
4892                let got = routed(app, get(&format!("/node/{key}{qs}"), Some(FULL))).await;
4893                let want = answer(legacy_node_info(db, key, mask, *stub)).await;
4894                assert_eq!(got, want, "{label}: GET /node/{key}{qs}");
4895
4896                let got = routed(app, get(&format!("/node/{key}/edges{qs}"), Some(FULL))).await;
4897                let want = answer(legacy_node_edges(db, key, mask, *stub)).await;
4898                assert_eq!(got, want, "{label}: GET /node/{key}/edges{qs}");
4899                cases += 2;
4900
4901                for (depth, dir, dir_name, types) in [
4902                    (1, Dir::Both, "both", None),
4903                    (2, Dir::Out, "out", None),
4904                    (3, Dir::In, "in", Some("KNOWS")),
4905                    (2, Dir::Both, "both", Some("WORKS_AT,LIKES")),
4906                ] {
4907                    let mut uri = format!("/node/{key}/neighborhood?depth={depth}&dir={dir_name}");
4908                    if let Some(t) = types {
4909                        uri.push_str(&format!("&edge_types={t}"));
4910                    }
4911                    if !mask.is_empty() {
4912                        uri.push_str(&format!("&mask={mask}&stub_hidden={stub}"));
4913                    }
4914                    let names: Option<Vec<&str>> = types.map(|t| t.split(',').collect());
4915                    let got = routed(app, get(&uri, Some(FULL))).await;
4916                    let want = answer(legacy_neighborhood(
4917                        db,
4918                        key,
4919                        depth,
4920                        names.as_deref(),
4921                        dir,
4922                        mask,
4923                        *stub,
4924                    ))
4925                    .await;
4926                    assert_eq!(got, want, "{label}: GET {uri}");
4927                    cases += 1;
4928                }
4929            }
4930        }
4931        cases
4932    }
4933
4934    const CYPHERS: &[&str] = &[
4935        "MATCH (n:Person {name: 'BOB'}) RETURN n.key, n.age",
4936        "MATCH (n:Person) WHERE n.name = 'CAROLINE' RETURN n.key",
4937        "MATCH (n:Person {name: 'CAROL'}) RETURN n.key",
4938        "MATCH (n:Person {name: 'ZED'}) RETURN n.key",
4939        "MATCH (n:Person {name: 'ERIN'}) RETURN n.key",
4940        "MATCH (o:Org {name: 'Acme'})<-[:WORKS_AT]-(p) RETURN p.key",
4941        "MATCH (n:Person) RETURN n.name ORDER BY n.name",
4942        "MATCH (n:Person)-[:KNOWS]->(m) RETURN n.name, m.name ORDER BY n.name, m.name",
4943        "MATCH (n:Person)-[:WORKS_AT]->(o:Org) RETURN n.name, o.name",
4944        "MATCH (n:Person)-[r:FITS]->(o:Org) RETURN n.name, o.name, r.score ORDER BY n.name, o.name",
4945        "MATCH (n:Person) WHERE n.age > 30 RETURN n.name, n.age",
4946        "MATCH (n:Person) RETURN count(n) AS c",
4947        "MATCH (n:Person) RETURN n.ns AS ns UNION MATCH (o:Org) RETURN o.ns AS ns",
4948        "MATCH (n:Person) RETURN n.org_id AS x UNION ALL MATCH (n:Person) RETURN n.org_id AS x",
4949        "MATCH (n:Person) RETURN n.name UNION MATCH (o:Org) RETURN o.key",
4950        "MATCH (n:Person RETURN n",
4951        "MATCH (n:Person) RETURN",
4952        "MATCH (n:Person) RETURN n.name UNION",
4953        "MATCH (n) RETURN nosuchfn(n.name)",
4954        "MATCH (n:Person) RETURN m.name",
4955        "MATCH (n) RETURN \u{1}",
4956        "MATCH (n:Person {key: 'zed'}) RETURN n",
4957        "CREATE (x:Person {key: 'mallory'})",
4958    ];
4959
4960    async fn query_routes_match(app: &Router, db: &SharedDb, label: &str) -> usize {
4961        let mut cases = 0;
4962        let masks: [Option<Vec<String>>; 3] = [
4963            None,
4964            Some(vec!["alice".into(), "carol".into(), "o2".into()]),
4965            Some(vec!["nobody".into()]),
4966        ];
4967        for cypher in CYPHERS {
4968            for mask in &masks {
4969                for namespace in [None, Some("tenant-a"), Some("no-such-ns")] {
4970                    for stub in [false, true] {
4971                        // A write without a mask or namespace is not a read route.
4972                        if cypher.starts_with("CREATE") && mask.is_none() && namespace.is_none() {
4973                            continue;
4974                        }
4975                        if stub && mask.is_none() && namespace.is_none() {
4976                            continue;
4977                        }
4978                        for format in ["json", ""] {
4979                            let mut body = json!({ "cypher": cypher });
4980                            if let Some(m) = mask {
4981                                body["mask"] = json!(m);
4982                            }
4983                            if let Some(ns) = namespace {
4984                                body["namespace"] = json!(ns);
4985                            }
4986                            if stub {
4987                                body["stub_hidden"] = json!(true);
4988                            }
4989                            let uri = if format.is_empty() {
4990                                "/query".to_string()
4991                            } else {
4992                                format!("/query?format={format}")
4993                            };
4994                            let got = routed(app, post(&uri, body.clone(), Some(FULL))).await;
4995                            let want = answer(legacy_query(
4996                                db,
4997                                cypher,
4998                                mask.as_deref(),
4999                                namespace,
5000                                stub,
5001                                format,
5002                            ))
5003                            .await;
5004                            assert_eq!(got, want, "{label}: POST {uri} {body}");
5005                            cases += 1;
5006                        }
5007                    }
5008                }
5009            }
5010        }
5011        cases
5012    }
5013
5014    async fn role_queries_match(app: &Router, db: &SharedDb, label: &str) -> usize {
5015        let mut cases = 0;
5016        let masks: [Option<Vec<String>>; 2] = [None, Some(vec!["alice".into(), "o1".into()])];
5017        for cypher in CYPHERS.iter().filter(|c| !c.starts_with("CREATE")) {
5018            for mask in &masks {
5019                for namespace in [None, Some("tenant-a")] {
5020                    let mut body = json!({ "cypher": cypher });
5021                    if let Some(m) = mask {
5022                        body["mask"] = json!(m);
5023                    }
5024                    if let Some(ns) = namespace {
5025                        body["namespace"] = json!(ns);
5026                    }
5027                    let got =
5028                        routed(app, post("/query?format=json", body.clone(), Some(ROLE))).await;
5029                    let want = answer(legacy_role_query(
5030                        db,
5031                        "analyst",
5032                        cypher,
5033                        mask.as_deref(),
5034                        namespace,
5035                    ))
5036                    .await;
5037                    assert_eq!(got, want, "{label}: role POST /query {body}");
5038                    cases += 1;
5039                }
5040            }
5041        }
5042        cases
5043    }
5044
5045    async fn all_routes_match(build: Build, name: &str) {
5046        let (app, db) = fixture(name, build);
5047        // The fixture must exercise what it claims to: a tail the reader applies.
5048        if !matches!(build, Build::Reopened) {
5049            assert!(
5050                !db.read().reader().deltas.is_empty(),
5051                "{name}: no delta tail"
5052            );
5053        }
5054        let label = format!("{build:?}");
5055        let n = node_routes_match(&app, &db, &label).await
5056            + query_routes_match(&app, &db, &label).await
5057            + role_queries_match(&app, &db, &label).await
5058            + counter_and_algo_routes_match(&app, &db, &label).await;
5059        assert!(n > 500, "{label}: only {n} cases ran");
5060    }
5061
5062    // ── /health, /metrics and /algo/* (moved in fix round 1) ──────────────────
5063
5064    fn legacy_health(db: &SharedDb) -> Response {
5065        let (nodes, edges) = {
5066            let g = db.read();
5067            let s = g.stats();
5068            (s.nodes_live, s.edges)
5069        };
5070        json_ok(json!({
5071            "ok": true,
5072            "nodes": nodes,
5073            "edges": edges,
5074            "addr": default_advertise_addr().to_string(),
5075        }))
5076    }
5077
5078    /// `/metrics` before TX-5, minus `rss_bytes` and `uptime_s`, which move
5079    /// between any two calls.
5080    fn legacy_metrics(db: &SharedDb) -> Js {
5081        let g = db.read();
5082        let s = g.stats();
5083        let slow = g.slow_query_snapshot();
5084        json!({
5085            "nodes_live": s.nodes_live,
5086            "nodes_tombstoned": s.nodes_tombstoned,
5087            "edges": s.edges,
5088            "commit_seq": g.commit_seq(),
5089            "wal_size_bytes": g.wal_size_bytes().ok(),
5090            "slow_queries": {
5091                "threshold_ms": slow.threshold_ms,
5092                "count": slow.count,
5093                "last": slow.last.iter().map(|e| json!({
5094                    "ms": e.ms, "query": e.query, "at_commit": e.at_commit,
5095                })).collect::<Vec<_>>(),
5096            },
5097        })
5098    }
5099
5100    fn legacy_algo(db: &SharedDb, route: &str, body: &Js) -> Response {
5101        let g = db.read();
5102        let v = match route {
5103            "pagerank" => {
5104                serde_json::to_value(g.pagerank(&serde_json::from_value(body.clone()).unwrap()))
5105            }
5106            "wcc" => serde_json::to_value(
5107                g.connected_components(&serde_json::from_value(body.clone()).unwrap()),
5108            ),
5109            _ => serde_json::to_value(
5110                g.degree_centrality(&serde_json::from_value(body.clone()).unwrap()),
5111            ),
5112        };
5113        json_ok(v.unwrap_or_else(|_| json!({})))
5114    }
5115
5116    async fn counter_and_algo_routes_match(app: &Router, db: &SharedDb, label: &str) -> usize {
5117        let mut cases = 0;
5118        let got = routed(app, get("/health", Some(FULL))).await;
5119        assert_eq!(got, answer(legacy_health(db)).await, "{label}: /health");
5120        let got = routed(app, get("/health", None)).await;
5121        assert_eq!(got.body, br#"{"ok":true}"#, "{label}: anonymous /health");
5122
5123        let got = routed(app, get("/metrics", Some(FULL))).await;
5124        assert_eq!(got.status, StatusCode::OK);
5125        let mut v: Js = serde_json::from_slice(&got.body).unwrap();
5126        let obj = v.as_object_mut().unwrap();
5127        assert!(obj.remove("rss_bytes").is_some() && obj.remove("uptime_s").is_some());
5128        assert_eq!(v, legacy_metrics(db), "{label}: /metrics");
5129        cases += 3;
5130
5131        for route in ["pagerank", "wcc", "degree"] {
5132            for body in [
5133                json!({}),
5134                json!({"edge_type": "KNOWS"}),
5135                json!({"edge_type": "FITS"}),
5136            ] {
5137                let got = routed(
5138                    app,
5139                    post(&format!("/algo/{route}"), body.clone(), Some(FULL)),
5140                )
5141                .await;
5142                let want = answer(legacy_algo(db, route, &body)).await;
5143                assert_eq!(got, want, "{label}: /algo/{route} {body}");
5144                cases += 1;
5145            }
5146        }
5147        cases
5148    }
5149
5150    #[tokio::test]
5151    async fn reader_routes_match_the_read_lock_separate_commits() {
5152        all_routes_match(Build::SeparateCommits, "separate").await;
5153    }
5154
5155    #[tokio::test]
5156    async fn reader_routes_match_the_read_lock_one_batch() {
5157        all_routes_match(Build::OneBatch, "batch").await;
5158    }
5159
5160    #[tokio::test]
5161    async fn reader_routes_match_the_read_lock_snapshotted() {
5162        all_routes_match(Build::Snapshotted, "snapshotted").await;
5163    }
5164
5165    #[tokio::test]
5166    async fn reader_routes_match_the_read_lock_reopened() {
5167        all_routes_match(Build::Reopened, "reopened").await;
5168    }
5169
5170    /// The cases are only a differential if the two sides can disagree: `UNION` on
5171    /// the role path, which the reader refused before TX-5, now answers rows.
5172    #[tokio::test]
5173    async fn union_answers_on_the_reader_for_both_identities() {
5174        let (app, _db) = fixture("union", Build::SeparateCommits);
5175        let cypher = "MATCH (n:Person) RETURN n.ns AS ns UNION MATCH (o:Org) RETURN o.ns AS ns";
5176        for token in [FULL, ROLE] {
5177            let got = routed(
5178                &app,
5179                post(
5180                    "/query?format=json",
5181                    json!({ "cypher": cypher }),
5182                    Some(token),
5183                ),
5184            )
5185            .await;
5186            assert_eq!(got.status, StatusCode::OK, "{token}: {:?}", got.body);
5187            let v: Js = serde_json::from_slice(&got.body).unwrap();
5188            assert!(
5189                !v["rows"].as_array().unwrap().is_empty(),
5190                "{token}: UNION returned no rows: {v}"
5191            );
5192        }
5193    }
5194
5195    /// A store whose indexes are in the fold, with a tail that moves, deletes
5196    /// and adds indexed values: the reader serves the lookups from the fold's
5197    /// index layered with the tail and answers what the read lock answered.
5198    #[tokio::test]
5199    async fn reader_routes_match_the_read_lock_indexed() {
5200        all_routes_match(Build::Indexed, "indexed").await;
5201    }
5202
5203    /// An unmasked reader query still lands in the `/metrics` slow-query ring.
5204    #[tokio::test]
5205    async fn a_slow_reader_query_is_still_recorded() {
5206        let (app, db) = fixture("slow", Build::SeparateCommits);
5207        let bulk = (0..5_000)
5208            .map(|i| BatchOp::InsertNode {
5209                label: "Bulk".into(),
5210                key: format!("bulk{i}"),
5211                props: vec![("n".into(), Value::Int(i))],
5212            })
5213            .collect();
5214        db.submit_batch(bulk).unwrap();
5215        let before = db.read().slow_query_snapshot().count;
5216        // Threshold 1 ms: a filtered scan of 5 000 nodes reaches it, and the count
5217        // is what is asserted, not the duration.
5218        db.write().set_slow_query_threshold_ms(1);
5219        let cypher = "MATCH (n:Bulk) WHERE n.n >= 0 RETURN count(n) AS c";
5220        let mut recorded = false;
5221        for _ in 0..20 {
5222            let got = routed(
5223                &app,
5224                post(
5225                    "/query?format=json",
5226                    json!({ "cypher": cypher }),
5227                    Some(FULL),
5228                ),
5229            )
5230            .await;
5231            assert_eq!(got.status, StatusCode::OK);
5232            if db.read().slow_query_snapshot().count > before {
5233                recorded = true;
5234                break;
5235            }
5236        }
5237        assert!(recorded, "a slow reader query never reached the ring");
5238        let last = db.read().slow_query_snapshot().last;
5239        assert_eq!(last.last().map(|e| e.query.as_str()), Some(cypher));
5240    }
5241}