nodedb 0.4.0

Local-first, real-time, edge-to-cloud hybrid database for multi-modal workloads
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
// SPDX-License-Identifier: BUSL-1.1

//! Integration coverage: HTTP-layer authentication invariant.
//!
//! The invariant: every HTTP handler that performs tenant-scoped work,
//! admin work, or any write to shared state must go through identity
//! resolution before acting, and must source `tenant_id` from the
//! resolved identity rather than from request parameters.
//!
//! This file captures the full surface of handlers that historically
//! violated the invariant:
//!   * `/ws` (WebSocket RPC) hardcoded `tenant_id = 1`
//!   * `/v1/streams/{s}/poll`, `/v1/streams/{s}/events` read
//!     `tenant_id` from the query string
//!   * `/v1/cdc/{collection}`, `/v1/cdc/{collection}/poll` same
//!   * `/cluster/status` performed no auth at all (admin info leak)
//!   * `/obsv/api/v1/*` (PromQL) performed no auth; remote_write
//!     hardcoded `tenant_id = 1`
//!   * `wasm_upload::upload_wasm` hardcoded `tenant_id = 0` and wrote
//!     to the catalog regardless of caller
//!   * `subscribe.rs` hardcoded `TenantId::new(1)` in its default
//!     subscription path
//!
//! Every assertion is written against the correct spec: under non-Trust
//! auth modes, every such route returns 401 without a bearer token, and
//! under any mode, a `tenant_id` passed via query string must not be
//! honoured in place of the caller's identity.

use std::sync::Arc;
use std::time::Duration;

use futures::{SinkExt, StreamExt};
use nodedb::bridge::dispatch::Dispatcher;
use nodedb::config::auth::AuthMode;
use nodedb::control::state::SharedState;
use nodedb::wal::WalManager;
use tokio_tungstenite::tungstenite::Message;

struct TestServer {
    local_addr: std::net::SocketAddr,
    shared: Arc<SharedState>,
    _server: tokio::task::JoinHandle<()>,
    _dir: tempfile::TempDir,
}

async fn start_http(auth_mode: AuthMode) -> TestServer {
    let dir = tempfile::tempdir().unwrap();
    let wal = Arc::new(WalManager::open_for_testing(&dir.path().join("auth.wal")).unwrap());
    let (dispatcher, _data_sides) = Dispatcher::new(1, 64);
    let shared = SharedState::new(dispatcher, wal).unwrap();
    if auth_mode == AuthMode::Trust {
        shared
            .credentials
            .bootstrap_trust_superuser("nodedb")
            .expect("bootstrap trust superuser");
    }

    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
    let local_addr = listener.local_addr().unwrap();

    let (bus, _) = nodedb::control::shutdown::ShutdownBus::new(Arc::clone(&shared.shutdown));
    let shared_http = Arc::clone(&shared);
    let handle = tokio::spawn(async move {
        nodedb::control::server::http::server::run_with_listener(
            listener,
            shared_http,
            auth_mode,
            None,
            bus,
        )
        .await
        .ok();
    });

    tokio::time::sleep(Duration::from_millis(50)).await;

    TestServer {
        local_addr,
        shared,
        _server: handle,
        _dir: dir,
    }
}

fn is_unauthorized_ish(status: reqwest::StatusCode) -> bool {
    status == reqwest::StatusCode::UNAUTHORIZED
        || status == reqwest::StatusCode::FORBIDDEN
        || status == reqwest::StatusCode::BAD_REQUEST
}

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn http_trust_auth_uses_configured_durable_identity() {
    let srv = start_http(AuthMode::Trust).await;
    let app_state = nodedb::control::server::http::auth::AppState {
        shared: Arc::clone(&srv.shared),
        auth_mode: AuthMode::Trust,
        query_ctx: Arc::new(nodedb::control::planner::context::QueryContext::for_state(
            &srv.shared,
        )),
    };

    let identity = nodedb::control::server::http::auth::resolve_identity(
        &axum::http::HeaderMap::new(),
        &app_state,
        "127.0.0.1:1",
    )
    .expect("configured HTTP trust identity");

    assert_eq!(identity.username, "nodedb");
    assert_ne!(identity.user_id, 0, "trust identity must be durable");
    assert!(identity.is_superuser);
}

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn http_trust_auth_rejects_without_configured_identity() {
    let dir = tempfile::tempdir().expect("tempdir");
    let wal = Arc::new(
        WalManager::open_for_testing(&dir.path().join("unconfigured-auth.wal")).expect("open WAL"),
    );
    let (dispatcher, _data_sides) = Dispatcher::new(1, 64);
    let shared = SharedState::new(dispatcher, wal).expect("shared state");
    let app_state = nodedb::control::server::http::auth::AppState {
        shared: Arc::clone(&shared),
        auth_mode: AuthMode::Trust,
        query_ctx: Arc::new(nodedb::control::planner::context::QueryContext::for_state(
            &shared,
        )),
    };

    let result = nodedb::control::server::http::auth::resolve_identity(
        &axum::http::HeaderMap::new(),
        &app_state,
        "127.0.0.1:1",
    );

    assert!(
        result.is_err(),
        "HTTP trust auth must fail closed before identity bootstrap"
    );
}

// ─── /v1/streams/{s}/poll ────────────────────────────────────────────────────

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn stream_poll_rejects_missing_bearer_token() {
    let srv = start_http(AuthMode::Password).await;
    let url = format!("http://{}/v1/streams/s1/poll?group=g1", srv.local_addr);
    let resp = reqwest::Client::new().get(&url).send().await.unwrap();
    assert_eq!(
        resp.status(),
        reqwest::StatusCode::UNAUTHORIZED,
        "poll_stream must require a bearer token under non-Trust auth modes"
    );
}

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn stream_poll_rejects_cross_tenant_tenant_id_parameter() {
    let srv = start_http(AuthMode::Trust).await;
    let url = format!(
        "http://{}/v1/streams/s1/poll?group=g1&tenant_id=42",
        srv.local_addr
    );
    let resp = reqwest::Client::new().get(&url).send().await.unwrap();
    assert!(
        is_unauthorized_ish(resp.status()),
        "poll_stream must reject a query-string tenant_id that does not match identity; got {}",
        resp.status()
    );
}

// ─── /v1/streams/{s}/events (SSE) ────────────────────────────────────────────

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn stream_sse_rejects_missing_bearer_token() {
    let srv = start_http(AuthMode::Password).await;
    let url = format!("http://{}/v1/streams/s1/events?group=g1", srv.local_addr);
    let resp = reqwest::Client::new().get(&url).send().await.unwrap();
    assert_eq!(
        resp.status(),
        reqwest::StatusCode::UNAUTHORIZED,
        "stream_sse must require a bearer token under non-Trust auth modes"
    );
}

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn stream_sse_rejects_cross_tenant_tenant_id_parameter() {
    let srv = start_http(AuthMode::Trust).await;
    let url = format!(
        "http://{}/v1/streams/s1/events?group=g1&tenant_id=42",
        srv.local_addr
    );
    let resp = reqwest::Client::new().get(&url).send().await.unwrap();
    assert!(
        is_unauthorized_ish(resp.status()),
        "stream_sse must reject a query-string tenant_id that does not match identity; got {}",
        resp.status()
    );
    assert_eq!(
        srv.shared
            .consumer_assignments
            .consumer_count(42, "s1", "g1"),
        0,
        "stream_sse must not register a consumer for a tenant the caller cannot act as"
    );
}

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn stream_sse_unauthenticated_request_does_not_claim_consumer_slot() {
    let srv = start_http(AuthMode::Password).await;
    let url = format!(
        "http://{}/v1/streams/s1/events?group=g1&tenant_id=42",
        srv.local_addr
    );
    let _ = tokio::time::timeout(
        Duration::from_millis(500),
        reqwest::Client::new().get(&url).send(),
    )
    .await;
    tokio::time::sleep(Duration::from_millis(50)).await;

    assert_eq!(
        srv.shared
            .consumer_assignments
            .consumer_count(42, "s1", "g1"),
        0,
        "an unauthenticated SSE request must not register a consumer — DoS guard"
    );
}

// ─── /ws (WebSocket RPC) ─────────────────────────────────────────────────────

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn ws_upgrade_refused_without_bearer_token() {
    // The only way to guarantee that an unauthenticated caller cannot execute
    // SQL as tenant 1 is to refuse the WebSocket upgrade before any handler
    // state is attached. Post-upgrade "reject the first message" is not
    // sufficient: the handler is already running, the tenant is already pinned.
    let srv = start_http(AuthMode::Password).await;
    let url = format!("ws://{}/v1/ws", srv.local_addr);
    let result = tokio_tungstenite::connect_async(&url).await;
    assert!(
        result.is_err(),
        "WS upgrade must be refused under non-Trust auth modes when no Bearer \
         token is presented; got a successful upgrade"
    );
}

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn ws_live_on_unauthenticated_socket_registers_no_subscription() {
    // Defence in depth: even if the upgrade somehow succeeds (trust-mode-like
    // escape hatch, future middleware bug), a `live` method without prior
    // successful `auth` must not register a change-stream subscription.
    let srv = start_http(AuthMode::Password).await;
    let url = format!("ws://{}/v1/ws", srv.local_addr);
    let Ok((mut ws, _)) = tokio_tungstenite::connect_async(&url).await else {
        // Upgrade refused — upstream guard caught it. Subscription count
        // must trivially still be zero.
        assert_eq!(srv.shared.change_stream.subscriber_count(), 0);
        return;
    };
    let req = serde_json::json!({
        "id": 1, "method": "live", "params": {"sql": "LIVE SELECT * FROM orders"}
    })
    .to_string();
    let _ = ws.send(Message::Text(req.into())).await;
    let _ = tokio::time::timeout(Duration::from_millis(300), ws.next()).await;

    assert_eq!(
        srv.shared.change_stream.subscriber_count(),
        0,
        "an unauthenticated `live` request must not register a change-stream subscription"
    );
}

// ─── /v1/cdc/{collection} (SSE) and /v1/cdc/{collection}/poll ───────────────

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn cdc_sse_stream_rejects_missing_bearer_token() {
    let srv = start_http(AuthMode::Password).await;
    let url = format!("http://{}/v1/cdc/orders", srv.local_addr);
    let resp = reqwest::Client::new().get(&url).send().await.unwrap();
    assert_eq!(
        resp.status(),
        reqwest::StatusCode::UNAUTHORIZED,
        "cdc::sse_stream must require a bearer token under non-Trust auth modes"
    );
}

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn cdc_sse_stream_rejects_cross_tenant_tenant_id_parameter() {
    let srv = start_http(AuthMode::Trust).await;
    let url = format!("http://{}/v1/cdc/orders?tenant_id=42", srv.local_addr);
    let resp = reqwest::Client::new().get(&url).send().await.unwrap();
    assert!(
        is_unauthorized_ish(resp.status()),
        "cdc::sse_stream must reject a query-string tenant_id mismatch; got {}",
        resp.status()
    );
}

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn cdc_poll_changes_rejects_missing_bearer_token() {
    let srv = start_http(AuthMode::Password).await;
    let url = format!("http://{}/v1/cdc/orders/poll", srv.local_addr);
    let resp = reqwest::Client::new().get(&url).send().await.unwrap();
    assert_eq!(
        resp.status(),
        reqwest::StatusCode::UNAUTHORIZED,
        "cdc::poll_changes must require a bearer token under non-Trust auth modes"
    );
}

// ─── /v1/cluster/status (admin info leak) ───────────────────────────────────

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn cluster_status_rejects_missing_bearer_token() {
    let srv = start_http(AuthMode::Password).await;
    let url = format!("http://{}/v1/cluster/status", srv.local_addr);
    let resp = reqwest::Client::new().get(&url).send().await.unwrap();
    assert_eq!(
        resp.status(),
        reqwest::StatusCode::UNAUTHORIZED,
        "cluster_status must require a bearer token — admin metadata must not leak to \
         unauthenticated callers"
    );
}

// ─── /v1/obsv/api/v1/* (PromQL) ──────────────────────────────────────────────

mod promql {
    use super::*;

    async fn expect_401(path: &str, method: reqwest::Method) {
        let srv = start_http(AuthMode::Password).await;
        let url = format!("http://{}{}", srv.local_addr, path);
        let resp = reqwest::Client::new()
            .request(method, &url)
            .send()
            .await
            .unwrap();
        assert_eq!(
            resp.status(),
            reqwest::StatusCode::UNAUTHORIZED,
            "{path} must require a bearer token under non-Trust auth modes"
        );
    }

    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
    async fn promql_instant_query_rejects_missing_bearer() {
        expect_401("/v1/obsv/api/v1/query?query=up", reqwest::Method::GET).await;
    }

    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
    async fn promql_range_query_rejects_missing_bearer() {
        expect_401(
            "/v1/obsv/api/v1/query_range?query=up&start=0&end=1&step=1",
            reqwest::Method::GET,
        )
        .await;
    }

    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
    async fn promql_series_rejects_missing_bearer() {
        expect_401("/v1/obsv/api/v1/series", reqwest::Method::GET).await;
    }

    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
    async fn promql_label_names_rejects_missing_bearer() {
        expect_401("/v1/obsv/api/v1/labels", reqwest::Method::GET).await;
    }

    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
    async fn promql_label_values_rejects_missing_bearer() {
        expect_401("/v1/obsv/api/v1/label/job/values", reqwest::Method::GET).await;
    }

    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
    async fn promql_metadata_rejects_missing_bearer() {
        expect_401("/v1/obsv/api/v1/metadata", reqwest::Method::GET).await;
    }

    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
    async fn promql_remote_write_rejects_missing_bearer() {
        // remote_write is the worst: hardcodes tenant=1 AND accepts writes.
        expect_401("/v1/obsv/api/v1/write", reqwest::Method::POST).await;
    }

    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
    async fn promql_remote_read_rejects_missing_bearer() {
        expect_401("/v1/obsv/api/v1/read", reqwest::Method::POST).await;
    }

    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
    async fn promql_annotations_rejects_missing_bearer() {
        expect_401("/v1/obsv/api/v1/annotations", reqwest::Method::POST).await;
    }
}

// ─── Latent handlers (bug baked into source, currently unmounted) ────────────
//
// `wasm_upload::upload_wasm` hardcodes `tenant_id = 0u32` (line 39) and
// `subscribe.rs` hardcodes `TenantId::new(1)` in its default subscription
// sites (lines 188, 205). Both handlers violate the same invariant as the
// mounted routes above; they are simply not yet wired into the production
// router. The spec these tests pin: neither handler may contain a
// hardcoded tenant literal — `tenant_id` must come from a resolved
// identity.
//
// These tests inspect the source text because the handlers aren't
// reachable through the production HTTP stack. If they are ever wired in
// without fixing the hardcoded identity, the mounted-route tests above
// and these source-level guards fire together.

#[test]
fn wasm_upload_handler_does_not_hardcode_tenant_identity() {
    let src = std::fs::read_to_string(concat!(
        env!("CARGO_MANIFEST_DIR"),
        "/src/control/server/http/routes/wasm_upload.rs"
    ))
    .expect("read wasm_upload.rs");
    assert!(
        !src.contains("let tenant_id = 0u32"),
        "upload_wasm must source tenant_id from a resolved identity, not a hardcoded 0u32"
    );
    assert!(
        src.contains("resolve_identity") || src.contains("resolve_auth"),
        "upload_wasm must call resolve_identity/resolve_auth before writing to the catalog"
    );
}