reddb_server/
api.rs

1//! Public API layer for the RedDB crate.
2//!
3//! This module is the first layer to consume from applications:
4//! - stable options and contracts
5//! - capability declarations
6//! - typed errors and lightweight metadata snapshots
7//! - cross-layer traits for catalog/operations observability
8
9use std::collections::{BTreeMap, BTreeSet};
10use std::fmt;
11use std::io;
12use std::path::{Path, PathBuf};
13use std::sync::Arc;
14use std::time::{SystemTime, UNIX_EPOCH};
15
16use crate::auth::AuthConfig;
17use crate::replication::ReplicationConfig;
18
19pub const DEFAULT_SNAPSHOT_RETENTION: usize = 16;
20pub const DEFAULT_EXPORT_RETENTION: usize = 16;
21
22pub const REDDB_PROTOCOL_VERSION: &str = "reddb-v2";
23pub const REDDB_FORMAT_VERSION: u32 = 2;
24/// Default group-commit window.
25///
26/// `0` = "no wait" — the background flusher fsyncs as soon as any
27/// writer's pending commit arrives. Under single-writer workloads a
28/// non-zero window would be pure latency (no one to batch with),
29/// capping individual commit throughput at ~1000/s for window=1.
30/// Concurrent writers still batch naturally via the `Mutex<WalWriter>`
31/// contention path without needing an explicit timer.
32///
33/// Operators with many concurrent clients can raise this (e.g. 1-5ms)
34/// to amortise fsync cost across a bigger batch — at the cost of p99
35/// tail latency going up by the window size.
36pub const DEFAULT_GROUP_COMMIT_WINDOW_MS: u64 = 0;
37pub const DEFAULT_GROUP_COMMIT_MAX_STATEMENTS: usize = 128;
38pub const DEFAULT_GROUP_COMMIT_MAX_WAL_BYTES: u64 = 1024 * 1024;
39pub(crate) const EPHEMERAL_RUNTIME_METADATA_KEY: &str = "__reddb_ephemeral_runtime";
40
41pub type RedDBResult<T> = Result<T, RedDBError>;
42
43#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)]
44pub enum StorageMode {
45    /// Durable, file-backed database with WAL + checkpointing.
46    #[default]
47    Persistent,
48}
49
50impl StorageMode {
51    pub const fn is_persistent(self) -> bool {
52        matches!(self, Self::Persistent)
53    }
54}
55
56#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)]
57pub enum DurabilityMode {
58    #[default]
59    Strict,
60    WalDurableGrouped,
61    /// Fire-and-forget. Writers return as soon as the WAL record is
62    /// in the in-memory ring buffer — they do NOT wait for fsync.
63    /// The group-commit background thread still flushes on its
64    /// cadence, so committed-but-unflushed work is bounded by
65    /// `GroupCommitOptions::window_ms`. A crash inside the window
66    /// drops whatever wasn't flushed — matches PG's
67    /// `synchronous_commit=off` contract.
68    Async,
69}
70
71impl DurabilityMode {
72    pub const fn as_str(self) -> &'static str {
73        match self {
74            Self::Strict => "strict",
75            Self::WalDurableGrouped => "wal_durable_grouped",
76            Self::Async => "async",
77        }
78    }
79
80    pub fn from_str(value: &str) -> Option<Self> {
81        let normalized = value.trim().to_ascii_lowercase();
82        match normalized.as_str() {
83            // Legacy / opt-out form. Every commit pays its own fsync.
84            "strict" => Some(Self::Strict),
85            // Group-commit sync path — the perf-parity default. Matches
86            // PostgreSQL's `synchronous_commit=on` behaviour: the
87            // writer waits for durability, but fsyncs are batched
88            // across concurrent writers so a burst of N commits pays
89            // ~O(1) fsyncs instead of O(N).
90            "sync"
91            | "wal_durable_grouped"
92            | "wal-durable-grouped"
93            | "grouped"
94            | "wal_grouped"
95            | "wal-grouped" => Some(Self::WalDurableGrouped),
96            // Fire-and-forget async: writers return as soon as the
97            // WAL buffer accepts the record; background flusher runs
98            // on its configured cadence.
99            "async" | "fire_and_forget" | "fire-and-forget" => Some(Self::Async),
100            _ => None,
101        }
102    }
103}
104
105#[derive(Debug, Clone, Copy, PartialEq, Eq)]
106pub struct GroupCommitOptions {
107    pub window_ms: u64,
108    pub max_statements: usize,
109    pub max_wal_bytes: u64,
110}
111
112impl Default for GroupCommitOptions {
113    fn default() -> Self {
114        Self {
115            window_ms: DEFAULT_GROUP_COMMIT_WINDOW_MS,
116            max_statements: DEFAULT_GROUP_COMMIT_MAX_STATEMENTS,
117            max_wal_bytes: DEFAULT_GROUP_COMMIT_MAX_WAL_BYTES,
118        }
119    }
120}
121
122#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
123pub enum Capability {
124    /// Structured row storage.
125    Table,
126    /// Graph nodes/edges.
127    Graph,
128    /// Vector collections and ANN search.
129    Vector,
130    /// Full-text / lexical search.
131    FullText,
132    /// Text/metadata security and enrichment modules.
133    Security,
134    /// Encryption at rest.
135    Encryption,
136}
137
138impl Capability {
139    pub const fn as_str(self) -> &'static str {
140        match self {
141            Self::Table => "table",
142            Self::Graph => "graph",
143            Self::Vector => "vector",
144            Self::FullText => "fulltext",
145            Self::Security => "security",
146            Self::Encryption => "encryption",
147        }
148    }
149}
150
151#[derive(Debug, Clone, Default)]
152pub struct CapabilitySet {
153    items: BTreeSet<Capability>,
154}
155
156impl CapabilitySet {
157    pub fn new() -> Self {
158        Self::default()
159    }
160
161    pub fn with(mut self, capability: Capability) -> Self {
162        self.items.insert(capability);
163        self
164    }
165
166    pub fn with_all(mut self, capabilities: &[Capability]) -> Self {
167        capabilities.iter().copied().for_each(|capability| {
168            self.items.insert(capability);
169        });
170        self
171    }
172
173    pub fn has(&self, capability: Capability) -> bool {
174        self.items.contains(&capability)
175    }
176
177    pub fn as_slice(&self) -> Vec<Capability> {
178        self.items.iter().copied().collect()
179    }
180}
181
182pub struct RedDBOptions {
183    pub mode: StorageMode,
184    pub data_path: Option<PathBuf>,
185    pub read_only: bool,
186    pub create_if_missing: bool,
187    pub verify_checksums: bool,
188    pub durability_mode: DurabilityMode,
189    pub group_commit: GroupCommitOptions,
190    pub auto_checkpoint_pages: u32,
191    /// Maximum hypertable chunks the checkpoint path may seal into derived
192    /// columnar artifacts per checkpoint. `usize::MAX` means no practical cap.
193    pub checkpoint_columnar_emission_budget_chunks: usize,
194    /// Minimum rows a sealed in-scope chunk must carry before the derived
195    /// columnar projection is materialized.
196    pub columnar_projection_size_floor_rows: usize,
197    pub cache_pages: usize,
198    pub snapshot_retention: usize,
199    pub export_retention: usize,
200    pub feature_gates: CapabilitySet,
201    pub force_create: bool,
202    pub metadata: BTreeMap<String, String>,
203    /// Optional remote storage backend for snapshot transport.
204    pub remote_backend: Option<Arc<dyn crate::storage::backend::RemoteBackend>>,
205    /// Optional CAS-capable handle to the same backend, populated by
206    /// the factory when the configured backend implements
207    /// `AtomicRemoteBackend` (S3/local always; HTTP only when
208    /// `RED_HTTP_CONDITIONAL_WRITES=true`). `None` for backends that
209    /// do not provide compare-and-swap (Turso, D1, plain HTTP).
210    /// `LeaseStore` and any future CAS consumer pull from this field.
211    pub remote_backend_atomic: Option<Arc<dyn crate::storage::backend::AtomicRemoteBackend>>,
212    /// Remote object key used by the remote backend.
213    pub remote_key: Option<String>,
214    /// Replication configuration.
215    pub replication: ReplicationConfig,
216    /// Whether replica runtimes start the background replication loop.
217    pub replica_loop_enabled: bool,
218    /// Authentication & authorization configuration.
219    pub auth: AuthConfig,
220    /// Control Event Ledger configuration (issue #652). Read from
221    /// `REDDB_COMPLIANCE_MODE` at boot.
222    pub control_events: crate::runtime::control_events::ControlEventConfig,
223    /// Scoped data-plane query audit configuration. Disabled by
224    /// default; the regulated preset enables the stream without
225    /// adding catch-all rules.
226    pub query_audit: crate::runtime::query_audit::QueryAuditConfig,
227    /// Auto-create a HASH index on a user `id` column the first time a
228    /// row carrying that column is inserted into a collection. See
229    /// `UnifiedStoreConfig::auto_index_id`. Defaults to `true`; set to
230    /// `false` to opt out per workload (e.g. ingest pipelines that
231    /// don't need point-lookups by `id`).
232    pub auto_index_id: bool,
233    /// Tiered storage layout preset. Drives the defaults of the six
234    /// tier-flag toggles (`.meta.json` sidecar, seq-N catalog journal,
235    /// `-shm` provisioning, audit/slow log destinations,
236    /// `fold_dwb_into_wal`). Explicit per-feature setters still win over
237    /// the tier default. See `crate::storage::layout::StorageLayout`.
238    pub layout: crate::storage::layout::StorageLayout,
239    /// Per-feature overrides applied on top of the resolved layout preset.
240    pub layout_overrides: crate::storage::layout::LayoutOverrides,
241    /// Operator-facing storage/deploy profile contract. This records the
242    /// selected posture before the physical directory layout is expanded.
243    pub storage_profile: crate::storage::profile::StorageProfileSelection,
244    /// True only when the caller explicitly selected a layout via
245    /// `with_layout`. When false, `apply_tier_defaults` short-circuits so
246    /// pre-existing process-global toggles (set by tests / env hatches
247    /// before opening a runtime) are not clobbered by the implicit
248    /// `Standard` default. The new tier-driven behavior is opt-in.
249    pub layout_explicit: bool,
250    /// Explicit operator memory budget in bytes — the top tier of the ADR
251    /// 0073 §1 precedence chain. `None` defers to `REDDB_MEMORY_BUDGET`, the
252    /// deployment-profile default, the cgroup limit, and finally a fraction
253    /// of physical RAM. There is no "unlimited" value: `Some(0)` is rejected
254    /// at boot, exactly like an unparseable env var.
255    pub memory_budget_bytes: Option<u64>,
256}
257
258impl fmt::Debug for RedDBOptions {
259    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
260        let backend_name = self.remote_backend.as_ref().map(|b| b.name().to_string());
261        f.debug_struct("RedDBOptions")
262            .field("mode", &self.mode)
263            .field("data_path", &self.data_path)
264            .field("read_only", &self.read_only)
265            .field("create_if_missing", &self.create_if_missing)
266            .field("verify_checksums", &self.verify_checksums)
267            .field("durability_mode", &self.durability_mode)
268            .field("group_commit", &self.group_commit)
269            .field("auto_checkpoint_pages", &self.auto_checkpoint_pages)
270            .field(
271                "checkpoint_columnar_emission_budget_chunks",
272                &self.checkpoint_columnar_emission_budget_chunks,
273            )
274            .field(
275                "columnar_projection_size_floor_rows",
276                &self.columnar_projection_size_floor_rows,
277            )
278            .field("cache_pages", &self.cache_pages)
279            .field("snapshot_retention", &self.snapshot_retention)
280            .field("export_retention", &self.export_retention)
281            .field("feature_gates", &self.feature_gates)
282            .field("force_create", &self.force_create)
283            .field("metadata", &self.metadata)
284            .field("remote_backend", &backend_name)
285            .field("remote_key", &self.remote_key)
286            .field("replication", &self.replication)
287            .field("replica_loop_enabled", &self.replica_loop_enabled)
288            .field("auth", &self.auth)
289            .field("control_events", &self.control_events)
290            .field("query_audit", &self.query_audit)
291            .field("layout", &self.layout)
292            .field("layout_overrides", &self.layout_overrides)
293            .field("storage_profile", &self.storage_profile)
294            .field("memory_budget_bytes", &self.memory_budget_bytes)
295            .finish()
296    }
297}
298
299impl Clone for RedDBOptions {
300    fn clone(&self) -> Self {
301        Self {
302            mode: self.mode,
303            data_path: self.data_path.clone(),
304            read_only: self.read_only,
305            create_if_missing: self.create_if_missing,
306            verify_checksums: self.verify_checksums,
307            durability_mode: self.durability_mode,
308            group_commit: self.group_commit,
309            auto_checkpoint_pages: self.auto_checkpoint_pages,
310            checkpoint_columnar_emission_budget_chunks: self
311                .checkpoint_columnar_emission_budget_chunks,
312            columnar_projection_size_floor_rows: self.columnar_projection_size_floor_rows,
313            cache_pages: self.cache_pages,
314            snapshot_retention: self.snapshot_retention,
315            export_retention: self.export_retention,
316            feature_gates: self.feature_gates.clone(),
317            force_create: self.force_create,
318            metadata: self.metadata.clone(),
319            remote_backend: self.remote_backend.clone(),
320            remote_backend_atomic: self.remote_backend_atomic.clone(),
321            remote_key: self.remote_key.clone(),
322            replication: self.replication.clone(),
323            replica_loop_enabled: self.replica_loop_enabled,
324            auth: self.auth.clone(),
325            control_events: self.control_events,
326            query_audit: self.query_audit.clone(),
327            auto_index_id: self.auto_index_id,
328            layout: self.layout,
329            layout_overrides: self.layout_overrides.clone(),
330            storage_profile: self.storage_profile,
331            layout_explicit: self.layout_explicit,
332            memory_budget_bytes: self.memory_budget_bytes,
333        }
334    }
335}
336
337impl Default for RedDBOptions {
338    fn default() -> Self {
339        Self {
340            mode: StorageMode::Persistent,
341            data_path: None,
342            read_only: false,
343            create_if_missing: true,
344            verify_checksums: true,
345            // Perf-parity default — `WalDurableGrouped` matches
346            // PostgreSQL's `synchronous_commit=on` behaviour while
347            // amortising fsync cost across concurrent writers. The
348            // legacy `Strict` tier (per-commit fsync) stays available
349            // via `durability.mode = "strict"` / `REDDB_DURABILITY=strict`.
350            durability_mode: DurabilityMode::WalDurableGrouped,
351            group_commit: GroupCommitOptions::default(),
352            auto_checkpoint_pages: 1000,
353            checkpoint_columnar_emission_budget_chunks: usize::MAX,
354            columnar_projection_size_floor_rows: 4,
355            cache_pages: 10_000,
356            snapshot_retention: DEFAULT_SNAPSHOT_RETENTION,
357            export_retention: DEFAULT_EXPORT_RETENTION,
358            feature_gates: CapabilitySet::new()
359                .with(Capability::Table)
360                .with(Capability::Graph)
361                .with(Capability::Vector),
362            force_create: true,
363            metadata: BTreeMap::new(),
364            remote_backend: None,
365            remote_backend_atomic: None,
366            remote_key: None,
367            replication: ReplicationConfig::standalone(),
368            replica_loop_enabled: true,
369            auth: AuthConfig::default(),
370            control_events: crate::runtime::control_events::ControlEventConfig::default(),
371            query_audit: crate::runtime::query_audit::QueryAuditConfig::default(),
372            auto_index_id: true,
373            layout: crate::storage::layout::StorageLayout::default(),
374            layout_overrides: crate::storage::layout::LayoutOverrides::default(),
375            storage_profile: crate::storage::profile::StorageProfileSelection::embedded_single_file(
376            ),
377            layout_explicit: false,
378            memory_budget_bytes: None,
379        }
380    }
381}
382
383impl RedDBOptions {
384    pub fn persistent<P: Into<PathBuf>>(path: P) -> Self {
385        Self {
386            mode: StorageMode::Persistent,
387            data_path: Some(path.into()),
388            ..Default::default()
389        }
390    }
391
392    /// Ephemeral, tempfile-backed database.
393    ///
394    /// The underlying storage is a real persistent file placed under the system
395    /// temp directory with a unique name — there is no longer a true in-memory
396    /// execution mode. Prefer [`RedDBOptions::persistent`] when the data should
397    /// outlive the process.
398    pub fn in_memory() -> Self {
399        static NEXT_EPHEMERAL_ID: std::sync::atomic::AtomicU64 =
400            std::sync::atomic::AtomicU64::new(0);
401
402        let now_nanos = std::time::SystemTime::now()
403            .duration_since(std::time::UNIX_EPOCH)
404            .map(|duration| duration.as_nanos())
405            .unwrap_or(0);
406        let unique = NEXT_EPHEMERAL_ID.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
407        // Each ephemeral runtime gets its OWN directory, not just a unique
408        // filename in the shared temp dir. Sibling artifacts (the audit log,
409        // WAL, snapshots) are derived from this data path's PARENT, so a
410        // shared parent collapses every ephemeral runtime's `.audit.log`
411        // onto one file — which parallel test processes (nextest runs each
412        // test in its own process, frequently under one shared `TMPDIR`)
413        // then truncate/remove out from under one another. A per-instance
414        // directory keeps each runtime's siblings self-contained.
415        let dir = std::env::temp_dir().join(format!(
416            "reddb-ephemeral-{}-{}-{}",
417            std::process::id(),
418            now_nanos,
419            unique
420        ));
421        let _ = std::fs::create_dir_all(&dir);
422        let path = dir.join("db.rdb");
423        let _ = std::fs::remove_file(&path);
424        let mut metadata = BTreeMap::new();
425        metadata.insert(
426            EPHEMERAL_RUNTIME_METADATA_KEY.to_string(),
427            "true".to_string(),
428        );
429        Self {
430            mode: StorageMode::Persistent,
431            data_path: Some(path),
432            auto_checkpoint_pages: 0,
433            checkpoint_columnar_emission_budget_chunks: usize::MAX,
434            columnar_projection_size_floor_rows: 4,
435            cache_pages: 2_000,
436            snapshot_retention: DEFAULT_SNAPSHOT_RETENTION,
437            export_retention: DEFAULT_EXPORT_RETENTION,
438            read_only: false,
439            force_create: true,
440            metadata,
441            ..Default::default()
442        }
443    }
444
445    pub fn with_mode(mut self, mode: StorageMode) -> Self {
446        self.mode = mode;
447        self
448    }
449
450    pub fn with_data_path<P: Into<PathBuf>>(mut self, path: P) -> Self {
451        // `in_memory()` eagerly creates a `reddb-ephemeral-*` dir and points
452        // `data_path` at it. Overriding the path abandons that dir — and the
453        // per-runtime cleanup keys off the FINAL `data_path`, so the orphan
454        // would leak in TMPDIR (caught by scripts/check-temp-residue.sh). Remove
455        // it now if the previous path was such an eagerly-created ephemeral dir.
456        if let Some(old) = self.data_path.take() {
457            let tmp = std::env::temp_dir();
458            if let Some(parent) = old.parent() {
459                let orphaned = parent
460                    .file_name()
461                    .and_then(|name| name.to_str())
462                    .is_some_and(|name| name.starts_with("reddb-ephemeral-"))
463                    && parent.parent() == Some(tmp.as_path());
464                if orphaned {
465                    let _ = std::fs::remove_dir_all(parent);
466                }
467            }
468        }
469        self.data_path = Some(path.into());
470        self
471    }
472
473    pub fn with_read_only(mut self, read_only: bool) -> Self {
474        self.read_only = read_only;
475        self
476    }
477
478    pub fn with_auto_checkpoint(mut self, pages: u32) -> Self {
479        self.auto_checkpoint_pages = pages;
480        self
481    }
482
483    pub fn with_checkpoint_columnar_emission_budget_chunks(mut self, chunks: usize) -> Self {
484        self.checkpoint_columnar_emission_budget_chunks = chunks;
485        self
486    }
487
488    pub fn with_columnar_projection_size_floor_rows(mut self, rows: usize) -> Self {
489        self.columnar_projection_size_floor_rows = rows;
490        self
491    }
492
493    pub fn with_durability_mode(mut self, mode: DurabilityMode) -> Self {
494        self.durability_mode = mode;
495        self
496    }
497
498    pub fn with_group_commit_window_ms(mut self, window_ms: u64) -> Self {
499        // `0` is a legitimate setting — "no wait, fsync on every wakeup".
500        // See `DEFAULT_GROUP_COMMIT_WINDOW_MS` docs for the tradeoff.
501        self.group_commit.window_ms = window_ms;
502        self
503    }
504
505    pub fn with_group_commit_max_statements(mut self, max_statements: usize) -> Self {
506        self.group_commit.max_statements = max_statements.max(1);
507        self
508    }
509
510    pub fn with_group_commit_max_wal_bytes(mut self, max_wal_bytes: u64) -> Self {
511        self.group_commit.max_wal_bytes = max_wal_bytes.max(1);
512        self
513    }
514
515    pub fn with_cache_pages(mut self, pages: usize) -> Self {
516        self.cache_pages = pages.max(2);
517        self
518    }
519
520    pub fn with_snapshot_retention(mut self, limit: usize) -> Self {
521        self.snapshot_retention = limit.max(1);
522        self
523    }
524
525    pub fn with_export_retention(mut self, limit: usize) -> Self {
526        self.export_retention = limit.max(1);
527        self
528    }
529
530    pub fn with_metadata<K: Into<String>, V: Into<String>>(mut self, key: K, value: V) -> Self {
531        self.metadata.insert(key.into(), value.into());
532        self
533    }
534
535    /// Toggle the implicit HASH index on user `id` columns at first
536    /// insert (#112). Defaults to enabled — pass `false` to fall back
537    /// to the legacy "scan unless `CREATE INDEX` is issued" behaviour.
538    pub fn with_auto_index_id(mut self, enabled: bool) -> Self {
539        self.auto_index_id = enabled;
540        self
541    }
542
543    pub fn with_capability(mut self, capability: Capability) -> Self {
544        self.feature_gates = self.feature_gates.with(capability);
545        self
546    }
547
548    /// Attach a remote storage backend for snapshot transport.
549    ///
550    /// On open, the database snapshot is downloaded from the remote `key`
551    /// to the local data path. On flush, the local file is uploaded back
552    /// to the remote backend under the same key.
553    pub fn with_remote_backend(
554        mut self,
555        backend: Arc<dyn crate::storage::backend::RemoteBackend>,
556        key: impl Into<String>,
557    ) -> Self {
558        self.remote_backend = Some(backend);
559        self.remote_key = Some(key.into());
560        self
561    }
562
563    /// Attach a CAS-capable backend handle. Pass the same `Arc` as
564    /// `with_remote_backend` (factories should construct the backend
565    /// once and call both setters); this method exists so the type
566    /// system, not runtime config, decides whether `LeaseStore` is
567    /// reachable.
568    pub fn with_atomic_remote_backend(
569        mut self,
570        backend: Arc<dyn crate::storage::backend::AtomicRemoteBackend>,
571    ) -> Self {
572        self.remote_backend_atomic = Some(backend);
573        self
574    }
575
576    pub fn with_replication(mut self, config: ReplicationConfig) -> Self {
577        self.replication = config;
578        self
579    }
580
581    pub fn with_replica_loop_enabled(mut self, enabled: bool) -> Self {
582        self.replica_loop_enabled = enabled;
583        self
584    }
585
586    pub fn with_auth(mut self, config: AuthConfig) -> Self {
587        self.auth = config;
588        self
589    }
590
591    pub fn resolved_path(&self, fallback: impl AsRef<Path>) -> PathBuf {
592        self.data_path
593            .clone()
594            .unwrap_or_else(|| fallback.as_ref().to_path_buf())
595    }
596
597    pub fn remote_namespace_prefix(&self) -> String {
598        let Some(remote_key) = &self.remote_key else {
599            return String::new();
600        };
601        let normalized = remote_key.trim_matches('/');
602        if normalized.is_empty() {
603            return String::new();
604        }
605        match normalized.rsplit_once('/') {
606            Some((parent, _)) if !parent.is_empty() => format!("{parent}/"),
607            _ => String::new(),
608        }
609    }
610
611    pub fn default_backup_head_key(&self) -> String {
612        if let Some(value) = self.metadata.get("red.config.backup.head_key") {
613            return value.clone();
614        }
615        reddb_file::backup_head_key(&self.remote_namespace_prefix())
616    }
617
618    pub fn default_snapshot_prefix(&self) -> String {
619        if let Some(value) = self.metadata.get("red.config.backup.snapshot_prefix") {
620            return value.clone();
621        }
622        reddb_file::backup_snapshot_prefix(&self.remote_namespace_prefix())
623    }
624
625    pub fn default_wal_archive_prefix(&self) -> String {
626        if let Some(value) = self.metadata.get("red.config.wal.archive.prefix") {
627            return value.clone();
628        }
629        reddb_file::backup_wal_prefix(&self.remote_namespace_prefix())
630    }
631
632    pub fn has_capability(&self, capability: Capability) -> bool {
633        self.feature_gates.has(capability)
634    }
635
636    /// Select the active storage-layout preset. Drives tier defaults for
637    /// the six tier-flag toggles. Per-feature setters still win.
638    pub fn with_layout(mut self, layout: crate::storage::layout::StorageLayout) -> Self {
639        self.layout = layout;
640        self.layout_explicit = true;
641        self
642    }
643
644    /// Override individual layout knobs (dedicated dirs + log routing) on
645    /// top of the active preset.
646    pub fn with_layout_overrides(
647        mut self,
648        overrides: crate::storage::layout::LayoutOverrides,
649    ) -> Self {
650        self.layout_overrides = overrides;
651        self
652    }
653
654    /// Declare the process memory budget explicitly (ADR 0073 §1, tier 1).
655    /// Wins over `REDDB_MEMORY_BUDGET`, the profile default, the cgroup limit
656    /// and physical-RAM detection. A zero budget is rejected when the runtime
657    /// opens, not silently swapped for a default.
658    pub fn with_memory_budget(mut self, bytes: u64) -> Self {
659        self.memory_budget_bytes = Some(bytes);
660        self
661    }
662
663    pub fn with_storage_profile(
664        mut self,
665        selection: crate::storage::profile::StorageProfileSelection,
666    ) -> Result<Self, String> {
667        self.storage_profile = selection.validate()?;
668        Ok(self)
669    }
670
671    /// Resolve `(data_path, TieredLayoutPaths)` for this options bundle.
672    /// Returns `None` when `data_path` is unset (in-memory ephemeral
673    /// instances don't materialise a support tree).
674    pub fn resolve_tiered_layout(
675        &self,
676    ) -> Option<(PathBuf, crate::storage::layout::TieredLayoutPaths)> {
677        let data_path = self.data_path.clone()?;
678        let paths = crate::storage::layout::TieredLayoutPaths::new(
679            &data_path,
680            self.layout,
681            self.layout_overrides.clone(),
682        );
683        Some((data_path, paths))
684    }
685
686    /// Flip the process-global tier-flag toggles to match this options
687    /// bundle's layout, and stash the resolved [`TieredLayoutPaths`] for
688    /// status surfaces. Idempotent. Per-feature env escape hatches
689    /// (`REDDB_META_JSON_SIDECAR=...` and friends) still override what
690    /// this method sets — they are read inside the per-toggle getter, not
691    /// here.
692    ///
693    /// Tier defaults:
694    ///
695    /// | toggle                     | minimal | standard | performance | max |
696    /// |----------------------------|:-------:|:--------:|:-----------:|:---:|
697    /// | `.meta.json` sidecar       |   off   |    off   |     off     |  on |
698    /// | seq-N catalog journal      |   off   |    off   |     off     |  on |
699    /// | `-shm` provisioning        |   off   | **on**   |   **on**    |  on |
700    /// | `fold_dwb_into_wal`        |   off   |    off   |     off     |  on |
701    /// | audit/slow log destination | stderr  |  stderr  |  file       | file|
702    ///
703    /// Retention for the seq-N journal: 32 at `Max`, 4 when the operator
704    /// opts in on a lower tier via `REDDB_SEQN_JOURNAL=1`.
705    pub fn apply_tier_defaults(&self) {
706        use crate::storage::layout::StorageLayout;
707
708        // Opt-in: only flip the process-global toggles when the operator
709        // explicitly chose a layout via `with_layout`. Tests and env
710        // hatches that pre-set toggles before opening a runtime must not
711        // be silently overridden by the implicit `Standard` default.
712        if !self.layout_explicit {
713            if let Some((_, paths)) = self.resolve_tiered_layout() {
714                tier_wiring::stash_layout_paths(paths);
715            }
716            return;
717        }
718
719        let layout = self.layout;
720        // .meta.json sidecar — Max only.
721        crate::physical::set_meta_json_sidecar_enabled(matches!(layout, StorageLayout::Max));
722
723        // Seq-N catalog journal — Max only. Retention = 32 for Max,
724        // OPT_IN baseline (4) for lower tiers (the value is consulted
725        // when an env hatch flips the toggle on outside Max).
726        crate::physical::set_seqn_journal_enabled(matches!(layout, StorageLayout::Max));
727        crate::physical::set_seqn_journal_retention(match layout {
728            StorageLayout::Max => crate::physical::DEFAULT_METADATA_JOURNAL_RETENTION,
729            _ => crate::physical::OPT_IN_METADATA_JOURNAL_RETENTION,
730        });
731
732        // `-shm` provisioning — anything ≥ Standard (gh-475 acceptance).
733        crate::physical::set_shm_provisioning_enabled(matches!(
734            layout,
735            StorageLayout::Standard | StorageLayout::Performance | StorageLayout::Max
736        ));
737
738        // Fold DWB into WAL — Max only (a single recovery substrate is the
739        // Max story). The pager manifest is folded into the `.rdb` on every
740        // tier since ADR 0038 §4 phase 1, so it has no toggle.
741        crate::physical::set_fold_dwb_into_wal_enabled(matches!(layout, StorageLayout::Max));
742
743        // Cache resolved layout paths for `red status` / diagnostics.
744        if let Some((_, paths)) = self.resolve_tiered_layout() {
745            tier_wiring::stash_layout_paths(paths);
746        }
747    }
748}
749
750/// Process-global cache of the most recently applied [`TieredLayoutPaths`].
751/// Read by `red status` to surface resolved log destinations. Written by
752/// `RedDBOptions::apply_tier_defaults` after each open.
753pub mod tier_wiring {
754    use std::sync::Mutex;
755
756    use crate::storage::layout::{LogDestination, TieredLayoutPaths};
757
758    static CURRENT_LAYOUT_PATHS: Mutex<Option<TieredLayoutPaths>> = Mutex::new(None);
759
760    pub fn stash_layout_paths(paths: TieredLayoutPaths) {
761        if let Ok(mut slot) = CURRENT_LAYOUT_PATHS.lock() {
762            *slot = Some(paths);
763        }
764    }
765
766    pub fn current_layout_paths() -> Option<TieredLayoutPaths> {
767        CURRENT_LAYOUT_PATHS
768            .lock()
769            .ok()
770            .and_then(|slot| slot.clone())
771    }
772
773    /// `(audit_log, slow_log)` destinations resolved from the active
774    /// layout. Falls back to `(Stderr, Stderr)` when no layout has been
775    /// applied yet (e.g. ephemeral in-memory paths).
776    pub fn current_log_destinations() -> (LogDestination, LogDestination) {
777        match current_layout_paths() {
778            Some(p) => (p.audit_log_destination, p.slow_log_destination),
779            None => (LogDestination::Stderr, LogDestination::Stderr),
780        }
781    }
782}
783
784#[derive(Debug, Clone, Default)]
785pub struct CollectionStats {
786    pub entities: usize,
787    pub cross_refs: usize,
788    pub segments: usize,
789}
790
791#[derive(Debug, Clone)]
792pub struct CatalogSnapshot {
793    pub name: String,
794    pub total_entities: usize,
795    pub total_collections: usize,
796    pub stats_by_collection: BTreeMap<String, CollectionStats>,
797    pub updated_at: SystemTime,
798}
799
800impl Default for CatalogSnapshot {
801    fn default() -> Self {
802        Self {
803            name: String::new(),
804            total_entities: 0,
805            total_collections: 0,
806            stats_by_collection: BTreeMap::new(),
807            updated_at: UNIX_EPOCH,
808        }
809    }
810}
811
812#[derive(Debug, Clone)]
813pub struct SchemaManifest {
814    pub format_version: u32,
815    pub created_at_unix_ms: u128,
816    pub updated_at_unix_ms: u128,
817    pub options: RedDBOptions,
818    pub collection_count: usize,
819}
820
821impl SchemaManifest {
822    pub fn now(options: RedDBOptions, collection_count: usize) -> Self {
823        let now = SystemTime::now()
824            .duration_since(UNIX_EPOCH)
825            .unwrap_or_default()
826            .as_millis();
827        Self {
828            format_version: REDDB_FORMAT_VERSION,
829            created_at_unix_ms: now,
830            updated_at_unix_ms: now,
831            options,
832            collection_count,
833        }
834    }
835}
836
837#[derive(Debug, Clone, PartialEq, Eq)]
838pub struct StorageIntegrityError {
839    pub zone: String,
840    pub id: String,
841    pub collection: Option<String>,
842    pub detail: String,
843}
844
845impl StorageIntegrityError {
846    pub fn new(
847        zone: impl Into<String>,
848        id: impl Into<String>,
849        collection: Option<String>,
850        detail: impl Into<String>,
851    ) -> Self {
852        Self {
853            zone: zone.into(),
854            id: id.into(),
855            collection,
856            detail: detail.into(),
857        }
858    }
859}
860
861impl fmt::Display for StorageIntegrityError {
862    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
863        write!(
864            f,
865            "storage integrity failure: zone={} id={}",
866            self.zone, self.id
867        )?;
868        if let Some(collection) = &self.collection {
869            write!(f, " collection={collection}")?;
870        }
871        write!(
872            f,
873            ": {}. RedDB refused to serve unverified bytes; run the `SCRUB` statement to classify the fault, or `red salvage` to recover trusted rows.",
874            self.detail
875        )
876    }
877}
878
879#[derive(Debug)]
880pub enum RedDBError {
881    InvalidConfig(String),
882    SchemaVersionMismatch {
883        expected: u32,
884        found: u32,
885    },
886    FeatureNotEnabled(String),
887    NotFound(String),
888    ReadOnly(String),
889    InvalidOperation(String),
890    Engine(String),
891    Catalog(String),
892    StorageIntegrity(StorageIntegrityError),
893    Query(String),
894    Validation {
895        message: String,
896        validation: crate::json::Value,
897    },
898    Io(io::Error),
899    VersionUnavailable,
900    /// Operator-pinned cap exceeded (PLAN.md Phase 4.1). The string
901    /// payload should follow the `quota_exceeded:<limit_name>:<current>:<max>`
902    /// shape so HTTP / wire surfaces can map to the right status
903    /// (507 for storage, 429 for rate, 504 for duration, 413 for
904    /// payload).
905    QuotaExceeded(String),
906    /// Issue #769 (PRD #759 / S10) — an aggregating executor
907    /// (`aggregation`, `sort`, `window`) exceeded
908    /// `stream.executor.max_materialized_rows` in its in-memory state.
909    /// Carries the executor that fired, the configured ceiling and the
910    /// live row count at the breach so the client can decide whether to
911    /// redesign the query, raise the limit, or pre-aggregate. Surfaces
912    /// as HTTP 507 with a structured `materialization_limit_exceeded`
913    /// envelope; NDJSON streams emit the same code mid-stream.
914    MaterializationLimitExceeded {
915        executor: &'static str,
916        limit: usize,
917        current: usize,
918    },
919    Internal(String),
920}
921
922/// The text a wire client may see for `err`.
923///
924/// `Io` and `Internal` carry OS messages, filesystem paths and storage
925/// invariants meant for the operator's log, not for a peer; every other
926/// variant is a statement-level message the caller needs verbatim.
927pub fn client_facing_message(err: &RedDBError) -> String {
928    match err {
929        RedDBError::Io(inner) => format!("io error: {:?}", inner.kind()),
930        RedDBError::Internal(_) => "internal error; see the server log".to_string(),
931        other => other.to_string(),
932    }
933}
934
935impl fmt::Display for RedDBError {
936    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
937        match self {
938            Self::InvalidConfig(msg) => write!(f, "invalid config: {msg}"),
939            Self::SchemaVersionMismatch { expected, found } => {
940                write!(
941                    f,
942                    "schema version mismatch: expected {expected}, found {found}"
943                )
944            }
945            Self::FeatureNotEnabled(msg) => write!(f, "feature disabled: {msg}"),
946            Self::NotFound(msg) => write!(f, "not found: {msg}"),
947            Self::ReadOnly(msg) => write!(f, "read-only violation: {msg}"),
948            Self::InvalidOperation(msg) => write!(f, "INVALID_OPERATION: {msg}"),
949            Self::Engine(msg) => write!(f, "engine error: {msg}"),
950            Self::Catalog(msg) => write!(f, "catalog error: {msg}"),
951            Self::StorageIntegrity(err) => write!(f, "{err}"),
952            Self::Query(msg) => write!(f, "query error: {msg}"),
953            Self::Validation { message, .. } => write!(f, "validation error: {message}"),
954            Self::Io(err) => write!(f, "io error: {err}"),
955            Self::VersionUnavailable => write!(f, "version information unavailable"),
956            Self::QuotaExceeded(msg) => write!(f, "quota exceeded: {msg}"),
957            Self::MaterializationLimitExceeded {
958                executor,
959                limit,
960                current,
961            } => write!(
962                f,
963                "materialization limit exceeded: executor={executor} current={current} limit={limit}"
964            ),
965            Self::Internal(msg) => write!(f, "internal error: {msg}"),
966        }
967    }
968}
969
970impl std::error::Error for RedDBError {}
971
972impl From<io::Error> for RedDBError {
973    fn from(err: io::Error) -> Self {
974        Self::Io(err)
975    }
976}
977
978impl From<crate::storage::engine::DatabaseError> for RedDBError {
979    fn from(err: crate::storage::engine::DatabaseError) -> Self {
980        Self::Engine(err.to_string())
981    }
982}
983
984impl From<crate::storage::wal::TxError> for RedDBError {
985    fn from(err: crate::storage::wal::TxError) -> Self {
986        Self::Engine(err.to_string())
987    }
988}
989
990impl From<crate::storage::StoreError> for RedDBError {
991    fn from(err: crate::storage::StoreError) -> Self {
992        match err {
993            crate::storage::StoreError::StorageIntegrity(err) => Self::StorageIntegrity(err),
994            other => Self::Catalog(other.to_string()),
995        }
996    }
997}
998
999impl From<crate::storage::unified::devx::DevXError> for RedDBError {
1000    fn from(err: crate::storage::unified::devx::DevXError) -> Self {
1001        match err {
1002            crate::storage::unified::devx::DevXError::Validation(msg) => Self::InvalidConfig(msg),
1003            crate::storage::unified::devx::DevXError::Storage(msg) => Self::Engine(msg),
1004            crate::storage::unified::devx::DevXError::NotFound(msg) => Self::NotFound(msg),
1005        }
1006    }
1007}
1008
1009pub trait CatalogService {
1010    fn list_collections(&self) -> Vec<String>;
1011    fn collection_stats(&self, collection: &str) -> Option<CollectionStats>;
1012    fn catalog_snapshot(&self) -> CatalogSnapshot;
1013}
1014
1015pub trait QueryPlanner {
1016    fn plan_cost(&self, query: &str) -> Option<f64>;
1017}
1018
1019pub trait DataOps {
1020    fn execute_query(&self, query: &str) -> RedDBResult<()>;
1021}
1022
1023pub mod prelude {
1024    pub use super::{
1025        Capability, CapabilitySet, CatalogService, CatalogSnapshot, CollectionStats, DataOps,
1026        QueryPlanner, RedDBError, RedDBOptions, RedDBResult, SchemaManifest, StorageMode,
1027        REDDB_FORMAT_VERSION, REDDB_PROTOCOL_VERSION,
1028    };
1029}
1030
1031#[cfg(test)]
1032mod client_facing_message_tests {
1033    use super::{client_facing_message, RedDBError};
1034
1035    #[test]
1036    fn client_facing_message_hides_io_and_internal_detail() {
1037        let io = RedDBError::Io(std::io::Error::new(
1038            std::io::ErrorKind::NotFound,
1039            "/var/lib/reddb/data.rdb: No such file",
1040        ));
1041        assert_eq!(client_facing_message(&io), "io error: NotFound");
1042        let internal = RedDBError::Internal("pager invariant broke at /srv/db.wal".to_string());
1043        assert!(!client_facing_message(&internal).contains("/srv"));
1044        let query = RedDBError::Query("unknown column `x`".to_string());
1045        assert_eq!(client_facing_message(&query), query.to_string());
1046    }
1047}
reddb_server/api.rs

reddb_server/
api.rs
