reddb_server/
api.rs

1//! Public API layer for the RedDB crate.
2//!
3//! This module is the first layer to consume from applications:
4//! - stable options and contracts
5//! - capability declarations
6//! - typed errors and lightweight metadata snapshots
7//! - cross-layer traits for catalog/operations observability
8
9use std::collections::{BTreeMap, BTreeSet};
10use std::fmt;
11use std::io;
12use std::path::{Path, PathBuf};
13use std::sync::Arc;
14use std::time::{SystemTime, UNIX_EPOCH};
15
16use crate::auth::AuthConfig;
17use crate::replication::ReplicationConfig;
18
19pub const DEFAULT_SNAPSHOT_RETENTION: usize = 16;
20pub const DEFAULT_EXPORT_RETENTION: usize = 16;
21
22pub const REDDB_PROTOCOL_VERSION: &str = "reddb-v2";
23pub const REDDB_FORMAT_VERSION: u32 = 2;
24/// Default group-commit window.
25///
26/// `0` = "no wait" — the background flusher fsyncs as soon as any
27/// writer's pending commit arrives. Under single-writer workloads a
28/// non-zero window would be pure latency (no one to batch with),
29/// capping individual commit throughput at ~1000/s for window=1.
30/// Concurrent writers still batch naturally via the `Mutex<WalWriter>`
31/// contention path without needing an explicit timer.
32///
33/// Operators with many concurrent clients can raise this (e.g. 1-5ms)
34/// to amortise fsync cost across a bigger batch — at the cost of p99
35/// tail latency going up by the window size.
36pub const DEFAULT_GROUP_COMMIT_WINDOW_MS: u64 = 0;
37pub const DEFAULT_GROUP_COMMIT_MAX_STATEMENTS: usize = 128;
38pub const DEFAULT_GROUP_COMMIT_MAX_WAL_BYTES: u64 = 1024 * 1024;
39pub(crate) const EPHEMERAL_RUNTIME_METADATA_KEY: &str = "__reddb_ephemeral_runtime";
40
41pub type RedDBResult<T> = Result<T, RedDBError>;
42
43#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)]
44pub enum StorageMode {
45    /// Durable, file-backed database with WAL + checkpointing.
46    #[default]
47    Persistent,
48}
49
50impl StorageMode {
51    pub const fn is_persistent(self) -> bool {
52        matches!(self, Self::Persistent)
53    }
54}
55
56#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)]
57pub enum DurabilityMode {
58    #[default]
59    Strict,
60    WalDurableGrouped,
61    /// Fire-and-forget. Writers return as soon as the WAL record is
62    /// in the in-memory ring buffer — they do NOT wait for fsync.
63    /// The group-commit background thread still flushes on its
64    /// cadence, so committed-but-unflushed work is bounded by
65    /// `GroupCommitOptions::window_ms`. A crash inside the window
66    /// drops whatever wasn't flushed — matches PG's
67    /// `synchronous_commit=off` contract.
68    Async,
69}
70
71impl DurabilityMode {
72    pub const fn as_str(self) -> &'static str {
73        match self {
74            Self::Strict => "strict",
75            Self::WalDurableGrouped => "wal_durable_grouped",
76            Self::Async => "async",
77        }
78    }
79
80    pub fn from_str(value: &str) -> Option<Self> {
81        let normalized = value.trim().to_ascii_lowercase();
82        match normalized.as_str() {
83            // Legacy / opt-out form. Every commit pays its own fsync.
84            "strict" => Some(Self::Strict),
85            // Group-commit sync path — the perf-parity default. Matches
86            // PostgreSQL's `synchronous_commit=on` behaviour: the
87            // writer waits for durability, but fsyncs are batched
88            // across concurrent writers so a burst of N commits pays
89            // ~O(1) fsyncs instead of O(N).
90            "sync"
91            | "wal_durable_grouped"
92            | "wal-durable-grouped"
93            | "grouped"
94            | "wal_grouped"
95            | "wal-grouped" => Some(Self::WalDurableGrouped),
96            // Fire-and-forget async: writers return as soon as the
97            // WAL buffer accepts the record; background flusher runs
98            // on its configured cadence.
99            "async" | "fire_and_forget" | "fire-and-forget" => Some(Self::Async),
100            _ => None,
101        }
102    }
103}
104
105#[derive(Debug, Clone, Copy, PartialEq, Eq)]
106pub struct GroupCommitOptions {
107    pub window_ms: u64,
108    pub max_statements: usize,
109    pub max_wal_bytes: u64,
110}
111
112impl Default for GroupCommitOptions {
113    fn default() -> Self {
114        Self {
115            window_ms: DEFAULT_GROUP_COMMIT_WINDOW_MS,
116            max_statements: DEFAULT_GROUP_COMMIT_MAX_STATEMENTS,
117            max_wal_bytes: DEFAULT_GROUP_COMMIT_MAX_WAL_BYTES,
118        }
119    }
120}
121
122#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
123pub enum Capability {
124    /// Structured row storage.
125    Table,
126    /// Graph nodes/edges.
127    Graph,
128    /// Vector collections and ANN search.
129    Vector,
130    /// Full-text / lexical search.
131    FullText,
132    /// Text/metadata security and enrichment modules.
133    Security,
134    /// Encryption at rest.
135    Encryption,
136}
137
138impl Capability {
139    pub const fn as_str(self) -> &'static str {
140        match self {
141            Self::Table => "table",
142            Self::Graph => "graph",
143            Self::Vector => "vector",
144            Self::FullText => "fulltext",
145            Self::Security => "security",
146            Self::Encryption => "encryption",
147        }
148    }
149}
150
151#[derive(Debug, Clone, Default)]
152pub struct CapabilitySet {
153    items: BTreeSet<Capability>,
154}
155
156impl CapabilitySet {
157    pub fn new() -> Self {
158        Self::default()
159    }
160
161    pub fn with(mut self, capability: Capability) -> Self {
162        self.items.insert(capability);
163        self
164    }
165
166    pub fn with_all(mut self, capabilities: &[Capability]) -> Self {
167        capabilities.iter().copied().for_each(|capability| {
168            self.items.insert(capability);
169        });
170        self
171    }
172
173    pub fn has(&self, capability: Capability) -> bool {
174        self.items.contains(&capability)
175    }
176
177    pub fn as_slice(&self) -> Vec<Capability> {
178        self.items.iter().copied().collect()
179    }
180}
181
182pub struct RedDBOptions {
183    pub mode: StorageMode,
184    pub data_path: Option<PathBuf>,
185    pub read_only: bool,
186    pub create_if_missing: bool,
187    pub verify_checksums: bool,
188    pub durability_mode: DurabilityMode,
189    pub group_commit: GroupCommitOptions,
190    pub auto_checkpoint_pages: u32,
191    /// Maximum hypertable chunks the checkpoint path may seal into derived
192    /// columnar artifacts per checkpoint. `usize::MAX` means no practical cap.
193    pub checkpoint_columnar_emission_budget_chunks: usize,
194    /// Minimum rows a sealed in-scope chunk must carry before the derived
195    /// columnar projection is materialized.
196    pub columnar_projection_size_floor_rows: usize,
197    pub cache_pages: usize,
198    pub snapshot_retention: usize,
199    pub export_retention: usize,
200    pub feature_gates: CapabilitySet,
201    pub force_create: bool,
202    pub metadata: BTreeMap<String, String>,
203    /// Optional remote storage backend for snapshot transport.
204    pub remote_backend: Option<Arc<dyn crate::storage::backend::RemoteBackend>>,
205    /// Optional CAS-capable handle to the same backend, populated by
206    /// the factory when the configured backend implements
207    /// `AtomicRemoteBackend` (S3/local always; HTTP only when
208    /// `RED_HTTP_CONDITIONAL_WRITES=true`). `None` for backends that
209    /// do not provide compare-and-swap (Turso, D1, plain HTTP).
210    /// `LeaseStore` and any future CAS consumer pull from this field.
211    pub remote_backend_atomic: Option<Arc<dyn crate::storage::backend::AtomicRemoteBackend>>,
212    /// Remote object key used by the remote backend.
213    pub remote_key: Option<String>,
214    /// Replication configuration.
215    pub replication: ReplicationConfig,
216    /// Authentication & authorization configuration.
217    pub auth: AuthConfig,
218    /// Control Event Ledger configuration (issue #652). Read from
219    /// `REDDB_COMPLIANCE_MODE` at boot.
220    pub control_events: crate::runtime::control_events::ControlEventConfig,
221    /// Scoped data-plane query audit configuration. Disabled by
222    /// default; the regulated preset enables the stream without
223    /// adding catch-all rules.
224    pub query_audit: crate::runtime::query_audit::QueryAuditConfig,
225    /// Auto-create a HASH index on a user `id` column the first time a
226    /// row carrying that column is inserted into a collection. See
227    /// `UnifiedStoreConfig::auto_index_id`. Defaults to `true`; set to
228    /// `false` to opt out per workload (e.g. ingest pipelines that
229    /// don't need point-lookups by `id`).
230    pub auto_index_id: bool,
231    /// Tiered storage layout preset. Drives the defaults of the six
232    /// tier-flag toggles (`.meta.json` sidecar, seq-N catalog journal,
233    /// `-shm` provisioning, audit/slow log destinations,
234    /// `fold_dwb_into_wal`). Explicit per-feature setters still win over
235    /// the tier default. See `crate::storage::layout::StorageLayout`.
236    pub layout: crate::storage::layout::StorageLayout,
237    /// Per-feature overrides applied on top of the resolved layout preset.
238    pub layout_overrides: crate::storage::layout::LayoutOverrides,
239    /// Operator-facing storage/deploy profile contract. This records the
240    /// selected posture before the physical directory layout is expanded.
241    pub storage_profile: crate::storage::profile::StorageProfileSelection,
242    /// True only when the caller explicitly selected a layout via
243    /// `with_layout`. When false, `apply_tier_defaults` short-circuits so
244    /// pre-existing process-global toggles (set by tests / env hatches
245    /// before opening a runtime) are not clobbered by the implicit
246    /// `Standard` default. The new tier-driven behavior is opt-in.
247    pub layout_explicit: bool,
248    /// Explicit operator memory budget in bytes — the top tier of the ADR
249    /// 0073 §1 precedence chain. `None` defers to `REDDB_MEMORY_BUDGET`, the
250    /// deployment-profile default, the cgroup limit, and finally a fraction
251    /// of physical RAM. There is no "unlimited" value: `Some(0)` is rejected
252    /// at boot, exactly like an unparseable env var.
253    pub memory_budget_bytes: Option<u64>,
254}
255
256impl fmt::Debug for RedDBOptions {
257    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
258        let backend_name = self.remote_backend.as_ref().map(|b| b.name().to_string());
259        f.debug_struct("RedDBOptions")
260            .field("mode", &self.mode)
261            .field("data_path", &self.data_path)
262            .field("read_only", &self.read_only)
263            .field("create_if_missing", &self.create_if_missing)
264            .field("verify_checksums", &self.verify_checksums)
265            .field("durability_mode", &self.durability_mode)
266            .field("group_commit", &self.group_commit)
267            .field("auto_checkpoint_pages", &self.auto_checkpoint_pages)
268            .field(
269                "checkpoint_columnar_emission_budget_chunks",
270                &self.checkpoint_columnar_emission_budget_chunks,
271            )
272            .field(
273                "columnar_projection_size_floor_rows",
274                &self.columnar_projection_size_floor_rows,
275            )
276            .field("cache_pages", &self.cache_pages)
277            .field("snapshot_retention", &self.snapshot_retention)
278            .field("export_retention", &self.export_retention)
279            .field("feature_gates", &self.feature_gates)
280            .field("force_create", &self.force_create)
281            .field("metadata", &self.metadata)
282            .field("remote_backend", &backend_name)
283            .field("remote_key", &self.remote_key)
284            .field("replication", &self.replication)
285            .field("auth", &self.auth)
286            .field("control_events", &self.control_events)
287            .field("query_audit", &self.query_audit)
288            .field("layout", &self.layout)
289            .field("layout_overrides", &self.layout_overrides)
290            .field("storage_profile", &self.storage_profile)
291            .field("memory_budget_bytes", &self.memory_budget_bytes)
292            .finish()
293    }
294}
295
296impl Clone for RedDBOptions {
297    fn clone(&self) -> Self {
298        Self {
299            mode: self.mode,
300            data_path: self.data_path.clone(),
301            read_only: self.read_only,
302            create_if_missing: self.create_if_missing,
303            verify_checksums: self.verify_checksums,
304            durability_mode: self.durability_mode,
305            group_commit: self.group_commit,
306            auto_checkpoint_pages: self.auto_checkpoint_pages,
307            checkpoint_columnar_emission_budget_chunks: self
308                .checkpoint_columnar_emission_budget_chunks,
309            columnar_projection_size_floor_rows: self.columnar_projection_size_floor_rows,
310            cache_pages: self.cache_pages,
311            snapshot_retention: self.snapshot_retention,
312            export_retention: self.export_retention,
313            feature_gates: self.feature_gates.clone(),
314            force_create: self.force_create,
315            metadata: self.metadata.clone(),
316            remote_backend: self.remote_backend.clone(),
317            remote_backend_atomic: self.remote_backend_atomic.clone(),
318            remote_key: self.remote_key.clone(),
319            replication: self.replication.clone(),
320            auth: self.auth.clone(),
321            control_events: self.control_events,
322            query_audit: self.query_audit.clone(),
323            auto_index_id: self.auto_index_id,
324            layout: self.layout,
325            layout_overrides: self.layout_overrides.clone(),
326            storage_profile: self.storage_profile,
327            layout_explicit: self.layout_explicit,
328            memory_budget_bytes: self.memory_budget_bytes,
329        }
330    }
331}
332
333impl Default for RedDBOptions {
334    fn default() -> Self {
335        Self {
336            mode: StorageMode::Persistent,
337            data_path: None,
338            read_only: false,
339            create_if_missing: true,
340            verify_checksums: true,
341            // Perf-parity default — `WalDurableGrouped` matches
342            // PostgreSQL's `synchronous_commit=on` behaviour while
343            // amortising fsync cost across concurrent writers. The
344            // legacy `Strict` tier (per-commit fsync) stays available
345            // via `durability.mode = "strict"` / `REDDB_DURABILITY=strict`.
346            durability_mode: DurabilityMode::WalDurableGrouped,
347            group_commit: GroupCommitOptions::default(),
348            auto_checkpoint_pages: 1000,
349            checkpoint_columnar_emission_budget_chunks: usize::MAX,
350            columnar_projection_size_floor_rows: 4,
351            cache_pages: 10_000,
352            snapshot_retention: DEFAULT_SNAPSHOT_RETENTION,
353            export_retention: DEFAULT_EXPORT_RETENTION,
354            feature_gates: CapabilitySet::new()
355                .with(Capability::Table)
356                .with(Capability::Graph)
357                .with(Capability::Vector),
358            force_create: true,
359            metadata: BTreeMap::new(),
360            remote_backend: None,
361            remote_backend_atomic: None,
362            remote_key: None,
363            replication: ReplicationConfig::standalone(),
364            auth: AuthConfig::default(),
365            control_events: crate::runtime::control_events::ControlEventConfig::default(),
366            query_audit: crate::runtime::query_audit::QueryAuditConfig::default(),
367            auto_index_id: true,
368            layout: crate::storage::layout::StorageLayout::default(),
369            layout_overrides: crate::storage::layout::LayoutOverrides::default(),
370            storage_profile: crate::storage::profile::StorageProfileSelection::embedded_single_file(
371            ),
372            layout_explicit: false,
373            memory_budget_bytes: None,
374        }
375    }
376}
377
378impl RedDBOptions {
379    pub fn persistent<P: Into<PathBuf>>(path: P) -> Self {
380        Self {
381            mode: StorageMode::Persistent,
382            data_path: Some(path.into()),
383            ..Default::default()
384        }
385    }
386
387    /// Ephemeral, tempfile-backed database.
388    ///
389    /// The underlying storage is a real persistent file placed under the system
390    /// temp directory with a unique name — there is no longer a true in-memory
391    /// execution mode. Prefer [`RedDBOptions::persistent`] when the data should
392    /// outlive the process.
393    pub fn in_memory() -> Self {
394        static NEXT_EPHEMERAL_ID: std::sync::atomic::AtomicU64 =
395            std::sync::atomic::AtomicU64::new(0);
396
397        let now_nanos = std::time::SystemTime::now()
398            .duration_since(std::time::UNIX_EPOCH)
399            .map(|duration| duration.as_nanos())
400            .unwrap_or(0);
401        let unique = NEXT_EPHEMERAL_ID.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
402        // Each ephemeral runtime gets its OWN directory, not just a unique
403        // filename in the shared temp dir. Sibling artifacts (the audit log,
404        // WAL, snapshots) are derived from this data path's PARENT, so a
405        // shared parent collapses every ephemeral runtime's `.audit.log`
406        // onto one file — which parallel test processes (nextest runs each
407        // test in its own process, frequently under one shared `TMPDIR`)
408        // then truncate/remove out from under one another. A per-instance
409        // directory keeps each runtime's siblings self-contained.
410        let dir = std::env::temp_dir().join(format!(
411            "reddb-ephemeral-{}-{}-{}",
412            std::process::id(),
413            now_nanos,
414            unique
415        ));
416        let _ = std::fs::create_dir_all(&dir);
417        let path = dir.join("db.rdb");
418        let _ = std::fs::remove_file(&path);
419        let mut metadata = BTreeMap::new();
420        metadata.insert(
421            EPHEMERAL_RUNTIME_METADATA_KEY.to_string(),
422            "true".to_string(),
423        );
424        Self {
425            mode: StorageMode::Persistent,
426            data_path: Some(path),
427            auto_checkpoint_pages: 0,
428            checkpoint_columnar_emission_budget_chunks: usize::MAX,
429            columnar_projection_size_floor_rows: 4,
430            cache_pages: 2_000,
431            snapshot_retention: DEFAULT_SNAPSHOT_RETENTION,
432            export_retention: DEFAULT_EXPORT_RETENTION,
433            read_only: false,
434            force_create: true,
435            metadata,
436            ..Default::default()
437        }
438    }
439
440    pub fn with_mode(mut self, mode: StorageMode) -> Self {
441        self.mode = mode;
442        self
443    }
444
445    pub fn with_data_path<P: Into<PathBuf>>(mut self, path: P) -> Self {
446        // `in_memory()` eagerly creates a `reddb-ephemeral-*` dir and points
447        // `data_path` at it. Overriding the path abandons that dir — and the
448        // per-runtime cleanup keys off the FINAL `data_path`, so the orphan
449        // would leak in TMPDIR (caught by scripts/check-temp-residue.sh). Remove
450        // it now if the previous path was such an eagerly-created ephemeral dir.
451        if let Some(old) = self.data_path.take() {
452            let tmp = std::env::temp_dir();
453            if let Some(parent) = old.parent() {
454                let orphaned = parent
455                    .file_name()
456                    .and_then(|name| name.to_str())
457                    .is_some_and(|name| name.starts_with("reddb-ephemeral-"))
458                    && parent.parent() == Some(tmp.as_path());
459                if orphaned {
460                    let _ = std::fs::remove_dir_all(parent);
461                }
462            }
463        }
464        self.data_path = Some(path.into());
465        self
466    }
467
468    pub fn with_read_only(mut self, read_only: bool) -> Self {
469        self.read_only = read_only;
470        self
471    }
472
473    pub fn with_auto_checkpoint(mut self, pages: u32) -> Self {
474        self.auto_checkpoint_pages = pages;
475        self
476    }
477
478    pub fn with_checkpoint_columnar_emission_budget_chunks(mut self, chunks: usize) -> Self {
479        self.checkpoint_columnar_emission_budget_chunks = chunks;
480        self
481    }
482
483    pub fn with_columnar_projection_size_floor_rows(mut self, rows: usize) -> Self {
484        self.columnar_projection_size_floor_rows = rows;
485        self
486    }
487
488    pub fn with_durability_mode(mut self, mode: DurabilityMode) -> Self {
489        self.durability_mode = mode;
490        self
491    }
492
493    pub fn with_group_commit_window_ms(mut self, window_ms: u64) -> Self {
494        // `0` is a legitimate setting — "no wait, fsync on every wakeup".
495        // See `DEFAULT_GROUP_COMMIT_WINDOW_MS` docs for the tradeoff.
496        self.group_commit.window_ms = window_ms;
497        self
498    }
499
500    pub fn with_group_commit_max_statements(mut self, max_statements: usize) -> Self {
501        self.group_commit.max_statements = max_statements.max(1);
502        self
503    }
504
505    pub fn with_group_commit_max_wal_bytes(mut self, max_wal_bytes: u64) -> Self {
506        self.group_commit.max_wal_bytes = max_wal_bytes.max(1);
507        self
508    }
509
510    pub fn with_cache_pages(mut self, pages: usize) -> Self {
511        self.cache_pages = pages.max(2);
512        self
513    }
514
515    pub fn with_snapshot_retention(mut self, limit: usize) -> Self {
516        self.snapshot_retention = limit.max(1);
517        self
518    }
519
520    pub fn with_export_retention(mut self, limit: usize) -> Self {
521        self.export_retention = limit.max(1);
522        self
523    }
524
525    pub fn with_metadata<K: Into<String>, V: Into<String>>(mut self, key: K, value: V) -> Self {
526        self.metadata.insert(key.into(), value.into());
527        self
528    }
529
530    /// Toggle the implicit HASH index on user `id` columns at first
531    /// insert (#112). Defaults to enabled — pass `false` to fall back
532    /// to the legacy "scan unless `CREATE INDEX` is issued" behaviour.
533    pub fn with_auto_index_id(mut self, enabled: bool) -> Self {
534        self.auto_index_id = enabled;
535        self
536    }
537
538    pub fn with_capability(mut self, capability: Capability) -> Self {
539        self.feature_gates = self.feature_gates.with(capability);
540        self
541    }
542
543    /// Attach a remote storage backend for snapshot transport.
544    ///
545    /// On open, the database snapshot is downloaded from the remote `key`
546    /// to the local data path. On flush, the local file is uploaded back
547    /// to the remote backend under the same key.
548    pub fn with_remote_backend(
549        mut self,
550        backend: Arc<dyn crate::storage::backend::RemoteBackend>,
551        key: impl Into<String>,
552    ) -> Self {
553        self.remote_backend = Some(backend);
554        self.remote_key = Some(key.into());
555        self
556    }
557
558    /// Attach a CAS-capable backend handle. Pass the same `Arc` as
559    /// `with_remote_backend` (factories should construct the backend
560    /// once and call both setters); this method exists so the type
561    /// system, not runtime config, decides whether `LeaseStore` is
562    /// reachable.
563    pub fn with_atomic_remote_backend(
564        mut self,
565        backend: Arc<dyn crate::storage::backend::AtomicRemoteBackend>,
566    ) -> Self {
567        self.remote_backend_atomic = Some(backend);
568        self
569    }
570
571    pub fn with_replication(mut self, config: ReplicationConfig) -> Self {
572        self.replication = config;
573        self
574    }
575
576    pub fn with_auth(mut self, config: AuthConfig) -> Self {
577        self.auth = config;
578        self
579    }
580
581    pub fn resolved_path(&self, fallback: impl AsRef<Path>) -> PathBuf {
582        self.data_path
583            .clone()
584            .unwrap_or_else(|| fallback.as_ref().to_path_buf())
585    }
586
587    pub fn remote_namespace_prefix(&self) -> String {
588        let Some(remote_key) = &self.remote_key else {
589            return String::new();
590        };
591        let normalized = remote_key.trim_matches('/');
592        if normalized.is_empty() {
593            return String::new();
594        }
595        match normalized.rsplit_once('/') {
596            Some((parent, _)) if !parent.is_empty() => format!("{parent}/"),
597            _ => String::new(),
598        }
599    }
600
601    pub fn default_backup_head_key(&self) -> String {
602        if let Some(value) = self.metadata.get("red.config.backup.head_key") {
603            return value.clone();
604        }
605        reddb_file::backup_head_key(&self.remote_namespace_prefix())
606    }
607
608    pub fn default_snapshot_prefix(&self) -> String {
609        if let Some(value) = self.metadata.get("red.config.backup.snapshot_prefix") {
610            return value.clone();
611        }
612        reddb_file::backup_snapshot_prefix(&self.remote_namespace_prefix())
613    }
614
615    pub fn default_wal_archive_prefix(&self) -> String {
616        if let Some(value) = self.metadata.get("red.config.wal.archive.prefix") {
617            return value.clone();
618        }
619        reddb_file::backup_wal_prefix(&self.remote_namespace_prefix())
620    }
621
622    pub fn has_capability(&self, capability: Capability) -> bool {
623        self.feature_gates.has(capability)
624    }
625
626    /// Select the active storage-layout preset. Drives tier defaults for
627    /// the six tier-flag toggles. Per-feature setters still win.
628    pub fn with_layout(mut self, layout: crate::storage::layout::StorageLayout) -> Self {
629        self.layout = layout;
630        self.layout_explicit = true;
631        self
632    }
633
634    /// Override individual layout knobs (dedicated dirs + log routing) on
635    /// top of the active preset.
636    pub fn with_layout_overrides(
637        mut self,
638        overrides: crate::storage::layout::LayoutOverrides,
639    ) -> Self {
640        self.layout_overrides = overrides;
641        self
642    }
643
644    /// Declare the process memory budget explicitly (ADR 0073 §1, tier 1).
645    /// Wins over `REDDB_MEMORY_BUDGET`, the profile default, the cgroup limit
646    /// and physical-RAM detection. A zero budget is rejected when the runtime
647    /// opens, not silently swapped for a default.
648    pub fn with_memory_budget(mut self, bytes: u64) -> Self {
649        self.memory_budget_bytes = Some(bytes);
650        self
651    }
652
653    pub fn with_storage_profile(
654        mut self,
655        selection: crate::storage::profile::StorageProfileSelection,
656    ) -> Result<Self, String> {
657        self.storage_profile = selection.validate()?;
658        Ok(self)
659    }
660
661    /// Resolve `(data_path, TieredLayoutPaths)` for this options bundle.
662    /// Returns `None` when `data_path` is unset (in-memory ephemeral
663    /// instances don't materialise a support tree).
664    pub fn resolve_tiered_layout(
665        &self,
666    ) -> Option<(PathBuf, crate::storage::layout::TieredLayoutPaths)> {
667        let data_path = self.data_path.clone()?;
668        let paths = crate::storage::layout::TieredLayoutPaths::new(
669            &data_path,
670            self.layout,
671            self.layout_overrides.clone(),
672        );
673        Some((data_path, paths))
674    }
675
676    /// Flip the process-global tier-flag toggles to match this options
677    /// bundle's layout, and stash the resolved [`TieredLayoutPaths`] for
678    /// status surfaces. Idempotent. Per-feature env escape hatches
679    /// (`REDDB_META_JSON_SIDECAR=...` and friends) still override what
680    /// this method sets — they are read inside the per-toggle getter, not
681    /// here.
682    ///
683    /// Tier defaults:
684    ///
685    /// | toggle                     | minimal | standard | performance | max |
686    /// |----------------------------|:-------:|:--------:|:-----------:|:---:|
687    /// | `.meta.json` sidecar       |   off   |    off   |     off     |  on |
688    /// | seq-N catalog journal      |   off   |    off   |     off     |  on |
689    /// | `-shm` provisioning        |   off   | **on**   |   **on**    |  on |
690    /// | `fold_dwb_into_wal`        |   off   |    off   |     off     |  on |
691    /// | audit/slow log destination | stderr  |  stderr  |  file       | file|
692    ///
693    /// Retention for the seq-N journal: 32 at `Max`, 4 when the operator
694    /// opts in on a lower tier via `REDDB_SEQN_JOURNAL=1`.
695    pub fn apply_tier_defaults(&self) {
696        use crate::storage::layout::StorageLayout;
697
698        // Opt-in: only flip the process-global toggles when the operator
699        // explicitly chose a layout via `with_layout`. Tests and env
700        // hatches that pre-set toggles before opening a runtime must not
701        // be silently overridden by the implicit `Standard` default.
702        if !self.layout_explicit {
703            if let Some((_, paths)) = self.resolve_tiered_layout() {
704                tier_wiring::stash_layout_paths(paths);
705            }
706            return;
707        }
708
709        let layout = self.layout;
710        // .meta.json sidecar — Max only.
711        crate::physical::set_meta_json_sidecar_enabled(matches!(layout, StorageLayout::Max));
712
713        // Seq-N catalog journal — Max only. Retention = 32 for Max,
714        // OPT_IN baseline (4) for lower tiers (the value is consulted
715        // when an env hatch flips the toggle on outside Max).
716        crate::physical::set_seqn_journal_enabled(matches!(layout, StorageLayout::Max));
717        crate::physical::set_seqn_journal_retention(match layout {
718            StorageLayout::Max => crate::physical::DEFAULT_METADATA_JOURNAL_RETENTION,
719            _ => crate::physical::OPT_IN_METADATA_JOURNAL_RETENTION,
720        });
721
722        // `-shm` provisioning — anything ≥ Standard (gh-475 acceptance).
723        crate::physical::set_shm_provisioning_enabled(matches!(
724            layout,
725            StorageLayout::Standard | StorageLayout::Performance | StorageLayout::Max
726        ));
727
728        // Fold DWB into WAL — Max only (a single recovery substrate is the
729        // Max story). The pager manifest is folded into the `.rdb` on every
730        // tier since ADR 0038 §4 phase 1, so it has no toggle.
731        crate::physical::set_fold_dwb_into_wal_enabled(matches!(layout, StorageLayout::Max));
732
733        // Cache resolved layout paths for `red status` / diagnostics.
734        if let Some((_, paths)) = self.resolve_tiered_layout() {
735            tier_wiring::stash_layout_paths(paths);
736        }
737    }
738}
739
740/// Process-global cache of the most recently applied [`TieredLayoutPaths`].
741/// Read by `red status` to surface resolved log destinations. Written by
742/// `RedDBOptions::apply_tier_defaults` after each open.
743pub mod tier_wiring {
744    use std::sync::Mutex;
745
746    use crate::storage::layout::{LogDestination, TieredLayoutPaths};
747
748    static CURRENT_LAYOUT_PATHS: Mutex<Option<TieredLayoutPaths>> = Mutex::new(None);
749
750    pub fn stash_layout_paths(paths: TieredLayoutPaths) {
751        if let Ok(mut slot) = CURRENT_LAYOUT_PATHS.lock() {
752            *slot = Some(paths);
753        }
754    }
755
756    pub fn current_layout_paths() -> Option<TieredLayoutPaths> {
757        CURRENT_LAYOUT_PATHS
758            .lock()
759            .ok()
760            .and_then(|slot| slot.clone())
761    }
762
763    /// `(audit_log, slow_log)` destinations resolved from the active
764    /// layout. Falls back to `(Stderr, Stderr)` when no layout has been
765    /// applied yet (e.g. ephemeral in-memory paths).
766    pub fn current_log_destinations() -> (LogDestination, LogDestination) {
767        match current_layout_paths() {
768            Some(p) => (p.audit_log_destination, p.slow_log_destination),
769            None => (LogDestination::Stderr, LogDestination::Stderr),
770        }
771    }
772}
773
774#[derive(Debug, Clone, Default)]
775pub struct CollectionStats {
776    pub entities: usize,
777    pub cross_refs: usize,
778    pub segments: usize,
779}
780
781#[derive(Debug, Clone)]
782pub struct CatalogSnapshot {
783    pub name: String,
784    pub total_entities: usize,
785    pub total_collections: usize,
786    pub stats_by_collection: BTreeMap<String, CollectionStats>,
787    pub updated_at: SystemTime,
788}
789
790impl Default for CatalogSnapshot {
791    fn default() -> Self {
792        Self {
793            name: String::new(),
794            total_entities: 0,
795            total_collections: 0,
796            stats_by_collection: BTreeMap::new(),
797            updated_at: UNIX_EPOCH,
798        }
799    }
800}
801
802#[derive(Debug, Clone)]
803pub struct SchemaManifest {
804    pub format_version: u32,
805    pub created_at_unix_ms: u128,
806    pub updated_at_unix_ms: u128,
807    pub options: RedDBOptions,
808    pub collection_count: usize,
809}
810
811impl SchemaManifest {
812    pub fn now(options: RedDBOptions, collection_count: usize) -> Self {
813        let now = SystemTime::now()
814            .duration_since(UNIX_EPOCH)
815            .unwrap_or_default()
816            .as_millis();
817        Self {
818            format_version: REDDB_FORMAT_VERSION,
819            created_at_unix_ms: now,
820            updated_at_unix_ms: now,
821            options,
822            collection_count,
823        }
824    }
825}
826
827#[derive(Debug, Clone, PartialEq, Eq)]
828pub struct StorageIntegrityError {
829    pub zone: String,
830    pub id: String,
831    pub collection: Option<String>,
832    pub detail: String,
833}
834
835impl StorageIntegrityError {
836    pub fn new(
837        zone: impl Into<String>,
838        id: impl Into<String>,
839        collection: Option<String>,
840        detail: impl Into<String>,
841    ) -> Self {
842        Self {
843            zone: zone.into(),
844            id: id.into(),
845            collection,
846            detail: detail.into(),
847        }
848    }
849}
850
851impl fmt::Display for StorageIntegrityError {
852    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
853        write!(
854            f,
855            "storage integrity failure: zone={} id={}",
856            self.zone, self.id
857        )?;
858        if let Some(collection) = &self.collection {
859            write!(f, " collection={collection}")?;
860        }
861        write!(
862            f,
863            ": {}. RedDB refused to serve unverified bytes; run scrub when available, or use red salvage to recover trusted rows.",
864            self.detail
865        )
866    }
867}
868
869#[derive(Debug)]
870pub enum RedDBError {
871    InvalidConfig(String),
872    SchemaVersionMismatch {
873        expected: u32,
874        found: u32,
875    },
876    FeatureNotEnabled(String),
877    NotFound(String),
878    ReadOnly(String),
879    InvalidOperation(String),
880    Engine(String),
881    Catalog(String),
882    StorageIntegrity(StorageIntegrityError),
883    Query(String),
884    Validation {
885        message: String,
886        validation: crate::json::Value,
887    },
888    Io(io::Error),
889    VersionUnavailable,
890    /// Operator-pinned cap exceeded (PLAN.md Phase 4.1). The string
891    /// payload should follow the `quota_exceeded:<limit_name>:<current>:<max>`
892    /// shape so HTTP / wire surfaces can map to the right status
893    /// (507 for storage, 429 for rate, 504 for duration, 413 for
894    /// payload).
895    QuotaExceeded(String),
896    /// Issue #769 (PRD #759 / S10) — an aggregating executor
897    /// (`aggregation`, `sort`, `window`) exceeded
898    /// `stream.executor.max_materialized_rows` in its in-memory state.
899    /// Carries the executor that fired, the configured ceiling and the
900    /// live row count at the breach so the client can decide whether to
901    /// redesign the query, raise the limit, or pre-aggregate. Surfaces
902    /// as HTTP 507 with a structured `materialization_limit_exceeded`
903    /// envelope; NDJSON streams emit the same code mid-stream.
904    MaterializationLimitExceeded {
905        executor: &'static str,
906        limit: usize,
907        current: usize,
908    },
909    Internal(String),
910}
911
912impl fmt::Display for RedDBError {
913    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
914        match self {
915            Self::InvalidConfig(msg) => write!(f, "invalid config: {msg}"),
916            Self::SchemaVersionMismatch { expected, found } => {
917                write!(
918                    f,
919                    "schema version mismatch: expected {expected}, found {found}"
920                )
921            }
922            Self::FeatureNotEnabled(msg) => write!(f, "feature disabled: {msg}"),
923            Self::NotFound(msg) => write!(f, "not found: {msg}"),
924            Self::ReadOnly(msg) => write!(f, "read-only violation: {msg}"),
925            Self::InvalidOperation(msg) => write!(f, "INVALID_OPERATION: {msg}"),
926            Self::Engine(msg) => write!(f, "engine error: {msg}"),
927            Self::Catalog(msg) => write!(f, "catalog error: {msg}"),
928            Self::StorageIntegrity(err) => write!(f, "{err}"),
929            Self::Query(msg) => write!(f, "query error: {msg}"),
930            Self::Validation { message, .. } => write!(f, "validation error: {message}"),
931            Self::Io(err) => write!(f, "io error: {err}"),
932            Self::VersionUnavailable => write!(f, "version information unavailable"),
933            Self::QuotaExceeded(msg) => write!(f, "quota exceeded: {msg}"),
934            Self::MaterializationLimitExceeded {
935                executor,
936                limit,
937                current,
938            } => write!(
939                f,
940                "materialization limit exceeded: executor={executor} current={current} limit={limit}"
941            ),
942            Self::Internal(msg) => write!(f, "internal error: {msg}"),
943        }
944    }
945}
946
947impl std::error::Error for RedDBError {}
948
949impl From<io::Error> for RedDBError {
950    fn from(err: io::Error) -> Self {
951        Self::Io(err)
952    }
953}
954
955impl From<crate::storage::engine::DatabaseError> for RedDBError {
956    fn from(err: crate::storage::engine::DatabaseError) -> Self {
957        Self::Engine(err.to_string())
958    }
959}
960
961impl From<crate::storage::wal::TxError> for RedDBError {
962    fn from(err: crate::storage::wal::TxError) -> Self {
963        Self::Engine(err.to_string())
964    }
965}
966
967impl From<crate::storage::StoreError> for RedDBError {
968    fn from(err: crate::storage::StoreError) -> Self {
969        match err {
970            crate::storage::StoreError::StorageIntegrity(err) => Self::StorageIntegrity(err),
971            other => Self::Catalog(other.to_string()),
972        }
973    }
974}
975
976impl From<crate::storage::unified::devx::DevXError> for RedDBError {
977    fn from(err: crate::storage::unified::devx::DevXError) -> Self {
978        match err {
979            crate::storage::unified::devx::DevXError::Validation(msg) => Self::InvalidConfig(msg),
980            crate::storage::unified::devx::DevXError::Storage(msg) => Self::Engine(msg),
981            crate::storage::unified::devx::DevXError::NotFound(msg) => Self::NotFound(msg),
982        }
983    }
984}
985
986pub trait CatalogService {
987    fn list_collections(&self) -> Vec<String>;
988    fn collection_stats(&self, collection: &str) -> Option<CollectionStats>;
989    fn catalog_snapshot(&self) -> CatalogSnapshot;
990}
991
992pub trait QueryPlanner {
993    fn plan_cost(&self, query: &str) -> Option<f64>;
994}
995
996pub trait DataOps {
997    fn execute_query(&self, query: &str) -> RedDBResult<()>;
998}
999
1000pub mod prelude {
1001    pub use super::{
1002        Capability, CapabilitySet, CatalogService, CatalogSnapshot, CollectionStats, DataOps,
1003        QueryPlanner, RedDBError, RedDBOptions, RedDBResult, SchemaManifest, StorageMode,
1004        REDDB_FORMAT_VERSION, REDDB_PROTOCOL_VERSION,
1005    };
1006}
reddb_server/api.rs

reddb_server/
api.rs
