reddb_server/

api.rs

//! Public API layer for the RedDB crate.
//!
//! This module is the first layer to consume from applications:
//! - stable options and contracts
//! - capability declarations
//! - typed errors and lightweight metadata snapshots
//! - cross-layer traits for catalog/operations observability

use std::collections::{BTreeMap, BTreeSet};
use std::fmt;
use std::io;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::{SystemTime, UNIX_EPOCH};

use crate::auth::AuthConfig;
use crate::replication::ReplicationConfig;

pub const DEFAULT_SNAPSHOT_RETENTION: usize = 16;
pub const DEFAULT_EXPORT_RETENTION: usize = 16;

pub const REDDB_PROTOCOL_VERSION: &str = "reddb-v2";
pub const REDDB_FORMAT_VERSION: u32 = 2;
/// Default group-commit window.
///
/// `0` = "no wait" — the background flusher fsyncs as soon as any
/// writer's pending commit arrives. Under single-writer workloads a
/// non-zero window would be pure latency (no one to batch with),
/// capping individual commit throughput at ~1000/s for window=1.
/// Concurrent writers still batch naturally via the `Mutex<WalWriter>`
/// contention path without needing an explicit timer.
///
/// Operators with many concurrent clients can raise this (e.g. 1-5ms)
/// to amortise fsync cost across a bigger batch — at the cost of p99
/// tail latency going up by the window size.
pub const DEFAULT_GROUP_COMMIT_WINDOW_MS: u64 = 0;
pub const DEFAULT_GROUP_COMMIT_MAX_STATEMENTS: usize = 128;
pub const DEFAULT_GROUP_COMMIT_MAX_WAL_BYTES: u64 = 1024 * 1024;
pub(crate) const EPHEMERAL_RUNTIME_METADATA_KEY: &str = "__reddb_ephemeral_runtime";

pub type RedDBResult<T> = Result<T, RedDBError>;

#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)]
pub enum StorageMode {
    /// Durable, file-backed database with WAL + checkpointing.
    #[default]
    Persistent,
}

impl StorageMode {
    pub const fn is_persistent(self) -> bool {
        matches!(self, Self::Persistent)
    }
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)]
pub enum DurabilityMode {
    #[default]
    Strict,
    WalDurableGrouped,
    /// Fire-and-forget. Writers return as soon as the WAL record is
    /// in the in-memory ring buffer — they do NOT wait for fsync.
    /// The group-commit background thread still flushes on its
    /// cadence, so committed-but-unflushed work is bounded by
    /// `GroupCommitOptions::window_ms`. A crash inside the window
    /// drops whatever wasn't flushed — matches PG's
    /// `synchronous_commit=off` contract.
    Async,
}

impl DurabilityMode {
    pub const fn as_str(self) -> &'static str {
        match self {
            Self::Strict => "strict",
            Self::WalDurableGrouped => "wal_durable_grouped",
            Self::Async => "async",
        }
    }

    pub fn from_str(value: &str) -> Option<Self> {
        let normalized = value.trim().to_ascii_lowercase();
        match normalized.as_str() {
            // Legacy / opt-out form. Every commit pays its own fsync.
            "strict" => Some(Self::Strict),
            // Group-commit sync path — the perf-parity default. Matches
            // PostgreSQL's `synchronous_commit=on` behaviour: the
            // writer waits for durability, but fsyncs are batched
            // across concurrent writers so a burst of N commits pays
            // ~O(1) fsyncs instead of O(N).
            "sync"
            | "wal_durable_grouped"
            | "wal-durable-grouped"
            | "grouped"
            | "wal_grouped"
            | "wal-grouped" => Some(Self::WalDurableGrouped),
            // Fire-and-forget async: writers return as soon as the
            // WAL buffer accepts the record; background flusher runs
            // on its configured cadence.
            "async" | "fire_and_forget" | "fire-and-forget" => Some(Self::Async),
            _ => None,
        }
    }
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct GroupCommitOptions {
    pub window_ms: u64,
    pub max_statements: usize,
    pub max_wal_bytes: u64,
}

impl Default for GroupCommitOptions {
    fn default() -> Self {
        Self {
            window_ms: DEFAULT_GROUP_COMMIT_WINDOW_MS,
            max_statements: DEFAULT_GROUP_COMMIT_MAX_STATEMENTS,
            max_wal_bytes: DEFAULT_GROUP_COMMIT_MAX_WAL_BYTES,
        }
    }
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum Capability {
    /// Structured row storage.
    Table,
    /// Graph nodes/edges.
    Graph,
    /// Vector collections and ANN search.
    Vector,
    /// Full-text / lexical search.
    FullText,
    /// Text/metadata security and enrichment modules.
    Security,
    /// Encryption at rest.
    Encryption,
}

impl Capability {
    pub const fn as_str(self) -> &'static str {
        match self {
            Self::Table => "table",
            Self::Graph => "graph",
            Self::Vector => "vector",
            Self::FullText => "fulltext",
            Self::Security => "security",
            Self::Encryption => "encryption",
        }
    }
}

#[derive(Debug, Clone, Default)]
pub struct CapabilitySet {
    items: BTreeSet<Capability>,
}

impl CapabilitySet {
    pub fn new() -> Self {
        Self::default()
    }

    pub fn with(mut self, capability: Capability) -> Self {
        self.items.insert(capability);
        self
    }

    pub fn with_all(mut self, capabilities: &[Capability]) -> Self {
        capabilities.iter().copied().for_each(|capability| {
            self.items.insert(capability);
        });
        self
    }

    pub fn has(&self, capability: Capability) -> bool {
        self.items.contains(&capability)
    }

    pub fn as_slice(&self) -> Vec<Capability> {
        self.items.iter().copied().collect()
    }
}

pub struct RedDBOptions {
    pub mode: StorageMode,
    pub data_path: Option<PathBuf>,
    pub read_only: bool,
    pub create_if_missing: bool,
    pub verify_checksums: bool,
    pub durability_mode: DurabilityMode,
    pub group_commit: GroupCommitOptions,
    pub auto_checkpoint_pages: u32,
    /// Maximum hypertable chunks the checkpoint path may seal into derived
    /// columnar artifacts per checkpoint. `usize::MAX` means no practical cap.
    pub checkpoint_columnar_emission_budget_chunks: usize,
    /// Minimum rows a sealed in-scope chunk must carry before the derived
    /// columnar projection is materialized.
    pub columnar_projection_size_floor_rows: usize,
    pub cache_pages: usize,
    pub snapshot_retention: usize,
    pub export_retention: usize,
    pub feature_gates: CapabilitySet,
    pub force_create: bool,
    pub metadata: BTreeMap<String, String>,
    /// Optional remote storage backend for snapshot transport.
    pub remote_backend: Option<Arc<dyn crate::storage::backend::RemoteBackend>>,
    /// Optional CAS-capable handle to the same backend, populated by
    /// the factory when the configured backend implements
    /// `AtomicRemoteBackend` (S3/local always; HTTP only when
    /// `RED_HTTP_CONDITIONAL_WRITES=true`). `None` for backends that
    /// do not provide compare-and-swap (Turso, D1, plain HTTP).
    /// `LeaseStore` and any future CAS consumer pull from this field.
    pub remote_backend_atomic: Option<Arc<dyn crate::storage::backend::AtomicRemoteBackend>>,
    /// Remote object key used by the remote backend.
    pub remote_key: Option<String>,
    /// Replication configuration.
    pub replication: ReplicationConfig,
    /// Whether replica runtimes start the background replication loop.
    pub replica_loop_enabled: bool,
    /// Authentication & authorization configuration.
    pub auth: AuthConfig,
    /// Control Event Ledger configuration (issue #652). Read from
    /// `REDDB_COMPLIANCE_MODE` at boot.
    pub control_events: crate::runtime::control_events::ControlEventConfig,
    /// Scoped data-plane query audit configuration. Disabled by
    /// default; the regulated preset enables the stream without
    /// adding catch-all rules.
    pub query_audit: crate::runtime::query_audit::QueryAuditConfig,
    /// Auto-create a HASH index on a user `id` column the first time a
    /// row carrying that column is inserted into a collection. See
    /// `UnifiedStoreConfig::auto_index_id`. Defaults to `true`; set to
    /// `false` to opt out per workload (e.g. ingest pipelines that
    /// don't need point-lookups by `id`).
    pub auto_index_id: bool,
    /// Tiered storage layout preset. Drives the defaults of the six
    /// tier-flag toggles (`.meta.json` sidecar, seq-N catalog journal,
    /// `-shm` provisioning, audit/slow log destinations,
    /// `fold_dwb_into_wal`). Explicit per-feature setters still win over
    /// the tier default. See `crate::storage::layout::StorageLayout`.
    pub layout: crate::storage::layout::StorageLayout,
    /// Per-feature overrides applied on top of the resolved layout preset.
    pub layout_overrides: crate::storage::layout::LayoutOverrides,
    /// Operator-facing storage/deploy profile contract. This records the
    /// selected posture before the physical directory layout is expanded.
    pub storage_profile: crate::storage::profile::StorageProfileSelection,
    /// True only when the caller explicitly selected a layout via
    /// `with_layout`. When false, `apply_tier_defaults` short-circuits so
    /// pre-existing process-global toggles (set by tests / env hatches
    /// before opening a runtime) are not clobbered by the implicit
    /// `Standard` default. The new tier-driven behavior is opt-in.
    pub layout_explicit: bool,
    /// Explicit operator memory budget in bytes — the top tier of the ADR
    /// 0073 §1 precedence chain. `None` defers to `REDDB_MEMORY_BUDGET`, the
    /// deployment-profile default, the cgroup limit, and finally a fraction
    /// of physical RAM. There is no "unlimited" value: `Some(0)` is rejected
    /// at boot, exactly like an unparseable env var.
    pub memory_budget_bytes: Option<u64>,
    /// Opt into bounded resident segments and generation-bound disk indexes.
    /// Currently supported only for standalone, single-file databases. Preparing
    /// the file upgrades its writer format; retain an unconverted downgrade copy.
    /// This is not a guarantee that every server workspace is memory-bounded.
    pub bounded_disk_residency: bool,
}

impl fmt::Debug for RedDBOptions {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        let backend_name = self.remote_backend.as_ref().map(|b| b.name().to_string());
        f.debug_struct("RedDBOptions")
            .field("mode", &self.mode)
            .field("data_path", &self.data_path)
            .field("read_only", &self.read_only)
            .field("create_if_missing", &self.create_if_missing)
            .field("verify_checksums", &self.verify_checksums)
            .field("durability_mode", &self.durability_mode)
            .field("group_commit", &self.group_commit)
            .field("auto_checkpoint_pages", &self.auto_checkpoint_pages)
            .field(
                "checkpoint_columnar_emission_budget_chunks",
                &self.checkpoint_columnar_emission_budget_chunks,
            )
            .field(
                "columnar_projection_size_floor_rows",
                &self.columnar_projection_size_floor_rows,
            )
            .field("cache_pages", &self.cache_pages)
            .field("snapshot_retention", &self.snapshot_retention)
            .field("export_retention", &self.export_retention)
            .field("feature_gates", &self.feature_gates)
            .field("force_create", &self.force_create)
            .field("metadata", &self.metadata)
            .field("remote_backend", &backend_name)
            .field("remote_key", &self.remote_key)
            .field("replication", &self.replication)
            .field("replica_loop_enabled", &self.replica_loop_enabled)
            .field("auth", &self.auth)
            .field("control_events", &self.control_events)
            .field("query_audit", &self.query_audit)
            .field("layout", &self.layout)
            .field("layout_overrides", &self.layout_overrides)
            .field("storage_profile", &self.storage_profile)
            .field("memory_budget_bytes", &self.memory_budget_bytes)
            .field("bounded_disk_residency", &self.bounded_disk_residency)
            .finish()
    }
}

impl Clone for RedDBOptions {
    fn clone(&self) -> Self {
        Self {
            mode: self.mode,
            data_path: self.data_path.clone(),
            read_only: self.read_only,
            create_if_missing: self.create_if_missing,
            verify_checksums: self.verify_checksums,
            durability_mode: self.durability_mode,
            group_commit: self.group_commit,
            auto_checkpoint_pages: self.auto_checkpoint_pages,
            checkpoint_columnar_emission_budget_chunks: self
                .checkpoint_columnar_emission_budget_chunks,
            columnar_projection_size_floor_rows: self.columnar_projection_size_floor_rows,
            cache_pages: self.cache_pages,
            snapshot_retention: self.snapshot_retention,
            export_retention: self.export_retention,
            feature_gates: self.feature_gates.clone(),
            force_create: self.force_create,
            metadata: self.metadata.clone(),
            remote_backend: self.remote_backend.clone(),
            remote_backend_atomic: self.remote_backend_atomic.clone(),
            remote_key: self.remote_key.clone(),
            replication: self.replication.clone(),
            replica_loop_enabled: self.replica_loop_enabled,
            auth: self.auth.clone(),
            control_events: self.control_events,
            query_audit: self.query_audit.clone(),
            auto_index_id: self.auto_index_id,
            layout: self.layout,
            layout_overrides: self.layout_overrides.clone(),
            storage_profile: self.storage_profile,
            layout_explicit: self.layout_explicit,
            memory_budget_bytes: self.memory_budget_bytes,
            bounded_disk_residency: self.bounded_disk_residency,
        }
    }
}

impl Default for RedDBOptions {
    fn default() -> Self {
        Self {
            mode: StorageMode::Persistent,
            data_path: None,
            read_only: false,
            create_if_missing: true,
            verify_checksums: true,
            // Perf-parity default — `WalDurableGrouped` matches
            // PostgreSQL's `synchronous_commit=on` behaviour while
            // amortising fsync cost across concurrent writers. The
            // legacy `Strict` tier (per-commit fsync) stays available
            // via `durability.mode = "strict"` / `REDDB_DURABILITY=strict`.
            durability_mode: DurabilityMode::WalDurableGrouped,
            group_commit: GroupCommitOptions::default(),
            auto_checkpoint_pages: 1000,
            checkpoint_columnar_emission_budget_chunks: usize::MAX,
            columnar_projection_size_floor_rows: 4,
            cache_pages: 10_000,
            snapshot_retention: DEFAULT_SNAPSHOT_RETENTION,
            export_retention: DEFAULT_EXPORT_RETENTION,
            feature_gates: CapabilitySet::new()
                .with(Capability::Table)
                .with(Capability::Graph)
                .with(Capability::Vector),
            force_create: true,
            metadata: BTreeMap::new(),
            remote_backend: None,
            remote_backend_atomic: None,
            remote_key: None,
            replication: ReplicationConfig::standalone(),
            replica_loop_enabled: true,
            auth: AuthConfig::default(),
            control_events: crate::runtime::control_events::ControlEventConfig::default(),
            query_audit: crate::runtime::query_audit::QueryAuditConfig::default(),
            auto_index_id: true,
            layout: crate::storage::layout::StorageLayout::default(),
            layout_overrides: crate::storage::layout::LayoutOverrides::default(),
            storage_profile: crate::storage::profile::StorageProfileSelection::embedded_single_file(
            ),
            layout_explicit: false,
            memory_budget_bytes: None,
            bounded_disk_residency: false,
        }
    }
}

impl RedDBOptions {
    pub fn persistent<P: Into<PathBuf>>(path: P) -> Self {
        Self {
            mode: StorageMode::Persistent,
            data_path: Some(path.into()),
            ..Default::default()
        }
    }

    /// Ephemeral, tempfile-backed database.
    ///
    /// The underlying storage is a real persistent file placed under the system
    /// temp directory with a unique name — there is no longer a true in-memory
    /// execution mode. Prefer [`RedDBOptions::persistent`] when the data should
    /// outlive the process.
    pub fn in_memory() -> Self {
        static NEXT_EPHEMERAL_ID: std::sync::atomic::AtomicU64 =
            std::sync::atomic::AtomicU64::new(0);

        let now_nanos = std::time::SystemTime::now()
            .duration_since(std::time::UNIX_EPOCH)
            .map(|duration| duration.as_nanos())
            .unwrap_or(0);
        let unique = NEXT_EPHEMERAL_ID.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
        // Each ephemeral runtime gets its OWN directory, not just a unique
        // filename in the shared temp dir. Sibling artifacts (the audit log,
        // WAL, snapshots) are derived from this data path's PARENT, so a
        // shared parent collapses every ephemeral runtime's `.audit.log`
        // onto one file — which parallel test processes (nextest runs each
        // test in its own process, frequently under one shared `TMPDIR`)
        // then truncate/remove out from under one another. A per-instance
        // directory keeps each runtime's siblings self-contained.
        let dir = std::env::temp_dir().join(format!(
            "reddb-ephemeral-{}-{}-{}",
            std::process::id(),
            now_nanos,
            unique
        ));
        let _ = std::fs::create_dir_all(&dir);
        let path = dir.join("db.rdb");
        let _ = std::fs::remove_file(&path);
        let mut metadata = BTreeMap::new();
        metadata.insert(
            EPHEMERAL_RUNTIME_METADATA_KEY.to_string(),
            "true".to_string(),
        );
        Self {
            mode: StorageMode::Persistent,
            data_path: Some(path),
            auto_checkpoint_pages: 0,
            checkpoint_columnar_emission_budget_chunks: usize::MAX,
            columnar_projection_size_floor_rows: 4,
            cache_pages: 2_000,
            snapshot_retention: DEFAULT_SNAPSHOT_RETENTION,
            export_retention: DEFAULT_EXPORT_RETENTION,
            read_only: false,
            force_create: true,
            metadata,
            ..Default::default()
        }
    }

    pub fn with_mode(mut self, mode: StorageMode) -> Self {
        self.mode = mode;
        self
    }

    pub fn with_data_path<P: Into<PathBuf>>(mut self, path: P) -> Self {
        // `in_memory()` eagerly creates a `reddb-ephemeral-*` dir and points
        // `data_path` at it. Overriding the path abandons that dir — and the
        // per-runtime cleanup keys off the FINAL `data_path`, so the orphan
        // would leak in TMPDIR (caught by scripts/check-temp-residue.sh). Remove
        // it now if the previous path was such an eagerly-created ephemeral dir.
        if let Some(old) = self.data_path.take() {
            let tmp = std::env::temp_dir();
            if let Some(parent) = old.parent() {
                let orphaned = parent
                    .file_name()
                    .and_then(|name| name.to_str())
                    .is_some_and(|name| name.starts_with("reddb-ephemeral-"))
                    && parent.parent() == Some(tmp.as_path());
                if orphaned {
                    let _ = std::fs::remove_dir_all(parent);
                }
            }
        }
        self.data_path = Some(path.into());
        self
    }

    pub fn with_read_only(mut self, read_only: bool) -> Self {
        self.read_only = read_only;
        self
    }

    pub fn with_auto_checkpoint(mut self, pages: u32) -> Self {
        self.auto_checkpoint_pages = pages;
        self
    }

    pub fn with_checkpoint_columnar_emission_budget_chunks(mut self, chunks: usize) -> Self {
        self.checkpoint_columnar_emission_budget_chunks = chunks;
        self
    }

    pub fn with_columnar_projection_size_floor_rows(mut self, rows: usize) -> Self {
        self.columnar_projection_size_floor_rows = rows;
        self
    }

    pub fn with_durability_mode(mut self, mode: DurabilityMode) -> Self {
        self.durability_mode = mode;
        self
    }

    pub fn with_group_commit_window_ms(mut self, window_ms: u64) -> Self {
        // `0` is a legitimate setting — "no wait, fsync on every wakeup".
        // See `DEFAULT_GROUP_COMMIT_WINDOW_MS` docs for the tradeoff.
        self.group_commit.window_ms = window_ms;
        self
    }

    pub fn with_group_commit_max_statements(mut self, max_statements: usize) -> Self {
        self.group_commit.max_statements = max_statements.max(1);
        self
    }

    pub fn with_group_commit_max_wal_bytes(mut self, max_wal_bytes: u64) -> Self {
        self.group_commit.max_wal_bytes = max_wal_bytes.max(1);
        self
    }

    pub fn with_cache_pages(mut self, pages: usize) -> Self {
        self.cache_pages = pages.max(2);
        self
    }

    pub fn with_snapshot_retention(mut self, limit: usize) -> Self {
        self.snapshot_retention = limit.max(1);
        self
    }

    pub fn with_export_retention(mut self, limit: usize) -> Self {
        self.export_retention = limit.max(1);
        self
    }

    pub fn with_metadata<K: Into<String>, V: Into<String>>(mut self, key: K, value: V) -> Self {
        self.metadata.insert(key.into(), value.into());
        self
    }

    /// Toggle the implicit HASH index on user `id` columns at first
    /// insert (#112). Defaults to enabled — pass `false` to fall back
    /// to the legacy "scan unless `CREATE INDEX` is issued" behaviour.
    pub fn with_auto_index_id(mut self, enabled: bool) -> Self {
        self.auto_index_id = enabled;
        self
    }

    pub fn with_capability(mut self, capability: Capability) -> Self {
        self.feature_gates = self.feature_gates.with(capability);
        self
    }

    /// Attach a remote storage backend for snapshot transport.
    ///
    /// On open, the database snapshot is downloaded from the remote `key`
    /// to the local data path. On flush, the local file is uploaded back
    /// to the remote backend under the same key.
    pub fn with_remote_backend(
        mut self,
        backend: Arc<dyn crate::storage::backend::RemoteBackend>,
        key: impl Into<String>,
    ) -> Self {
        self.remote_backend = Some(backend);
        self.remote_key = Some(key.into());
        self
    }

    /// Attach a CAS-capable backend handle. Pass the same `Arc` as
    /// `with_remote_backend` (factories should construct the backend
    /// once and call both setters); this method exists so the type
    /// system, not runtime config, decides whether `LeaseStore` is
    /// reachable.
    pub fn with_atomic_remote_backend(
        mut self,
        backend: Arc<dyn crate::storage::backend::AtomicRemoteBackend>,
    ) -> Self {
        self.remote_backend_atomic = Some(backend);
        self
    }

    pub fn with_replication(mut self, config: ReplicationConfig) -> Self {
        self.replication = config;
        self
    }

    pub fn with_replica_loop_enabled(mut self, enabled: bool) -> Self {
        self.replica_loop_enabled = enabled;
        self
    }

    pub fn with_auth(mut self, config: AuthConfig) -> Self {
        self.auth = config;
        self
    }

    pub fn resolved_path(&self, fallback: impl AsRef<Path>) -> PathBuf {
        self.data_path
            .clone()
            .unwrap_or_else(|| fallback.as_ref().to_path_buf())
    }

    pub fn remote_namespace_prefix(&self) -> String {
        let Some(remote_key) = &self.remote_key else {
            return String::new();
        };
        let normalized = remote_key.trim_matches('/');
        if normalized.is_empty() {
            return String::new();
        }
        match normalized.rsplit_once('/') {
            Some((parent, _)) if !parent.is_empty() => format!("{parent}/"),
            _ => String::new(),
        }
    }

    pub fn default_backup_head_key(&self) -> String {
        if let Some(value) = self.metadata.get("red.config.backup.head_key") {
            return value.clone();
        }
        reddb_file::backup_head_key(&self.remote_namespace_prefix())
    }

    pub fn default_snapshot_prefix(&self) -> String {
        if let Some(value) = self.metadata.get("red.config.backup.snapshot_prefix") {
            return value.clone();
        }
        reddb_file::backup_snapshot_prefix(&self.remote_namespace_prefix())
    }

    pub fn default_wal_archive_prefix(&self) -> String {
        if let Some(value) = self.metadata.get("red.config.wal.archive.prefix") {
            return value.clone();
        }
        reddb_file::backup_wal_prefix(&self.remote_namespace_prefix())
    }

    pub fn has_capability(&self, capability: Capability) -> bool {
        self.feature_gates.has(capability)
    }

    /// Select the active storage-layout preset. Drives tier defaults for
    /// the six tier-flag toggles. Per-feature setters still win.
    pub fn with_layout(mut self, layout: crate::storage::layout::StorageLayout) -> Self {
        self.layout = layout;
        self.layout_explicit = true;
        self
    }

    /// Override individual layout knobs (dedicated dirs + log routing) on
    /// top of the active preset.
    pub fn with_layout_overrides(
        mut self,
        overrides: crate::storage::layout::LayoutOverrides,
    ) -> Self {
        self.layout_overrides = overrides;
        self
    }

    /// Declare the process memory budget explicitly (ADR 0073 §1, tier 1).
    /// Wins over `REDDB_MEMORY_BUDGET`, the profile default, the cgroup limit
    /// and physical-RAM detection. A zero budget is rejected when the runtime
    /// opens, not silently swapped for a default.
    pub fn with_memory_budget(mut self, bytes: u64) -> Self {
        self.memory_budget_bytes = Some(bytes);
        self
    }

    /// Select disk-backed residency explicitly, using the segment share of the
    /// resolved memory budget. Opening an unsupported profile fails before I/O.
    pub fn with_bounded_disk_residency(mut self, enabled: bool) -> Self {
        self.bounded_disk_residency = enabled;
        self
    }

    pub fn with_storage_profile(
        mut self,
        selection: crate::storage::profile::StorageProfileSelection,
    ) -> Result<Self, String> {
        self.storage_profile = selection.validate()?;
        Ok(self)
    }

    /// Resolve `(data_path, TieredLayoutPaths)` for this options bundle.
    /// Returns `None` when `data_path` is unset (in-memory ephemeral
    /// instances don't materialise a support tree).
    pub fn resolve_tiered_layout(
        &self,
    ) -> Option<(PathBuf, crate::storage::layout::TieredLayoutPaths)> {
        let data_path = self.data_path.clone()?;
        let paths = crate::storage::layout::TieredLayoutPaths::new(
            &data_path,
            self.layout,
            self.layout_overrides.clone(),
        );
        Some((data_path, paths))
    }

    /// Flip the process-global tier-flag toggles to match this options
    /// bundle's layout, and stash the resolved [`TieredLayoutPaths`] for
    /// status surfaces. Idempotent. Per-feature env escape hatches
    /// (`REDDB_META_JSON_SIDECAR=...` and friends) still override what
    /// this method sets — they are read inside the per-toggle getter, not
    /// here.
    ///
    /// Tier defaults:
    ///
    /// | toggle                     | minimal | standard | performance | max |
    /// |----------------------------|:-------:|:--------:|:-----------:|:---:|
    /// | `.meta.json` sidecar       |   off   |    off   |     off     |  on |
    /// | seq-N catalog journal      |   off   |    off   |     off     |  on |
    /// | `-shm` provisioning        |   off   | **on**   |   **on**    |  on |
    /// | `fold_dwb_into_wal`        |   off   |    off   |     off     |  on |
    /// | audit/slow log destination | stderr  |  stderr  |  file       | file|
    ///
    /// Retention for the seq-N journal: 32 at `Max`, 4 when the operator
    /// opts in on a lower tier via `REDDB_SEQN_JOURNAL=1`.
    pub fn apply_tier_defaults(&self) {
        use crate::storage::layout::StorageLayout;

        // Opt-in: only flip the process-global toggles when the operator
        // explicitly chose a layout via `with_layout`. Tests and env
        // hatches that pre-set toggles before opening a runtime must not
        // be silently overridden by the implicit `Standard` default.
        if !self.layout_explicit {
            if let Some((_, paths)) = self.resolve_tiered_layout() {
                tier_wiring::stash_layout_paths(paths);
            }
            return;
        }

        let layout = self.layout;
        // .meta.json sidecar — Max only.
        crate::physical::set_meta_json_sidecar_enabled(matches!(layout, StorageLayout::Max));

        // Seq-N catalog journal — Max only. Retention = 32 for Max,
        // OPT_IN baseline (4) for lower tiers (the value is consulted
        // when an env hatch flips the toggle on outside Max).
        crate::physical::set_seqn_journal_enabled(matches!(layout, StorageLayout::Max));
        crate::physical::set_seqn_journal_retention(match layout {
            StorageLayout::Max => crate::physical::DEFAULT_METADATA_JOURNAL_RETENTION,
            _ => crate::physical::OPT_IN_METADATA_JOURNAL_RETENTION,
        });

        // `-shm` provisioning — anything ≥ Standard (gh-475 acceptance).
        crate::physical::set_shm_provisioning_enabled(matches!(
            layout,
            StorageLayout::Standard | StorageLayout::Performance | StorageLayout::Max
        ));

        // Fold DWB into WAL — Max only (a single recovery substrate is the
        // Max story). The pager manifest is folded into the `.rdb` on every
        // tier since ADR 0038 §4 phase 1, so it has no toggle.
        crate::physical::set_fold_dwb_into_wal_enabled(matches!(layout, StorageLayout::Max));

        // Cache resolved layout paths for `red status` / diagnostics.
        if let Some((_, paths)) = self.resolve_tiered_layout() {
            tier_wiring::stash_layout_paths(paths);
        }
    }
}

/// Process-global cache of the most recently applied [`TieredLayoutPaths`].
/// Read by `red status` to surface resolved log destinations. Written by
/// `RedDBOptions::apply_tier_defaults` after each open.
pub mod tier_wiring {
    use std::sync::Mutex;

    use crate::storage::layout::{LogDestination, TieredLayoutPaths};

    static CURRENT_LAYOUT_PATHS: Mutex<Option<TieredLayoutPaths>> = Mutex::new(None);

    pub fn stash_layout_paths(paths: TieredLayoutPaths) {
        if let Ok(mut slot) = CURRENT_LAYOUT_PATHS.lock() {
            *slot = Some(paths);
        }
    }

    pub fn current_layout_paths() -> Option<TieredLayoutPaths> {
        CURRENT_LAYOUT_PATHS
            .lock()
            .ok()
            .and_then(|slot| slot.clone())
    }

    /// `(audit_log, slow_log)` destinations resolved from the active
    /// layout. Falls back to `(Stderr, Stderr)` when no layout has been
    /// applied yet (e.g. ephemeral in-memory paths).
    pub fn current_log_destinations() -> (LogDestination, LogDestination) {
        match current_layout_paths() {
            Some(p) => (p.audit_log_destination, p.slow_log_destination),
            None => (LogDestination::Stderr, LogDestination::Stderr),
        }
    }
}

#[derive(Debug, Clone, Default)]
pub struct CollectionStats {
    pub entities: usize,
    pub cross_refs: usize,
    pub segments: usize,
}

#[derive(Debug, Clone)]
pub struct CatalogSnapshot {
    pub name: String,
    pub total_entities: usize,
    pub total_collections: usize,
    pub stats_by_collection: BTreeMap<String, CollectionStats>,
    pub updated_at: SystemTime,
}

impl Default for CatalogSnapshot {
    fn default() -> Self {
        Self {
            name: String::new(),
            total_entities: 0,
            total_collections: 0,
            stats_by_collection: BTreeMap::new(),
            updated_at: UNIX_EPOCH,
        }
    }
}

#[derive(Debug, Clone)]
pub struct SchemaManifest {
    pub format_version: u32,
    pub created_at_unix_ms: u128,
    pub updated_at_unix_ms: u128,
    pub options: RedDBOptions,
    pub collection_count: usize,
}

impl SchemaManifest {
    pub fn now(options: RedDBOptions, collection_count: usize) -> Self {
        let now = SystemTime::now()
            .duration_since(UNIX_EPOCH)
            .unwrap_or_default()
            .as_millis();
        Self {
            format_version: REDDB_FORMAT_VERSION,
            created_at_unix_ms: now,
            updated_at_unix_ms: now,
            options,
            collection_count,
        }
    }
}

pub use reddb_engine::storage::error::StorageIntegrityError;

#[derive(Debug)]
pub enum RedDBError {
    InvalidConfig(String),
    SchemaVersionMismatch {
        expected: u32,
        found: u32,
    },
    FeatureNotEnabled(String),
    NotFound(String),
    ReadOnly(String),
    InvalidOperation(String),
    Engine(String),
    Catalog(String),
    StorageIntegrity(StorageIntegrityError),
    Query(String),
    Validation {
        message: String,
        validation: crate::json::Value,
    },
    Io(io::Error),
    VersionUnavailable,
    /// Operator-pinned cap exceeded (PLAN.md Phase 4.1). The string
    /// payload should follow the `quota_exceeded:<limit_name>:<current>:<max>`
    /// shape so HTTP / wire surfaces can map to the right status
    /// (507 for storage, 429 for rate, 504 for duration, 413 for
    /// payload).
    QuotaExceeded(String),
    /// Issue #769 (PRD #759 / S10) — an aggregating executor
    /// (`aggregation`, `sort`, `window`) exceeded
    /// `stream.executor.max_materialized_rows` in its in-memory state.
    /// Carries the executor that fired, the configured ceiling and the
    /// live row count at the breach so the client can decide whether to
    /// redesign the query, raise the limit, or pre-aggregate. Surfaces
    /// as HTTP 507 with a structured `materialization_limit_exceeded`
    /// envelope; NDJSON streams emit the same code mid-stream.
    MaterializationLimitExceeded {
        executor: &'static str,
        limit: usize,
        current: usize,
    },
    Internal(String),
}

/// The text a wire client may see for `err`.
///
/// `Io` and `Internal` carry OS messages, filesystem paths and storage
/// invariants meant for the operator's log, not for a peer; every other
/// variant is a statement-level message the caller needs verbatim.
pub fn client_facing_message(err: &RedDBError) -> String {
    match err {
        RedDBError::Io(inner) => format!("io error: {:?}", inner.kind()),
        RedDBError::Internal(_) => "internal error; see the server log".to_string(),
        other => other.to_string(),
    }
}

impl fmt::Display for RedDBError {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            Self::InvalidConfig(msg) => write!(f, "invalid config: {msg}"),
            Self::SchemaVersionMismatch { expected, found } => {
                write!(
                    f,
                    "schema version mismatch: expected {expected}, found {found}"
                )
            }
            Self::FeatureNotEnabled(msg) => write!(f, "feature disabled: {msg}"),
            Self::NotFound(msg) => write!(f, "not found: {msg}"),
            Self::ReadOnly(msg) => write!(f, "read-only violation: {msg}"),
            Self::InvalidOperation(msg) => write!(f, "INVALID_OPERATION: {msg}"),
            Self::Engine(msg) => write!(f, "engine error: {msg}"),
            Self::Catalog(msg) => write!(f, "catalog error: {msg}"),
            Self::StorageIntegrity(err) => write!(f, "{err}"),
            Self::Query(msg) => write!(f, "query error: {msg}"),
            Self::Validation { message, .. } => write!(f, "validation error: {message}"),
            Self::Io(err) => write!(f, "io error: {err}"),
            Self::VersionUnavailable => write!(f, "version information unavailable"),
            Self::QuotaExceeded(msg) => write!(f, "quota exceeded: {msg}"),
            Self::MaterializationLimitExceeded {
                executor,
                limit,
                current,
            } => write!(
                f,
                "materialization limit exceeded: executor={executor} current={current} limit={limit}"
            ),
            Self::Internal(msg) => write!(f, "internal error: {msg}"),
        }
    }
}

impl std::error::Error for RedDBError {}

impl From<io::Error> for RedDBError {
    fn from(err: io::Error) -> Self {
        Self::Io(err)
    }
}

impl From<crate::storage::engine::DatabaseError> for RedDBError {
    fn from(err: crate::storage::engine::DatabaseError) -> Self {
        Self::Engine(err.to_string())
    }
}

impl From<crate::storage::wal::TxError> for RedDBError {
    fn from(err: crate::storage::wal::TxError) -> Self {
        Self::Engine(err.to_string())
    }
}

impl From<reddb_engine::catalog::enforcement::ContractError> for RedDBError {
    fn from(error: reddb_engine::catalog::enforcement::ContractError) -> Self {
        match error {
            reddb_engine::catalog::enforcement::ContractError::Query(message) => {
                Self::Query(message)
            }
            reddb_engine::catalog::enforcement::ContractError::InvalidOperation(message) => {
                Self::InvalidOperation(message)
            }
        }
    }
}

impl From<reddb_engine::catalog::schema::ContractExpressionError> for RedDBError {
    fn from(error: reddb_engine::catalog::schema::ContractExpressionError) -> Self {
        Self::Query(error.message)
    }
}

impl From<crate::storage::StoreError> for RedDBError {
    fn from(err: crate::storage::StoreError) -> Self {
        match err {
            crate::storage::StoreError::StorageIntegrity(err) => Self::StorageIntegrity(err),
            other => Self::Catalog(other.to_string()),
        }
    }
}

impl From<crate::storage::unified::devx::DevXError> for RedDBError {
    fn from(err: crate::storage::unified::devx::DevXError) -> Self {
        match err {
            crate::storage::unified::devx::DevXError::Validation(msg) => Self::InvalidConfig(msg),
            crate::storage::unified::devx::DevXError::Storage(msg) => Self::Engine(msg),
            crate::storage::unified::devx::DevXError::NotFound(msg) => Self::NotFound(msg),
        }
    }
}

pub trait CatalogService {
    fn list_collections(&self) -> Vec<String>;
    fn collection_stats(&self, collection: &str) -> Option<CollectionStats>;
    fn catalog_snapshot(&self) -> CatalogSnapshot;
}

pub trait QueryPlanner {
    fn plan_cost(&self, query: &str) -> Option<f64>;
}

pub trait DataOps {
    fn execute_query(&self, query: &str) -> RedDBResult<()>;
}

pub mod prelude {
    pub use super::{
        Capability, CapabilitySet, CatalogService, CatalogSnapshot, CollectionStats, DataOps,
        QueryPlanner, RedDBError, RedDBOptions, RedDBResult, SchemaManifest, StorageMode,
        REDDB_FORMAT_VERSION, REDDB_PROTOCOL_VERSION,
    };
}

#[cfg(test)]
mod client_facing_message_tests {
    use super::{client_facing_message, RedDBError};

    #[test]
    fn client_facing_message_hides_io_and_internal_detail() {
        let io = RedDBError::Io(std::io::Error::new(
            std::io::ErrorKind::NotFound,
            "/var/lib/reddb/data.rdb: No such file",
        ));
        assert_eq!(client_facing_message(&io), "io error: NotFound");
        let internal = RedDBError::Internal("pager invariant broke at /srv/db.wal".to_string());
        assert!(!client_facing_message(&internal).contains("/srv"));
        let query = RedDBError::Query("unknown column `x`".to_string());
        assert_eq!(client_facing_message(&query), query.to_string());
    }
}