reddb_server/
physical.rs

1//! Physical storage design primitives for RedDB's deterministic on-disk layout.
2
3use std::collections::BTreeMap;
4use std::fs;
5use std::io;
6use std::path::{Path, PathBuf};
7use std::time::{SystemTime, UNIX_EPOCH};
8
9use crate::api::{CatalogSnapshot, CollectionStats, RedDBOptions, SchemaManifest, StorageMode};
10use crate::index::IndexKind;
11use crate::serde_json::{Map, Value as JsonValue};
12
13pub const DEFAULT_GRID_BLOCK_SIZE: usize = 512 * 1024;
14pub const DEFAULT_PAGE_SIZE: usize = 4096;
15pub use reddb_file::layout::PHYSICAL_METADATA_BINARY_EXTENSION;
16pub use reddb_file::{
17    fold_dwb_into_wal_enabled, meta_json_sidecar_enabled, seqn_journal_enabled,
18    seqn_journal_retention, set_fold_dwb_into_wal_enabled, set_meta_json_sidecar_enabled,
19    set_seqn_journal_enabled, set_seqn_journal_retention, BlockReference, ExportDescriptor,
20    ManifestEvent, ManifestEventKind, ManifestPointers, PhysicalAnalyticsJob,
21    PhysicalGraphProjection, PhysicalTreeDefinition, SnapshotDescriptor, SuperblockHeader,
22    DEFAULT_METADATA_JOURNAL_RETENTION, DEFAULT_SUPERBLOCK_COPIES,
23    OPT_IN_METADATA_JOURNAL_RETENTION, PHYSICAL_METADATA_PROTOCOL_VERSION,
24};
25pub const DEFAULT_MANIFEST_EVENT_HISTORY: usize = 256;
26
27#[derive(Debug, Clone, Copy, PartialEq, Eq)]
28pub enum PhysicalMetadataSource {
29    Binary,
30    BinaryJournal,
31    Json,
32}
33
34impl PhysicalMetadataSource {
35    pub fn as_str(self) -> &'static str {
36        match self {
37            Self::Binary => "binary",
38            Self::BinaryJournal => "binary_journal",
39            Self::Json => "json",
40        }
41    }
42}
43#[derive(Debug, Clone, Copy, PartialEq, Eq)]
44pub enum CompactionPolicy {
45    Incremental,
46    Manual,
47}
48
49#[derive(Debug, Clone)]
50pub struct WalPolicy {
51    pub auto_checkpoint_pages: u32,
52    pub fsync_on_commit: bool,
53    pub ring_buffer_bytes: u64,
54}
55
56impl Default for WalPolicy {
57    fn default() -> Self {
58        Self {
59            auto_checkpoint_pages: 1000,
60            fsync_on_commit: true,
61            ring_buffer_bytes: 64 * 1024 * 1024,
62        }
63    }
64}
65
66#[derive(Debug, Clone)]
67pub struct GridLayout {
68    pub block_size: usize,
69    pub page_size: usize,
70    pub superblock_copies: u8,
71}
72
73impl Default for GridLayout {
74    fn default() -> Self {
75        Self {
76            block_size: DEFAULT_GRID_BLOCK_SIZE,
77            page_size: DEFAULT_PAGE_SIZE,
78            superblock_copies: DEFAULT_SUPERBLOCK_COPIES,
79        }
80    }
81}
82
83#[derive(Debug, Clone)]
84pub struct PhysicalLayout {
85    pub mode: StorageMode,
86    pub grid: GridLayout,
87    pub wal: WalPolicy,
88    pub compaction: CompactionPolicy,
89}
90
91impl PhysicalLayout {
92    pub fn from_options(options: &RedDBOptions) -> Self {
93        Self {
94            mode: options.mode,
95            grid: GridLayout::default(),
96            wal: WalPolicy {
97                auto_checkpoint_pages: options.auto_checkpoint_pages,
98                ..WalPolicy::default()
99            },
100            compaction: CompactionPolicy::Incremental,
101        }
102    }
103
104    pub fn is_persistent(&self) -> bool {
105        self.mode == StorageMode::Persistent
106    }
107}
108
109#[derive(Debug, Clone, Copy, PartialEq, Eq)]
110pub enum ContractOrigin {
111    Explicit,
112    Implicit,
113    Migrated,
114}
115
116impl ContractOrigin {
117    pub fn as_str(self) -> &'static str {
118        match self {
119            Self::Explicit => "explicit",
120            Self::Implicit => "implicit",
121            Self::Migrated => "migrated",
122        }
123    }
124}
125
126/// A decoded contract is authoritative; it cannot be repaired from row contents.
127#[derive(Debug)]
128pub(crate) struct CollectionContractDecodeError(pub String);
129
130impl std::fmt::Display for CollectionContractDecodeError {
131    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
132        write!(formatter, "invalid collection contract: {}", self.0)
133    }
134}
135
136impl std::error::Error for CollectionContractDecodeError {}
137
138pub(crate) fn is_collection_contract_decode_error(error: &std::io::Error) -> bool {
139    error
140        .get_ref()
141        .is_some_and(|source| source.is::<CollectionContractDecodeError>())
142}
143
144#[derive(Debug, Clone)]
145pub struct DeclaredColumnContract {
146    pub name: String,
147    pub data_type: String,
148    pub sql_type: Option<reddb_types::SqlTypeName>,
149    pub not_null: bool,
150    pub default: Option<String>,
151    /// Persisted, deterministic expression for a stored generated field.
152    pub generated: Option<reddb_rql::schema_expression::SchemaExpression>,
153    /// SQL CHECK: FALSE rejects, TRUE or NULL accepts.
154    pub check: Option<reddb_rql::schema_expression::SchemaExpression>,
155    pub compress: Option<u8>,
156    pub unique: bool,
157    pub primary_key: bool,
158    pub enum_variants: Vec<String>,
159    pub array_element: Option<String>,
160    pub decimal_precision: Option<u8>,
161}
162
163#[derive(Debug, Clone)]
164pub struct CollectionContract {
165    pub name: String,
166    pub declared_model: crate::catalog::CollectionModel,
167    pub schema_mode: crate::catalog::SchemaMode,
168    pub origin: ContractOrigin,
169    pub version: u32,
170    pub created_at_unix_ms: u128,
171    pub updated_at_unix_ms: u128,
172    pub default_ttl_ms: Option<u64>,
173    pub vector_dimension: Option<usize>,
174    pub vector_metric: Option<crate::storage::engine::distance::DistanceMetric>,
175    pub context_index_fields: Vec<String>,
176    pub declared_columns: Vec<DeclaredColumnContract>,
177    pub table_def: Option<reddb_types::TableDef>,
178    /// Enabled by `CREATE TABLE ... WITH timestamps = true`. When true,
179    /// the runtime auto-populates two user-visible columns
180    /// `created_at` + `updated_at` (BIGINT unix-ms) sourced from the
181    /// `UnifiedEntity::created_at/updated_at` fields. `created_at` is
182    /// immutable after insert; `updated_at` is bumped on every mutation.
183    pub timestamps_enabled: bool,
184    /// Enabled by `CREATE TABLE ... WITH context_index = true` (or by
185    /// naming specific `context_index_fields`). When true, every INSERT
186    /// tokenises the row's text fields and populates the global context
187    /// index that backs `SEARCH CONTEXT` / `SEARCH SIMILAR TEXT` / `ASK`
188    /// (RAG). When false (default), inserts skip the tokenisation +
189    /// 3-way RwLock write storm entirely — ~800 ns faster per insert,
190    /// and SEARCH returns empty for this collection.
191    ///
192    /// Opt-in by design: pure OLTP tables (accounts, orders, events)
193    /// pay zero indexing tax; search-oriented tables (articles, docs)
194    /// flip the switch at CREATE time.
195    pub context_index_enabled: bool,
196    /// Metrics collections are backed by time-series storage but carry a
197    /// metrics-specific raw sample retention contract.
198    pub metrics_raw_retention_ms: Option<u64>,
199    /// Metrics rollup tiers declared by `CREATE METRICS ... DOWNSAMPLE`.
200    pub metrics_rollup_policies: Vec<String>,
201    /// Metrics tenant identity source. Defaults to current tenant context and
202    /// can be declared as a stable identity path for future ingestion slices.
203    pub metrics_tenant_identity: Option<String>,
204    /// Metrics namespace identity. v0 starts with a default namespace so
205    /// series identity is namespace-aware before Prometheus ingestion exists.
206    pub metrics_namespace: Option<String>,
207    /// Enabled by `CREATE TABLE ... APPEND ONLY` or `WITH
208    /// (append_only = true)`. When true, the runtime rejects
209    /// `UPDATE` and `DELETE` against this collection at parse time
210    /// with a clear error — the operator's immutability intent
211    /// becomes a first-class catalog fact rather than an RLS-shaped
212    /// approximation. Default `false` so legacy DDL keeps its
213    /// mutable semantics.
214    pub append_only: bool,
215    /// Declarative subscriptions created by `WITH EVENTS`. This is
216    /// metadata only in #291; event emission is wired by the outbox slice.
217    pub subscriptions: Vec<crate::catalog::SubscriptionDescriptor>,
218    /// Analytics views declared by `CREATE GRAPH ... WITH ANALYTICS (...)`.
219    /// Persisted as part of the contract so each enabled `<graph>.<output>`
220    /// virtual view survives restarts and crash recovery (issue #800).
221    pub analytics_config: Vec<crate::catalog::AnalyticsViewDescriptor>,
222    /// `CREATE TIMESERIES ... WITH SESSION_KEY <col>` — the column the
223    /// `SESSIONIZE` operator partitions by when no key is supplied at
224    /// query-time. `None` for non-timeseries collections and for
225    /// timeseries created without the clause. Issue #576 slice 1.
226    pub session_key: Option<String>,
227    /// `CREATE TIMESERIES ... SESSION_GAP <duration>` — the default
228    /// inactivity gap (milliseconds) the `SESSIONIZE` operator uses to
229    /// close a session when no gap is supplied at query-time. `None`
230    /// for non-timeseries collections and for timeseries created
231    /// without the clause. Issue #576 slice 1.
232    pub session_gap_ms: Option<u64>,
233    /// `ALTER COLLECTION ... SET RETENTION <duration>` — declarative
234    /// retention policy in milliseconds. `None` means retention is
235    /// not enforced. Reads filter out rows older than `now -
236    /// retention_duration_ms` by the collection's timestamp column.
237    /// Issue #580 — DeclarativeRetention slice 1.
238    pub retention_duration_ms: Option<u64>,
239    /// Analytical-storage seam (PRD #850, Phase 1). When present and
240    /// `columnar = true`, sealing this collection's hypertable chunks
241    /// routes to the columnar `ColumnBlock` writer; `None` (the default)
242    /// keeps the row engine. Decodes to `None` on sidecars written before
243    /// the feature.
244    pub analytical_storage: Option<crate::catalog::AnalyticalStorageConfig>,
245    /// Per-collection AI policy declared by `WITH (EMBED|MODERATE|VISION
246    /// (...))` (PRD #1267, issue #1271). `None` when no AI clause is
247    /// present, and decodes to `None` on sidecars written before the
248    /// feature (versioned/migrated with the schema). Validated against
249    /// the provider capability matrix (#1269) at DDL execution time.
250    pub ai_policy: Option<crate::catalog::AiPolicy>,
251}
252
253/// Canonical artifact lifecycle states.
254///
255/// State machine transitions:
256/// ```text
257///   Declared ──► Building ──► Ready ──► Stale ──► RequiresRebuild
258///       │            │          │                       │
259///       │            ▼          ▼                       │
260///       │         Failed    Disabled                    │
261///       │            │                                  │
262///       └────────────┴──────────────────────────────────┘
263///                    (rebuild restarts from Building)
264/// ```
265#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
266pub enum ArtifactState {
267    /// Index declared but never materialized.
268    Declared,
269    /// Artifact is being built or rebuilt.
270    Building,
271    /// Artifact is materialized and queryable.
272    Ready,
273    /// Artifact is explicitly disabled by the operator.
274    Disabled,
275    /// Underlying data changed; artifact is out of date.
276    Stale,
277    /// Build or warmup failed; manual intervention may be needed.
278    Failed,
279    /// Artifact must be rebuilt before it can serve reads.
280    RequiresRebuild,
281}
282
283impl ArtifactState {
284    /// Parse from the legacy string representation stored in physical metadata.
285    pub fn from_build_state(s: &str, enabled: bool) -> Self {
286        if !enabled {
287            return Self::Disabled;
288        }
289        match s {
290            "ready" => Self::Ready,
291            "building" | "catalog-derived" | "metadata-only" | "artifact-published"
292            | "registry-loaded" => Self::Building,
293            "stale" => Self::Stale,
294            "failed" => Self::Failed,
295            "requires_rebuild" | "requires-rebuild" => Self::RequiresRebuild,
296            _ => Self::Declared,
297        }
298    }
299
300    /// Canonical string representation for storage and API surfaces.
301    pub fn as_str(&self) -> &'static str {
302        match self {
303            Self::Declared => "declared",
304            Self::Building => "building",
305            Self::Ready => "ready",
306            Self::Disabled => "disabled",
307            Self::Stale => "stale",
308            Self::Failed => "failed",
309            Self::RequiresRebuild => "requires_rebuild",
310        }
311    }
312
313    /// Whether this artifact is safe for query reads.
314    pub fn is_queryable(&self) -> bool {
315        matches!(self, Self::Ready)
316    }
317
318    /// Whether a rebuild operation is valid from this state.
319    pub fn can_rebuild(&self) -> bool {
320        matches!(
321            self,
322            Self::Declared | Self::Stale | Self::Failed | Self::RequiresRebuild
323        )
324    }
325
326    /// Whether this state indicates the artifact needs attention.
327    pub fn needs_attention(&self) -> bool {
328        matches!(self, Self::Failed | Self::RequiresRebuild | Self::Stale)
329    }
330}
331
332impl std::fmt::Display for ArtifactState {
333    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
334        f.write_str(self.as_str())
335    }
336}
337
338#[derive(Debug, Clone)]
339pub struct PhysicalIndexState {
340    pub name: String,
341    pub kind: IndexKind,
342    pub collection: Option<String>,
343    pub enabled: bool,
344    pub entries: usize,
345    pub estimated_memory_bytes: u64,
346    pub last_refresh_ms: Option<u128>,
347    pub backend: String,
348    pub artifact_kind: Option<String>,
349    pub artifact_root_page: Option<u32>,
350    pub artifact_checksum: Option<u64>,
351    pub build_state: String,
352}
353
354impl PhysicalIndexState {
355    /// Canonical artifact lifecycle state derived from physical state.
356    pub fn artifact_state(&self) -> ArtifactState {
357        ArtifactState::from_build_state(&self.build_state, self.enabled)
358    }
359}
360
361/// A single persisted hypertable chunk. Mirror of
362/// `storage::timeseries::ChunkMeta`, flattened for the metadata
363/// sidecar so the registry's routing spine survives a restart
364/// (issue #866). `start_ns` plus the owning hypertable name is the
365/// chunk's stable identity.
366#[derive(Debug, Clone)]
367pub struct PhysicalHypertableChunk {
368    pub start_ns: u64,
369    pub end_ns_exclusive: u64,
370    pub row_count: u64,
371    pub min_ts_ns: u64,
372    pub max_ts_ns: u64,
373    pub sealed: bool,
374    pub ttl_override_ns: Option<u64>,
375    /// Columnar-vs-row migration discriminant — mirror of
376    /// `ChunkMeta.columnar_page` (PRD #850, Phase 1). `Some` → the chunk's
377    /// `RDCC` `ColumnBlock` location; `None` → legacy row-stored. Absent on
378    /// sidecars written before the feature, decoding to `None`.
379    pub columnar_page: Option<crate::storage::engine::PageLocation>,
380    /// Columnar blocks are derived, rebuildable artifacts (ADR 0069), not
381    /// source-of-truth row data. Backup/restore may skip or rebuild them.
382    pub columnar_derived: bool,
383}
384
385/// A persisted hypertable spec plus all of its chunks. Stored in the
386/// physical metadata sidecar alongside collection contracts so chunk
387/// bounds / routing / TTL are recovered identically after a restart
388/// — the same durability path the rest of the catalog already uses,
389/// not a parallel one (issue #866).
390#[derive(Debug, Clone)]
391pub struct PhysicalHypertable {
392    pub name: String,
393    pub time_column: String,
394    pub chunk_interval_ns: u64,
395    pub default_ttl_ns: Option<u64>,
396    pub chunks: Vec<PhysicalHypertableChunk>,
397}
398
399#[derive(Debug, Clone)]
400pub struct PhysicalMetadataFile {
401    pub protocol_version: String,
402    pub generated_at_unix_ms: u128,
403    pub last_loaded_from: Option<String>,
404    pub last_healed_at_unix_ms: Option<u128>,
405    pub manifest: SchemaManifest,
406    pub catalog: CatalogSnapshot,
407    pub manifest_events: Vec<ManifestEvent>,
408    pub indexes: Vec<PhysicalIndexState>,
409    pub graph_projections: Vec<PhysicalGraphProjection>,
410    pub analytics_jobs: Vec<PhysicalAnalyticsJob>,
411    pub tree_definitions: Vec<PhysicalTreeDefinition>,
412    pub collection_ttl_defaults_ms: BTreeMap<String, u64>,
413    pub collection_contracts: Vec<CollectionContract>,
414    /// Persisted hypertable chunk spine (issue #866). Empty on legacy
415    /// sidecars written before the feature and for non-hypertable
416    /// databases.
417    pub hypertables: Vec<PhysicalHypertable>,
418    pub exports: Vec<ExportDescriptor>,
419    pub superblock: SuperblockHeader,
420    pub snapshots: Vec<SnapshotDescriptor>,
421}
422
423mod helpers;
424mod json_codec;
425mod metadata_file;
426pub mod shm;
427
428pub(crate) use self::json_codec::{
429    deserialize_collection_contracts, serialize_collection_contracts,
430};
431
432pub use self::shm::{
433    provision_shm, read_shm_header, set_shm_provisioning_enabled, shm_path_for,
434    shm_provisioning_enabled, ShmHandle, ShmHeader, ShmProvisionState, SHM_FILE_SIZE,
435    SHM_HEADER_SIZE, SHM_MAGIC, SHM_VERSION,
436};
437
438use self::helpers::*;
439use self::json_codec::*;
reddb_server/physical.rs

reddb_server/
physical.rs
