reddb_server/
physical.rs

1//! Physical storage design primitives for RedDB's deterministic on-disk layout.
2
3use std::collections::BTreeMap;
4use std::fs;
5use std::io;
6use std::path::{Path, PathBuf};
7use std::time::{SystemTime, UNIX_EPOCH};
8
9use crate::api::{CatalogSnapshot, CollectionStats, RedDBOptions, SchemaManifest, StorageMode};
10use crate::index::IndexKind;
11use crate::serde_json::{Map, Value as JsonValue};
12
13pub const DEFAULT_GRID_BLOCK_SIZE: usize = 512 * 1024;
14pub const DEFAULT_PAGE_SIZE: usize = 4096;
15pub use reddb_file::layout::PHYSICAL_METADATA_BINARY_EXTENSION;
16pub use reddb_file::{
17    fold_dwb_into_wal_enabled, meta_json_sidecar_enabled, seqn_journal_enabled,
18    seqn_journal_retention, set_fold_dwb_into_wal_enabled, set_meta_json_sidecar_enabled,
19    set_seqn_journal_enabled, set_seqn_journal_retention, BlockReference, ExportDescriptor,
20    ManifestEvent, ManifestEventKind, ManifestPointers, PhysicalAnalyticsJob,
21    PhysicalGraphProjection, PhysicalTreeDefinition, SnapshotDescriptor, SuperblockHeader,
22    DEFAULT_METADATA_JOURNAL_RETENTION, DEFAULT_SUPERBLOCK_COPIES,
23    OPT_IN_METADATA_JOURNAL_RETENTION, PHYSICAL_METADATA_PROTOCOL_VERSION,
24};
25pub const DEFAULT_MANIFEST_EVENT_HISTORY: usize = 256;
26
27#[derive(Debug, Clone, Copy, PartialEq, Eq)]
28pub enum PhysicalMetadataSource {
29    Binary,
30    BinaryJournal,
31    Json,
32}
33
34impl PhysicalMetadataSource {
35    pub fn as_str(self) -> &'static str {
36        match self {
37            Self::Binary => "binary",
38            Self::BinaryJournal => "binary_journal",
39            Self::Json => "json",
40        }
41    }
42}
43#[derive(Debug, Clone, Copy, PartialEq, Eq)]
44pub enum CompactionPolicy {
45    Incremental,
46    Manual,
47}
48
49#[derive(Debug, Clone)]
50pub struct WalPolicy {
51    pub auto_checkpoint_pages: u32,
52    pub fsync_on_commit: bool,
53    pub ring_buffer_bytes: u64,
54}
55
56impl Default for WalPolicy {
57    fn default() -> Self {
58        Self {
59            auto_checkpoint_pages: 1000,
60            fsync_on_commit: true,
61            ring_buffer_bytes: 64 * 1024 * 1024,
62        }
63    }
64}
65
66#[derive(Debug, Clone)]
67pub struct GridLayout {
68    pub block_size: usize,
69    pub page_size: usize,
70    pub superblock_copies: u8,
71}
72
73impl Default for GridLayout {
74    fn default() -> Self {
75        Self {
76            block_size: DEFAULT_GRID_BLOCK_SIZE,
77            page_size: DEFAULT_PAGE_SIZE,
78            superblock_copies: DEFAULT_SUPERBLOCK_COPIES,
79        }
80    }
81}
82
83#[derive(Debug, Clone)]
84pub struct PhysicalLayout {
85    pub mode: StorageMode,
86    pub grid: GridLayout,
87    pub wal: WalPolicy,
88    pub compaction: CompactionPolicy,
89}
90
91impl PhysicalLayout {
92    pub fn from_options(options: &RedDBOptions) -> Self {
93        Self {
94            mode: options.mode,
95            grid: GridLayout::default(),
96            wal: WalPolicy {
97                auto_checkpoint_pages: options.auto_checkpoint_pages,
98                ..WalPolicy::default()
99            },
100            compaction: CompactionPolicy::Incremental,
101        }
102    }
103
104    pub fn is_persistent(&self) -> bool {
105        self.mode == StorageMode::Persistent
106    }
107}
108
109#[derive(Debug, Clone, Copy, PartialEq, Eq)]
110pub enum ContractOrigin {
111    Explicit,
112    Implicit,
113    Migrated,
114}
115
116impl ContractOrigin {
117    pub fn as_str(self) -> &'static str {
118        match self {
119            Self::Explicit => "explicit",
120            Self::Implicit => "implicit",
121            Self::Migrated => "migrated",
122        }
123    }
124}
125
126#[derive(Debug, Clone)]
127pub struct DeclaredColumnContract {
128    pub name: String,
129    pub data_type: String,
130    pub sql_type: Option<crate::storage::schema::SqlTypeName>,
131    pub not_null: bool,
132    pub default: Option<String>,
133    pub compress: Option<u8>,
134    pub unique: bool,
135    pub primary_key: bool,
136    pub enum_variants: Vec<String>,
137    pub array_element: Option<String>,
138    pub decimal_precision: Option<u8>,
139}
140
141#[derive(Debug, Clone)]
142pub struct CollectionContract {
143    pub name: String,
144    pub declared_model: crate::catalog::CollectionModel,
145    pub schema_mode: crate::catalog::SchemaMode,
146    pub origin: ContractOrigin,
147    pub version: u32,
148    pub created_at_unix_ms: u128,
149    pub updated_at_unix_ms: u128,
150    pub default_ttl_ms: Option<u64>,
151    pub vector_dimension: Option<usize>,
152    pub vector_metric: Option<crate::storage::engine::distance::DistanceMetric>,
153    pub context_index_fields: Vec<String>,
154    pub declared_columns: Vec<DeclaredColumnContract>,
155    pub table_def: Option<crate::storage::schema::TableDef>,
156    /// Enabled by `CREATE TABLE ... WITH timestamps = true`. When true,
157    /// the runtime auto-populates two user-visible columns
158    /// `created_at` + `updated_at` (BIGINT unix-ms) sourced from the
159    /// `UnifiedEntity::created_at/updated_at` fields. `created_at` is
160    /// immutable after insert; `updated_at` is bumped on every mutation.
161    pub timestamps_enabled: bool,
162    /// Enabled by `CREATE TABLE ... WITH context_index = true` (or by
163    /// naming specific `context_index_fields`). When true, every INSERT
164    /// tokenises the row's text fields and populates the global context
165    /// index that backs `SEARCH CONTEXT` / `SEARCH SIMILAR TEXT` / `ASK`
166    /// (RAG). When false (default), inserts skip the tokenisation +
167    /// 3-way RwLock write storm entirely — ~800 ns faster per insert,
168    /// and SEARCH returns empty for this collection.
169    ///
170    /// Opt-in by design: pure OLTP tables (accounts, orders, events)
171    /// pay zero indexing tax; search-oriented tables (articles, docs)
172    /// flip the switch at CREATE time.
173    pub context_index_enabled: bool,
174    /// Metrics collections are backed by time-series storage but carry a
175    /// metrics-specific raw sample retention contract.
176    pub metrics_raw_retention_ms: Option<u64>,
177    /// Metrics rollup tiers declared by `CREATE METRICS ... DOWNSAMPLE`.
178    pub metrics_rollup_policies: Vec<String>,
179    /// Metrics tenant identity source. Defaults to current tenant context and
180    /// can be declared as a stable identity path for future ingestion slices.
181    pub metrics_tenant_identity: Option<String>,
182    /// Metrics namespace identity. v0 starts with a default namespace so
183    /// series identity is namespace-aware before Prometheus ingestion exists.
184    pub metrics_namespace: Option<String>,
185    /// Enabled by `CREATE TABLE ... APPEND ONLY` or `WITH
186    /// (append_only = true)`. When true, the runtime rejects
187    /// `UPDATE` and `DELETE` against this collection at parse time
188    /// with a clear error — the operator's immutability intent
189    /// becomes a first-class catalog fact rather than an RLS-shaped
190    /// approximation. Default `false` so legacy DDL keeps its
191    /// mutable semantics.
192    pub append_only: bool,
193    /// Declarative subscriptions created by `WITH EVENTS`. This is
194    /// metadata only in #291; event emission is wired by the outbox slice.
195    pub subscriptions: Vec<crate::catalog::SubscriptionDescriptor>,
196    /// Analytics views declared by `CREATE GRAPH ... WITH ANALYTICS (...)`.
197    /// Persisted as part of the contract so each enabled `<graph>.<output>`
198    /// virtual view survives restarts and crash recovery (issue #800).
199    pub analytics_config: Vec<crate::catalog::AnalyticsViewDescriptor>,
200    /// `CREATE TIMESERIES ... WITH SESSION_KEY <col>` — the column the
201    /// `SESSIONIZE` operator partitions by when no key is supplied at
202    /// query-time. `None` for non-timeseries collections and for
203    /// timeseries created without the clause. Issue #576 slice 1.
204    pub session_key: Option<String>,
205    /// `CREATE TIMESERIES ... SESSION_GAP <duration>` — the default
206    /// inactivity gap (milliseconds) the `SESSIONIZE` operator uses to
207    /// close a session when no gap is supplied at query-time. `None`
208    /// for non-timeseries collections and for timeseries created
209    /// without the clause. Issue #576 slice 1.
210    pub session_gap_ms: Option<u64>,
211    /// `ALTER COLLECTION ... SET RETENTION <duration>` — declarative
212    /// retention policy in milliseconds. `None` means retention is
213    /// not enforced. Reads filter out rows older than `now -
214    /// retention_duration_ms` by the collection's timestamp column.
215    /// Issue #580 — DeclarativeRetention slice 1.
216    pub retention_duration_ms: Option<u64>,
217    /// Analytical-storage seam (PRD #850, Phase 1). When present and
218    /// `columnar = true`, sealing this collection's hypertable chunks
219    /// routes to the columnar `ColumnBlock` writer; `None` (the default)
220    /// keeps the row engine. Decodes to `None` on sidecars written before
221    /// the feature.
222    pub analytical_storage: Option<crate::catalog::AnalyticalStorageConfig>,
223    /// Per-collection AI policy declared by `WITH (EMBED|MODERATE|VISION
224    /// (...))` (PRD #1267, issue #1271). `None` when no AI clause is
225    /// present, and decodes to `None` on sidecars written before the
226    /// feature (versioned/migrated with the schema). Validated against
227    /// the provider capability matrix (#1269) at DDL execution time.
228    pub ai_policy: Option<crate::catalog::AiPolicy>,
229}
230
231/// Canonical artifact lifecycle states.
232///
233/// State machine transitions:
234/// ```text
235///   Declared ──► Building ──► Ready ──► Stale ──► RequiresRebuild
236///       │            │          │                       │
237///       │            ▼          ▼                       │
238///       │         Failed    Disabled                    │
239///       │            │                                  │
240///       └────────────┴──────────────────────────────────┘
241///                    (rebuild restarts from Building)
242/// ```
243#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
244pub enum ArtifactState {
245    /// Index declared but never materialized.
246    Declared,
247    /// Artifact is being built or rebuilt.
248    Building,
249    /// Artifact is materialized and queryable.
250    Ready,
251    /// Artifact is explicitly disabled by the operator.
252    Disabled,
253    /// Underlying data changed; artifact is out of date.
254    Stale,
255    /// Build or warmup failed; manual intervention may be needed.
256    Failed,
257    /// Artifact must be rebuilt before it can serve reads.
258    RequiresRebuild,
259}
260
261impl ArtifactState {
262    /// Parse from the legacy string representation stored in physical metadata.
263    pub fn from_build_state(s: &str, enabled: bool) -> Self {
264        if !enabled {
265            return Self::Disabled;
266        }
267        match s {
268            "ready" => Self::Ready,
269            "building" | "catalog-derived" | "metadata-only" | "artifact-published"
270            | "registry-loaded" => Self::Building,
271            "stale" => Self::Stale,
272            "failed" => Self::Failed,
273            "requires_rebuild" | "requires-rebuild" => Self::RequiresRebuild,
274            _ => Self::Declared,
275        }
276    }
277
278    /// Canonical string representation for storage and API surfaces.
279    pub fn as_str(&self) -> &'static str {
280        match self {
281            Self::Declared => "declared",
282            Self::Building => "building",
283            Self::Ready => "ready",
284            Self::Disabled => "disabled",
285            Self::Stale => "stale",
286            Self::Failed => "failed",
287            Self::RequiresRebuild => "requires_rebuild",
288        }
289    }
290
291    /// Whether this artifact is safe for query reads.
292    pub fn is_queryable(&self) -> bool {
293        matches!(self, Self::Ready)
294    }
295
296    /// Whether a rebuild operation is valid from this state.
297    pub fn can_rebuild(&self) -> bool {
298        matches!(
299            self,
300            Self::Declared | Self::Stale | Self::Failed | Self::RequiresRebuild
301        )
302    }
303
304    /// Whether this state indicates the artifact needs attention.
305    pub fn needs_attention(&self) -> bool {
306        matches!(self, Self::Failed | Self::RequiresRebuild | Self::Stale)
307    }
308}
309
310impl std::fmt::Display for ArtifactState {
311    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
312        f.write_str(self.as_str())
313    }
314}
315
316#[derive(Debug, Clone)]
317pub struct PhysicalIndexState {
318    pub name: String,
319    pub kind: IndexKind,
320    pub collection: Option<String>,
321    pub enabled: bool,
322    pub entries: usize,
323    pub estimated_memory_bytes: u64,
324    pub last_refresh_ms: Option<u128>,
325    pub backend: String,
326    pub artifact_kind: Option<String>,
327    pub artifact_root_page: Option<u32>,
328    pub artifact_checksum: Option<u64>,
329    pub build_state: String,
330}
331
332impl PhysicalIndexState {
333    /// Canonical artifact lifecycle state derived from physical state.
334    pub fn artifact_state(&self) -> ArtifactState {
335        ArtifactState::from_build_state(&self.build_state, self.enabled)
336    }
337}
338
339/// A single persisted hypertable chunk. Mirror of
340/// `storage::timeseries::ChunkMeta`, flattened for the metadata
341/// sidecar so the registry's routing spine survives a restart
342/// (issue #866). `start_ns` plus the owning hypertable name is the
343/// chunk's stable identity.
344#[derive(Debug, Clone)]
345pub struct PhysicalHypertableChunk {
346    pub start_ns: u64,
347    pub end_ns_exclusive: u64,
348    pub row_count: u64,
349    pub min_ts_ns: u64,
350    pub max_ts_ns: u64,
351    pub sealed: bool,
352    pub ttl_override_ns: Option<u64>,
353    /// Columnar-vs-row migration discriminant — mirror of
354    /// `ChunkMeta.columnar_page` (PRD #850, Phase 1). `Some` → the chunk's
355    /// `RDCC` `ColumnBlock` location; `None` → legacy row-stored. Absent on
356    /// sidecars written before the feature, decoding to `None`.
357    pub columnar_page: Option<crate::storage::engine::PageLocation>,
358    /// Columnar blocks are derived, rebuildable artifacts (ADR 0069), not
359    /// source-of-truth row data. Backup/restore may skip or rebuild them.
360    pub columnar_derived: bool,
361}
362
363/// A persisted hypertable spec plus all of its chunks. Stored in the
364/// physical metadata sidecar alongside collection contracts so chunk
365/// bounds / routing / TTL are recovered identically after a restart
366/// — the same durability path the rest of the catalog already uses,
367/// not a parallel one (issue #866).
368#[derive(Debug, Clone)]
369pub struct PhysicalHypertable {
370    pub name: String,
371    pub time_column: String,
372    pub chunk_interval_ns: u64,
373    pub default_ttl_ns: Option<u64>,
374    pub chunks: Vec<PhysicalHypertableChunk>,
375}
376
377#[derive(Debug, Clone)]
378pub struct PhysicalMetadataFile {
379    pub protocol_version: String,
380    pub generated_at_unix_ms: u128,
381    pub last_loaded_from: Option<String>,
382    pub last_healed_at_unix_ms: Option<u128>,
383    pub manifest: SchemaManifest,
384    pub catalog: CatalogSnapshot,
385    pub manifest_events: Vec<ManifestEvent>,
386    pub indexes: Vec<PhysicalIndexState>,
387    pub graph_projections: Vec<PhysicalGraphProjection>,
388    pub analytics_jobs: Vec<PhysicalAnalyticsJob>,
389    pub tree_definitions: Vec<PhysicalTreeDefinition>,
390    pub collection_ttl_defaults_ms: BTreeMap<String, u64>,
391    pub collection_contracts: Vec<CollectionContract>,
392    /// Persisted hypertable chunk spine (issue #866). Empty on legacy
393    /// sidecars written before the feature and for non-hypertable
394    /// databases.
395    pub hypertables: Vec<PhysicalHypertable>,
396    pub exports: Vec<ExportDescriptor>,
397    pub superblock: SuperblockHeader,
398    pub snapshots: Vec<SnapshotDescriptor>,
399}
400
401mod helpers;
402mod json_codec;
403mod metadata_file;
404pub mod shm;
405
406pub(crate) use self::json_codec::{
407    deserialize_collection_contracts, serialize_collection_contracts,
408};
409
410pub use self::shm::{
411    provision_shm, read_shm_header, set_shm_provisioning_enabled, shm_path_for,
412    shm_provisioning_enabled, ShmHandle, ShmHeader, ShmProvisionState, SHM_FILE_SIZE,
413    SHM_HEADER_SIZE, SHM_MAGIC, SHM_VERSION,
414};
415
416use self::helpers::*;
417use self::json_codec::*;
reddb_server/physical.rs

reddb_server/
physical.rs
