reddb_server/
physical.rs

1//! Physical storage design primitives for RedDB's deterministic on-disk layout.
2
3use std::collections::BTreeMap;
4use std::fs;
5use std::io;
6use std::path::{Path, PathBuf};
7use std::time::{SystemTime, UNIX_EPOCH};
8
9use crate::api::{CatalogSnapshot, CollectionStats, RedDBOptions, SchemaManifest, StorageMode};
10use crate::index::IndexKind;
11use crate::serde_json::{Map, Value as JsonValue};
12
13pub const DEFAULT_GRID_BLOCK_SIZE: usize = 512 * 1024;
14pub const DEFAULT_PAGE_SIZE: usize = 4096;
15pub use reddb_file::layout::PHYSICAL_METADATA_BINARY_EXTENSION;
16pub use reddb_file::{
17    fold_dwb_into_wal_enabled, meta_json_sidecar_enabled, seqn_journal_enabled,
18    seqn_journal_retention, set_fold_dwb_into_wal_enabled, set_meta_json_sidecar_enabled,
19    set_seqn_journal_enabled, set_seqn_journal_retention, BlockReference, ExportDescriptor,
20    ManifestEvent, ManifestEventKind, ManifestPointers, PhysicalAnalyticsJob,
21    PhysicalGraphProjection, PhysicalTreeDefinition, SnapshotDescriptor, SuperblockHeader,
22    DEFAULT_METADATA_JOURNAL_RETENTION, DEFAULT_SUPERBLOCK_COPIES,
23    OPT_IN_METADATA_JOURNAL_RETENTION, PHYSICAL_METADATA_PROTOCOL_VERSION,
24};
25pub const DEFAULT_MANIFEST_EVENT_HISTORY: usize = 256;
26
27#[derive(Debug, Clone, Copy, PartialEq, Eq)]
28pub enum PhysicalMetadataSource {
29    Binary,
30    BinaryJournal,
31    Json,
32}
33
34impl PhysicalMetadataSource {
35    pub fn as_str(self) -> &'static str {
36        match self {
37            Self::Binary => "binary",
38            Self::BinaryJournal => "binary_journal",
39            Self::Json => "json",
40        }
41    }
42}
43#[derive(Debug, Clone, Copy, PartialEq, Eq)]
44pub enum CompactionPolicy {
45    Incremental,
46    Manual,
47}
48
49#[derive(Debug, Clone)]
50pub struct WalPolicy {
51    pub auto_checkpoint_pages: u32,
52    pub fsync_on_commit: bool,
53    pub ring_buffer_bytes: u64,
54}
55
56impl Default for WalPolicy {
57    fn default() -> Self {
58        Self {
59            auto_checkpoint_pages: 1000,
60            fsync_on_commit: true,
61            ring_buffer_bytes: 64 * 1024 * 1024,
62        }
63    }
64}
65
66#[derive(Debug, Clone)]
67pub struct GridLayout {
68    pub block_size: usize,
69    pub page_size: usize,
70    pub superblock_copies: u8,
71}
72
73impl Default for GridLayout {
74    fn default() -> Self {
75        Self {
76            block_size: DEFAULT_GRID_BLOCK_SIZE,
77            page_size: DEFAULT_PAGE_SIZE,
78            superblock_copies: DEFAULT_SUPERBLOCK_COPIES,
79        }
80    }
81}
82
83#[derive(Debug, Clone)]
84pub struct PhysicalLayout {
85    pub mode: StorageMode,
86    pub grid: GridLayout,
87    pub wal: WalPolicy,
88    pub compaction: CompactionPolicy,
89}
90
91impl PhysicalLayout {
92    pub fn from_options(options: &RedDBOptions) -> Self {
93        Self {
94            mode: options.mode,
95            grid: GridLayout::default(),
96            wal: WalPolicy {
97                auto_checkpoint_pages: options.auto_checkpoint_pages,
98                ..WalPolicy::default()
99            },
100            compaction: CompactionPolicy::Incremental,
101        }
102    }
103
104    pub fn is_persistent(&self) -> bool {
105        self.mode == StorageMode::Persistent
106    }
107}
108
109pub(crate) use reddb_engine::catalog::{
110    is_collection_contract_decode_error, CollectionContractDecodeError,
111};
112pub use reddb_engine::catalog::{CollectionContract, ContractOrigin, DeclaredColumnContract};
113
114/// Canonical artifact lifecycle states.
115///
116/// State machine transitions:
117/// ```text
118///   Declared ──► Building ──► Ready ──► Stale ──► RequiresRebuild
119///       │            │          │                       │
120///       │            ▼          ▼                       │
121///       │         Failed    Disabled                    │
122///       │            │                                  │
123///       └────────────┴──────────────────────────────────┘
124///                    (rebuild restarts from Building)
125/// ```
126#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
127pub enum ArtifactState {
128    /// Index declared but never materialized.
129    Declared,
130    /// Artifact is being built or rebuilt.
131    Building,
132    /// Artifact is materialized and queryable.
133    Ready,
134    /// Artifact is explicitly disabled by the operator.
135    Disabled,
136    /// Underlying data changed; artifact is out of date.
137    Stale,
138    /// Build or warmup failed; manual intervention may be needed.
139    Failed,
140    /// Artifact must be rebuilt before it can serve reads.
141    RequiresRebuild,
142}
143
144impl ArtifactState {
145    /// Parse from the legacy string representation stored in physical metadata.
146    pub fn from_build_state(s: &str, enabled: bool) -> Self {
147        if !enabled {
148            return Self::Disabled;
149        }
150        match s {
151            // Both states are emitted only after the operational registry has
152            // found and loaded the physical index. Treating them as Building
153            // after reopen made the planner ignore every recovered B-tree.
154            "ready" | "catalog-derived" | "registry-loaded" => Self::Ready,
155            "building" | "metadata-only" | "artifact-published" => Self::Building,
156            "stale" => Self::Stale,
157            "failed" => Self::Failed,
158            "requires_rebuild" | "requires-rebuild" => Self::RequiresRebuild,
159            _ => Self::Declared,
160        }
161    }
162
163    /// Canonical string representation for storage and API surfaces.
164    pub fn as_str(&self) -> &'static str {
165        match self {
166            Self::Declared => "declared",
167            Self::Building => "building",
168            Self::Ready => "ready",
169            Self::Disabled => "disabled",
170            Self::Stale => "stale",
171            Self::Failed => "failed",
172            Self::RequiresRebuild => "requires_rebuild",
173        }
174    }
175
176    /// Whether this artifact is safe for query reads.
177    pub fn is_queryable(&self) -> bool {
178        matches!(self, Self::Ready)
179    }
180
181    /// Whether a rebuild operation is valid from this state.
182    pub fn can_rebuild(&self) -> bool {
183        matches!(
184            self,
185            Self::Declared | Self::Stale | Self::Failed | Self::RequiresRebuild
186        )
187    }
188
189    /// Whether this state indicates the artifact needs attention.
190    pub fn needs_attention(&self) -> bool {
191        matches!(self, Self::Failed | Self::RequiresRebuild | Self::Stale)
192    }
193}
194
195impl std::fmt::Display for ArtifactState {
196    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
197        f.write_str(self.as_str())
198    }
199}
200
201#[derive(Debug, Clone)]
202pub struct PhysicalIndexState {
203    pub name: String,
204    pub kind: IndexKind,
205    pub collection: Option<String>,
206    pub enabled: bool,
207    pub entries: usize,
208    pub estimated_memory_bytes: u64,
209    pub last_refresh_ms: Option<u128>,
210    pub backend: String,
211    pub artifact_kind: Option<String>,
212    pub artifact_root_page: Option<u32>,
213    pub artifact_checksum: Option<u64>,
214    pub build_state: String,
215}
216
217impl PhysicalIndexState {
218    /// Canonical artifact lifecycle state derived from physical state.
219    pub fn artifact_state(&self) -> ArtifactState {
220        ArtifactState::from_build_state(&self.build_state, self.enabled)
221    }
222}
223
224/// A single persisted hypertable chunk. Mirror of
225/// `storage::timeseries::ChunkMeta`, flattened for the metadata
226/// sidecar so the registry's routing spine survives a restart
227/// (issue #866). `start_ns` plus the owning hypertable name is the
228/// chunk's stable identity.
229#[derive(Debug, Clone)]
230pub struct PhysicalHypertableChunk {
231    pub start_ns: u64,
232    pub end_ns_exclusive: u64,
233    pub row_count: u64,
234    pub min_ts_ns: u64,
235    pub max_ts_ns: u64,
236    pub sealed: bool,
237    pub ttl_override_ns: Option<u64>,
238    /// Columnar-vs-row migration discriminant — mirror of
239    /// `ChunkMeta.columnar_page` (PRD #850, Phase 1). `Some` → the chunk's
240    /// `RDCC` `ColumnBlock` location; `None` → legacy row-stored. Absent on
241    /// sidecars written before the feature, decoding to `None`.
242    pub columnar_page: Option<crate::storage::engine::PageLocation>,
243    /// Columnar blocks are derived, rebuildable artifacts (ADR 0069), not
244    /// source-of-truth row data. Backup/restore may skip or rebuild them.
245    pub columnar_derived: bool,
246}
247
248/// A persisted hypertable spec plus all of its chunks. Stored in the
249/// physical metadata sidecar alongside collection contracts so chunk
250/// bounds / routing / TTL are recovered identically after a restart
251/// — the same durability path the rest of the catalog already uses,
252/// not a parallel one (issue #866).
253#[derive(Debug, Clone)]
254pub struct PhysicalHypertable {
255    pub name: String,
256    pub time_column: String,
257    pub chunk_interval_ns: u64,
258    pub default_ttl_ns: Option<u64>,
259    pub chunks: Vec<PhysicalHypertableChunk>,
260}
261
262#[derive(Debug, Clone)]
263pub struct PhysicalMetadataFile {
264    pub protocol_version: String,
265    pub generated_at_unix_ms: u128,
266    pub last_loaded_from: Option<String>,
267    pub last_healed_at_unix_ms: Option<u128>,
268    pub manifest: SchemaManifest,
269    pub catalog: CatalogSnapshot,
270    pub manifest_events: Vec<ManifestEvent>,
271    pub indexes: Vec<PhysicalIndexState>,
272    pub graph_projections: Vec<PhysicalGraphProjection>,
273    pub analytics_jobs: Vec<PhysicalAnalyticsJob>,
274    pub tree_definitions: Vec<PhysicalTreeDefinition>,
275    pub collection_ttl_defaults_ms: BTreeMap<String, u64>,
276    pub collection_contracts: Vec<CollectionContract>,
277    /// Persisted hypertable chunk spine (issue #866). Empty on legacy
278    /// sidecars written before the feature and for non-hypertable
279    /// databases.
280    pub hypertables: Vec<PhysicalHypertable>,
281    pub exports: Vec<ExportDescriptor>,
282    pub superblock: SuperblockHeader,
283    pub snapshots: Vec<SnapshotDescriptor>,
284}
285
286mod helpers;
287mod json_codec;
288mod metadata_file;
289pub mod shm;
290
291pub(crate) use reddb_engine::catalog::codec::{
292    deserialize_collection_contracts, serialize_collection_contracts,
293};
294
295pub use self::shm::{
296    provision_shm, read_shm_header, set_shm_provisioning_enabled, shm_path_for,
297    shm_provisioning_enabled, ShmHandle, ShmHeader, ShmProvisionState, SHM_FILE_SIZE,
298    SHM_HEADER_SIZE, SHM_MAGIC, SHM_VERSION,
299};
300
301use self::helpers::*;
302use self::json_codec::*;
reddb_server/physical.rs

reddb_server/
physical.rs
