mongreldb_core/
manifest.rs

1//! Manifest — the atomic pointer to the current set of sorted runs.
2//!
3//! A commit writes `_mf.tmp` then `rename(_mf.tmp, _mf)`, which is atomic on
4//! POSIX, giving crash-safe commit.
5//! For encrypted DBs the whole blob is AES-256-GCM sealed under the DB-wide
6//! meta DEK (confidential + authenticated); for plaintext DBs it carries a
7//! SHA-256 integrity tag. Either way the parent directory is fsynced after the
8//! rename so the new manifest is durable across a crash (review fix #19).
9
10use crate::encryption::DEK_LEN;
11#[cfg(feature = "encryption")]
12use crate::encryption::{decrypt_blob, encrypt_blob};
13use crate::{MongrelError, Result};
14use serde::{Deserialize, Serialize};
15use sha2::{Digest, Sha256};
16use std::fs;
17use std::io::Write;
18use std::path::{Path, PathBuf};
19
20pub const MANIFEST_MAGIC: [u8; 8] = *b"MONGRMFT";
21/// Bumped to 3 when the per-table `auto_inc_next` counter was added. NOTE: the
22/// manifest is serialized with `bincode`, which is positional and not self-
23/// describing, so `#[serde(default)]` on new fields does NOT give cross-version
24/// read compatibility — a manifest written by an older binary cannot be decoded
25/// by a newer one (and vice-versa). The project carries no on-disk
26/// compatibility guarantee pre-1.0; instead, [`read`] peeks `format_version`
27/// and deserializes a [`LegacyManifestV2`] for older files (synthesizing
28/// `auto_inc_next = 0` = unseeded) so existing database directories still open.
29pub const MANIFEST_VERSION: u16 = 3;
30pub const MANIFEST_FILENAME: &str = "_mf";
31/// 32-byte meta DEK length (matches [`crate::encryption::DEK_LEN`]).
32pub const META_DEK_LEN: usize = DEK_LEN;
33
34#[derive(Debug, Clone, Serialize, Deserialize)]
35pub struct RunRef {
36    pub run_id: u128,
37    pub level: u8,
38    pub epoch_created: u64,
39    pub row_count: u64,
40}
41
42/// A run that compaction superseded but kept on disk for snapshot retention
43/// (spec §6.4/§7.4). Its file is physically deleted by `gc()` only once
44/// `min_active_snapshot` has advanced past `retire_epoch` — i.e. no pinned
45/// reader can still need it. Persisted in the manifest so the reaper survives
46/// a reopen (otherwise the file would linger as an orphan).
47#[derive(Debug, Clone, Serialize, Deserialize)]
48pub struct RetiredRun {
49    pub run_id: u128,
50    /// The compaction epoch at which this run was superseded. Reapable once
51    /// `min_active_snapshot >= retire_epoch` (a reader pinned exactly at the
52    /// compaction epoch is served by the merged run, whose `epoch_created`
53    /// equals `retire_epoch`).
54    pub retire_epoch: u64,
55}
56
57#[derive(Debug, Clone, Serialize, Deserialize)]
58pub struct Manifest {
59    pub magic: [u8; 8],
60    pub format_version: u16,
61    pub table_id: u64,
62    pub current_epoch: u64,
63    pub next_row_id: u64,
64    pub schema_id: u64,
65    pub runs: Vec<RunRef>,
66    pub global_idx_epoch: u64,
67    /// Live (non-deleted) row count, maintained incrementally so `COUNT(*)` is
68    /// O(1) from the manifest without a scan.
69    pub live_count: u64,
70    /// Highest epoch whose data is durable in a sorted run (spec §7.1). Recovery
71    /// may skip replaying WAL records for this table whose commit epoch is
72    /// `<= flushed_epoch` (they are already represented by runs).
73    #[serde(default)]
74    pub flushed_epoch: u64,
75    /// Runs superseded by compaction but retained for snapshot retention,
76    /// pending physical deletion by `gc()` (spec §6.4). See [`RetiredRun`].
77    /// (`serde(default)` is a no-op under bincode — see [`MANIFEST_VERSION`] —
78    /// kept only so a future move to a self-describing codec would degrade
79    /// gracefully.)
80    #[serde(default)]
81    pub retiring: Vec<RetiredRun>,
82    /// Next value to hand out for the table's `AUTO_INCREMENT` primary key. `0`
83    /// means *unseeded* — the counter has never been advanced (or the manifest
84    /// predates this field), so the engine must seed it from `max(existing id)`
85    /// on first use. Always `0` for tables without an `AUTO_INCREMENT` column.
86    #[serde(default)]
87    pub auto_inc_next: i64,
88    pub checksum: [u8; 32],
89}
90
91/// The on-disk manifest shape written by mongreldb `MANIFEST_VERSION == 2`
92/// (before `auto_inc_next` was added). Kept so [`read`] can decode older
93/// database directories and migrate them forward.
94#[derive(Debug, Clone, Serialize, Deserialize)]
95pub struct LegacyManifestV2 {
96    pub magic: [u8; 8],
97    pub format_version: u16,
98    pub table_id: u64,
99    pub current_epoch: u64,
100    pub next_row_id: u64,
101    pub schema_id: u64,
102    pub runs: Vec<RunRef>,
103    pub global_idx_epoch: u64,
104    pub live_count: u64,
105    pub flushed_epoch: u64,
106    pub retiring: Vec<RetiredRun>,
107    pub checksum: [u8; 32],
108}
109
110impl Manifest {
111    pub fn new(table_id: u64, schema_id: u64) -> Self {
112        Self {
113            magic: MANIFEST_MAGIC,
114            format_version: MANIFEST_VERSION,
115            table_id,
116            current_epoch: 0,
117            next_row_id: 0,
118            schema_id,
119            runs: Vec::new(),
120            global_idx_epoch: 0,
121            live_count: 0,
122            flushed_epoch: 0,
123            retiring: Vec::new(),
124            auto_inc_next: 0,
125            checksum: [0u8; 32],
126        }
127    }
128
129    /// Migrate a legacy (pre-`auto_inc_next`) manifest forward. The identity
130    /// counter is left unseeded (`0`); the engine seeds it from `max(id)` on
131    /// first use, so upgraded tables never collide with legacy rows.
132    pub fn from_legacy(legacy: LegacyManifestV2) -> Self {
133        Manifest {
134            magic: legacy.magic,
135            format_version: MANIFEST_VERSION,
136            table_id: legacy.table_id,
137            current_epoch: legacy.current_epoch,
138            next_row_id: legacy.next_row_id,
139            schema_id: legacy.schema_id,
140            runs: legacy.runs,
141            global_idx_epoch: legacy.global_idx_epoch,
142            live_count: legacy.live_count,
143            flushed_epoch: legacy.flushed_epoch,
144            retiring: legacy.retiring,
145            auto_inc_next: 0,
146            checksum: legacy.checksum,
147        }
148    }
149
150    fn compute_checksum(&mut self) {
151        self.checksum = [0u8; 32];
152        let bytes = bincode::serialize(self).expect("manifest serializable");
153        self.checksum = Sha256::digest(&bytes).into();
154    }
155}
156
157/// First two fields of every manifest shape, used to peek `format_version`
158/// before choosing a struct to deserialize (bincode is positional, so the full
159/// `Manifest` cannot decode an older file).
160#[derive(Deserialize)]
161struct ManifestHeader {
162    magic: [u8; 8],
163    format_version: u16,
164}
165
166/// Verify the trailing SHA-256 checksum over `body` (the last 32 bytes are the
167/// stored tag; the hash is computed over the body with those bytes zeroed).
168/// Version-agnostic: works for both legacy and current manifest shapes because
169/// `checksum` is always the final field.
170fn verify_trailing_checksum(body: &[u8]) -> Result<()> {
171    if body.len() < 32 {
172        return Err(MongrelError::ChecksumMismatch {
173            expected: 0,
174            actual: 0,
175            context: "manifest (too short)".into(),
176        });
177    }
178    let split = body.len() - 32;
179    let expected: [u8; 32] = body[split..].try_into().unwrap();
180    let mut zeroed = body.to_vec();
181    zeroed[split..].fill(0);
182    let recomputed: [u8; 32] = Sha256::digest(&zeroed).into();
183    if recomputed != expected {
184        return Err(MongrelError::ChecksumMismatch {
185            expected: u64::from_be_bytes(expected[..8].try_into().unwrap()),
186            actual: u64::from_be_bytes(recomputed[..8].try_into().unwrap()),
187            context: "manifest".into(),
188        });
189    }
190    Ok(())
191}
192
193/// Atomically write the manifest to `<dir>/_mf`. When `meta_dek` is `Some` the
194/// blob is AES-256-GCM sealed (confidential + authenticated); otherwise it
195/// carries a SHA-256 integrity tag. The parent directory is fsynced after the
196/// rename (review fix #19).
197pub fn write_atomic(
198    dir: impl AsRef<Path>,
199    manifest: &mut Manifest,
200    meta_dek: Option<&[u8; META_DEK_LEN]>,
201) -> Result<()> {
202    let dir = dir.as_ref();
203    let final_path: PathBuf = dir.join(MANIFEST_FILENAME);
204    let tmp_path: PathBuf = dir.join(format!("{MANIFEST_FILENAME}.tmp"));
205
206    manifest.compute_checksum();
207    let bytes = bincode::serialize(manifest)?;
208    let payload = seal(&bytes, meta_dek)?;
209    {
210        let mut file = fs::File::create(&tmp_path)?;
211        file.write_all(&payload)?;
212        file.sync_all()?;
213    }
214    fs::rename(&tmp_path, &final_path)?;
215    if let Ok(d) = fs::File::open(dir) {
216        let _ = d.sync_all();
217    }
218    Ok(())
219}
220
221/// Read the manifest from `<dir>/_mf`, verifying magic and checksum (plaintext)
222/// or the GCM tag (encrypted). `meta_dek` must match the one used at write.
223///
224/// Older on-disk manifests (written with `MANIFEST_VERSION < 3`) are decoded via
225/// [`LegacyManifestV2`] and migrated forward; their `auto_inc_next` synthesizes
226/// to `0` (unseeded), so existing database directories open unchanged and the
227/// engine seeds the identity counter from `max(id)` on first use.
228pub fn read(dir: impl AsRef<Path>, meta_dek: Option<&[u8; META_DEK_LEN]>) -> Result<Manifest> {
229    let path = dir.as_ref().join(MANIFEST_FILENAME);
230    let bytes = fs::read(&path)?;
231    let plaintext = open_payload(&bytes, meta_dek)?;
232    // The checksum is the trailing 32 bytes; verify it before trusting the body
233    // (works for any manifest version since `checksum` is always last).
234    verify_trailing_checksum(&plaintext)?;
235    let header: ManifestHeader = bincode::deserialize(&plaintext)?;
236    if header.magic != MANIFEST_MAGIC {
237        return Err(MongrelError::MagicMismatch {
238            what: "manifest",
239            expected: MANIFEST_MAGIC,
240            got: header.magic,
241        });
242    }
243    let manifest = if header.format_version < MANIFEST_VERSION {
244        let legacy: LegacyManifestV2 = bincode::deserialize(&plaintext)?;
245        Manifest::from_legacy(legacy)
246    } else {
247        bincode::deserialize::<Manifest>(&plaintext)?
248    };
249    Ok(manifest)
250}
251
252#[cfg(feature = "encryption")]
253fn seal(body: &[u8], meta_dek: Option<&[u8; META_DEK_LEN]>) -> Result<Vec<u8>> {
254    match meta_dek {
255        Some(dek) => encrypt_blob(dek, body),
256        None => Ok(body.to_vec()),
257    }
258}
259
260#[cfg(not(feature = "encryption"))]
261fn seal(body: &[u8], _meta_dek: Option<&[u8; META_DEK_LEN]>) -> Result<Vec<u8>> {
262    Ok(body.to_vec())
263}
264
265#[cfg(feature = "encryption")]
266fn open_payload(bytes: &[u8], meta_dek: Option<&[u8; META_DEK_LEN]>) -> Result<Vec<u8>> {
267    match meta_dek {
268        Some(dek) => decrypt_blob(dek, bytes),
269        None => Ok(bytes.to_vec()),
270    }
271}
272
273#[cfg(not(feature = "encryption"))]
274fn open_payload(bytes: &[u8], _meta_dek: Option<&[u8; META_DEK_LEN]>) -> Result<Vec<u8>> {
275    Ok(bytes.to_vec())
276}
277
278#[cfg(test)]
279mod tests {
280    use super::*;
281    use tempfile::tempdir;
282
283    #[test]
284    fn write_then_read_roundtrips() {
285        let dir = tempdir().unwrap();
286        let mut m = Manifest::new(10, 3);
287        m.current_epoch = 9;
288        m.next_row_id = 100;
289        m.flushed_epoch = 7;
290        m.auto_inc_next = 42;
291        m.runs.push(RunRef {
292            run_id: 0xDEAD,
293            level: 0,
294            epoch_created: 8,
295            row_count: 42,
296        });
297        write_atomic(dir.path(), &mut m, None).unwrap();
298
299        let read_back = read(dir.path(), None).unwrap();
300        assert_eq!(read_back.table_id, 10);
301        assert_eq!(read_back.current_epoch, 9);
302        assert_eq!(read_back.next_row_id, 100);
303        assert_eq!(read_back.flushed_epoch, 7);
304        assert_eq!(read_back.auto_inc_next, 42);
305        assert_eq!(read_back.format_version, MANIFEST_VERSION);
306        assert_eq!(read_back.runs.len(), 1);
307        assert_eq!(read_back.runs[0].run_id, 0xDEAD);
308    }
309
310    #[test]
311    fn reads_legacy_v2_manifest_migrating_auto_inc() {
312        // Hand-build a v2 manifest (the pre-auto_inc_next on-disk shape) and
313        // confirm `read` migrates it forward with an unseeded counter.
314        let dir = tempdir().unwrap();
315        let mut legacy = LegacyManifestV2 {
316            magic: MANIFEST_MAGIC,
317            format_version: 2,
318            table_id: 7,
319            current_epoch: 4,
320            next_row_id: 18,
321            schema_id: 1,
322            runs: Vec::new(),
323            global_idx_epoch: 0,
324            live_count: 9,
325            flushed_epoch: 3,
326            retiring: Vec::new(),
327            checksum: [0u8; 32],
328        };
329        let bytes = bincode::serialize(&legacy).unwrap();
330        legacy.checksum = Sha256::digest(&bytes).into();
331        let sealed = bincode::serialize(&legacy).unwrap();
332        fs::write(dir.path().join(MANIFEST_FILENAME), sealed).unwrap();
333
334        let m = read(dir.path(), None).unwrap();
335        assert_eq!(m.table_id, 7);
336        assert_eq!(m.next_row_id, 18);
337        assert_eq!(m.format_version, MANIFEST_VERSION);
338        assert_eq!(m.auto_inc_next, 0, "legacy counter must come back unseeded");
339    }
340
341    #[test]
342    fn detects_tampering() {
343        let dir = tempdir().unwrap();
344        let mut m = Manifest::new(1, 1);
345        m.current_epoch = 5;
346        write_atomic(dir.path(), &mut m, None).unwrap();
347
348        // Corrupt a byte.
349        let path = dir.path().join(MANIFEST_FILENAME);
350        let mut bytes = fs::read(&path).unwrap();
351        bytes[20] ^= 0xFF;
352        fs::write(&path, bytes).unwrap();
353
354        let err = read(dir.path(), None).unwrap_err();
355        assert!(
356            matches!(
357                err,
358                MongrelError::ChecksumMismatch { .. } | MongrelError::MagicMismatch { .. }
359            ),
360            "got {err:?}"
361        );
362    }
363
364    #[cfg(feature = "encryption")]
365    #[test]
366    fn encrypted_manifest_roundtrips_and_rejects_wrong_key() {
367        let dir = tempdir().unwrap();
368        let dek = [42u8; 32];
369        let mut m = Manifest::new(2, 9);
370        m.current_epoch = 3;
371        m.flushed_epoch = 2;
372        write_atomic(dir.path(), &mut m, Some(&dek)).unwrap();
373        let back = read(dir.path(), Some(&dek)).unwrap();
374        assert_eq!(back.current_epoch, 3);
375        assert_eq!(back.flushed_epoch, 2);
376        // wrong key -> GCM auth failure
377        let wrong = [0u8; 32];
378        assert!(read(dir.path(), Some(&wrong)).is_err());
379    }
380}
mongreldb_core/manifest.rs

mongreldb_core/
manifest.rs
