mongreldb_core/
backup.rs

1//! Online backup manifest and verification.
2//!
3//! The manifest audit fields (Stage 1G, spec section 10.7) are additive and
4//! every one of them defaults, so backups written before Stage 1G still
5//! deserialize and validate unchanged: `database_id`/`encryption` decode as
6//! `None`, and `catalog_version`, `snapshot_unix_micros`, and
7//! `open_generation` decode as `0` ("unknown"). Conversely, pre-1G binaries
8//! ignore the new keys because the manifest never denied unknown fields.
9
10use crate::{MongrelError, Result};
11use mongreldb_types::ids::DatabaseId;
12use serde::{Deserialize, Serialize};
13use sha2::{Digest, Sha256};
14use std::collections::HashSet;
15use std::io::Read;
16use std::path::{Component, Path, PathBuf};
17
18pub const BACKUP_FORMAT_VERSION: u16 = 1;
19pub const BACKUP_MANIFEST_PATH: &str = "_meta/backup.json";
20const MAX_BACKUP_MANIFEST_BYTES: u64 = 16 * 1024 * 1024;
21/// Persisted replication identity marker reused as the backup database ID.
22const REPLICATION_ID_PATH: &str = "_meta/replication_id";
23const REPLICATION_ID_LEN: usize = 32;
24/// KEK salt marker whose presence identifies an encrypted database.
25const KEYS_PATH: &str = "_meta/keys";
26
27#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
28pub struct BackupFile {
29    pub path: PathBuf,
30    pub bytes: u64,
31    pub sha256: String,
32}
33
34/// Encryption metadata recorded in a backup manifest (Stage 1G). The KEK
35/// salt itself travels inside the backup as the `_meta/keys` file, so only
36/// the scheme identifiers live here.
37#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
38pub struct BackupEncryptionMetadata {
39    /// KEK derivation scheme for the database passphrase.
40    pub kdf: String,
41    /// Page/record cipher guarding the backup files.
42    pub cipher: String,
43}
44
45#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
46pub struct BackupManifest {
47    pub format_version: u16,
48    pub epoch: u64,
49    pub created_unix_nanos: u64,
50    /// Database identity (spec 10.7). Derived from the persisted replication
51    /// identity (`_meta/replication_id`, 32 CSPRNG bytes created on first
52    /// open): the `DatabaseId` is its first 16 bytes. The derivation is
53    /// deterministic, stable across backups of one database, and travels
54    /// with the backup because the marker file is part of the copied file
55    /// set. `None` only when the source directory lacks the marker (a
56    /// database never opened by a replication-aware binary); backup never
57    /// invents or persists a fresh identity.
58    #[serde(default, skip_serializing_if = "Option::is_none")]
59    pub database_id: Option<DatabaseId>,
60    /// Catalog command state-machine version at the backup boundary
61    /// (S1F). `0` means "unknown": pre-1G manifests, legacy catalogs, and
62    /// encrypted catalogs whose bytes cannot be decoded here without the
63    /// database passphrase.
64    #[serde(default)]
65    pub catalog_version: u64,
66    /// Snapshot wall clock in HLC physical units (microseconds since the
67    /// UNIX epoch), captured at manifest creation right after the commit
68    /// boundary was copied. `0` means "unknown" (pre-1G manifests). The
69    /// exact boundary nanoseconds remain available to the caller through
70    /// [`BackupReport::boundary_unix_nanos`].
71    #[serde(default)]
72    pub snapshot_unix_micros: u64,
73    /// WAL open generation scoping `epoch`, read from `_meta/generation`.
74    /// Together `(epoch, open_generation)` is the log continuation position:
75    /// the durable commit watermark plus the generation that scopes the WAL
76    /// record sequence it refers to. `0` means "unknown" (pre-1G manifests
77    /// or a missing sidecar).
78    #[serde(default)]
79    pub open_generation: u64,
80    /// Encryption metadata; `None` identifies a plaintext backup.
81    #[serde(default, skip_serializing_if = "Option::is_none")]
82    pub encryption: Option<BackupEncryptionMetadata>,
83    pub files: Vec<BackupFile>,
84}
85
86#[derive(Debug, Clone, PartialEq, Eq)]
87pub struct BackupReport {
88    pub destination: PathBuf,
89    pub epoch: u64,
90    /// Wall-clock time captured under the same commit boundary as `epoch`.
91    pub boundary_unix_nanos: u64,
92    pub files: usize,
93    pub bytes: u64,
94}
95
96/// Outcome of a post-restore validation pass (Stage 1G, spec 10.7), in the
97/// idiom of [`crate::gc::CheckReport`]/[`crate::gc::DoctorReport`]: soft
98/// findings collect in `issues`, hard corruption returns `Err`.
99#[derive(Debug, Default, Clone, PartialEq, Eq)]
100pub struct RestoreReport {
101    /// Manifest-listed files whose size and SHA-256 were re-verified.
102    pub files_checked: usize,
103    /// Files whose checksum matched the manifest.
104    pub files_ok: usize,
105    /// Total bytes re-hashed.
106    pub bytes_checked: u64,
107    /// The catalog decoded. `false` for encrypted trees validated without
108    /// the database passphrase (recorded as an issue, not a failure).
109    pub catalog_loaded: bool,
110    /// Manifest structure, file hashes, and the manifest/file-set equality
111    /// all held.
112    pub manifest_consistent: bool,
113    /// Non-fatal findings.
114    pub issues: Vec<String>,
115}
116
117/// Post-restore validation pass over a backup or restored database tree:
118/// re-verifies the manifest, every listed file size/hash, and the manifest/
119/// file-set equality, then loads the catalog. Row counts are intentionally
120/// out of scope: opening the tree as a database would take its lock and
121/// bump its open generation.
122pub fn validate_restore(root: impl AsRef<Path>) -> Result<RestoreReport> {
123    let root = crate::durable_file::DurableRoot::open(root)?;
124    validate_restore_durable(&root)
125}
126
127pub(crate) fn validate_restore_durable(
128    root: &crate::durable_file::DurableRoot,
129) -> Result<RestoreReport> {
130    let (manifest, _) = verify_backup_durable_with_manifest_sha256(root)?;
131    let mut report = RestoreReport {
132        files_checked: manifest.files.len(),
133        files_ok: manifest.files.len(),
134        bytes_checked: manifest.total_bytes(),
135        manifest_consistent: true,
136        ..RestoreReport::default()
137    };
138    match crate::catalog::read_durable(root, None)? {
139        Some(_) => report.catalog_loaded = true,
140        None => report
141            .issues
142            .push("catalog did not decode without a passphrase (encrypted)".into()),
143    }
144    Ok(report)
145}
146
147impl BackupManifest {
148    pub(crate) fn create_controlled_durable(
149        root: &crate::durable_file::DurableRoot,
150        epoch: u64,
151        paths: &[PathBuf],
152        control: &crate::ExecutionControl,
153    ) -> Result<Self> {
154        let mut paths = paths.to_vec();
155        paths.sort();
156        paths.dedup();
157        let mut files = Vec::with_capacity(paths.len());
158        for (index, path) in paths.into_iter().enumerate() {
159            if index % 256 == 0 {
160                control.checkpoint()?;
161            }
162            validate_relative_path(&path)?;
163            let mut source = root.open_regular(&path)?;
164            let bytes = source.metadata()?.len();
165            files.push(BackupFile {
166                path,
167                bytes,
168                sha256: sha256_open_file_inner(&mut source, Some(control))?,
169            });
170        }
171        let now = std::time::SystemTime::now()
172            .duration_since(std::time::UNIX_EPOCH)
173            .unwrap_or_default();
174        Ok(Self {
175            format_version: BACKUP_FORMAT_VERSION,
176            epoch,
177            created_unix_nanos: now.as_nanos() as u64,
178            database_id: backup_database_id(root)?,
179            catalog_version: backup_catalog_version(root)?,
180            snapshot_unix_micros: now.as_micros() as u64,
181            open_generation: crate::catalog::read_generation(root)?.unwrap_or(0),
182            encryption: backup_encryption_metadata(root)?,
183            files,
184        })
185    }
186
187    pub(crate) fn write_to_durable(&self, root: &crate::durable_file::DurableRoot) -> Result<()> {
188        root.create_directory_all("_meta")?;
189        let bytes = serde_json::to_vec_pretty(self)
190            .map_err(|error| MongrelError::Other(format!("backup manifest encode: {error}")))?;
191        root.write_new(BACKUP_MANIFEST_PATH, &bytes)?;
192        Ok(())
193    }
194
195    pub fn total_bytes(&self) -> u64 {
196        self.files.iter().map(|file| file.bytes).sum()
197    }
198}
199
200/// Derive the backup-scoped database ID from the persisted replication
201/// identity marker. Read-only: a missing marker degrades to `None` rather
202/// than inventing an identity (database identity lifecycle belongs to
203/// `database.rs`), while a malformed marker fails closed because the marker
204/// is immutable once created.
205fn backup_database_id(root: &crate::durable_file::DurableRoot) -> Result<Option<DatabaseId>> {
206    let file = match root.open_regular(REPLICATION_ID_PATH) {
207        Ok(file) => file,
208        Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
209        Err(error) => return Err(error.into()),
210    };
211    let mut bytes = Vec::with_capacity(REPLICATION_ID_LEN);
212    file.take(REPLICATION_ID_LEN as u64 + 1)
213        .read_to_end(&mut bytes)?;
214    if bytes.len() != REPLICATION_ID_LEN || bytes.iter().all(|byte| *byte == 0) {
215        return Err(MongrelError::Other(format!(
216            "invalid database replication identity length: got {}, expected {REPLICATION_ID_LEN} nonzero bytes",
217            bytes.len()
218        )));
219    }
220    let mut id = [0u8; 16];
221    id.copy_from_slice(&bytes[..16]);
222    Ok(Some(DatabaseId::from_bytes(id)))
223}
224
225/// Catalog version at the backup boundary. Encrypted catalogs cannot be
226/// decoded without the database passphrase, which the manifest builder does
227/// not hold; those record `0` ("unknown", see [`BackupManifest`]).
228fn backup_catalog_version(root: &crate::durable_file::DurableRoot) -> Result<u64> {
229    Ok(crate::catalog::read_durable(root, None)?
230        .map(|catalog| catalog.catalog_version())
231        .unwrap_or(0))
232}
233
234fn backup_encryption_metadata(
235    root: &crate::durable_file::DurableRoot,
236) -> Result<Option<BackupEncryptionMetadata>> {
237    match root.open_regular(KEYS_PATH) {
238        Ok(_) => Ok(Some(BackupEncryptionMetadata {
239            kdf: "argon2id-hkdf-sha256".into(),
240            cipher: "aes-256-gcm".into(),
241        })),
242        Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
243        Err(error) => Err(error.into()),
244    }
245}
246
247/// Verify a backup manifest, every listed file size/hash, and the catalog.
248pub fn verify_backup(root: impl AsRef<Path>) -> Result<BackupManifest> {
249    let root = crate::durable_file::DurableRoot::open(root)?;
250    verify_backup_durable(&root)
251}
252
253pub(crate) fn verify_backup_durable(
254    root: &crate::durable_file::DurableRoot,
255) -> Result<BackupManifest> {
256    verify_backup_durable_with_manifest_sha256(root).map(|(manifest, _)| manifest)
257}
258
259pub(crate) fn verify_backup_durable_with_manifest_sha256(
260    root: &crate::durable_file::DurableRoot,
261) -> Result<(BackupManifest, String)> {
262    let manifest_bytes = read_backup_manifest_durable(root)?;
263    let manifest_sha256 = Sha256::digest(&manifest_bytes)
264        .iter()
265        .map(|byte| format!("{byte:02x}"))
266        .collect();
267    let manifest: BackupManifest = serde_json::from_slice(&manifest_bytes).map_err(|error| {
268        MongrelError::InvalidArgument(format!("invalid backup manifest: {error}"))
269    })?;
270    if manifest.format_version != BACKUP_FORMAT_VERSION {
271        return Err(MongrelError::InvalidArgument(format!(
272            "unsupported backup format version {}",
273            manifest.format_version
274        )));
275    }
276    if root.open_regular(crate::catalog::CATALOG_FILENAME).is_err() {
277        return Err(MongrelError::InvalidArgument(
278            "backup has no catalog".into(),
279        ));
280    }
281    let mut expected = HashSet::with_capacity(manifest.files.len() + 1);
282    expected.insert(PathBuf::from(BACKUP_MANIFEST_PATH));
283    for file in &manifest.files {
284        validate_relative_path(&file.path)?;
285        if !expected.insert(file.path.clone()) {
286            return Err(MongrelError::InvalidArgument(format!(
287                "backup manifest lists {} more than once",
288                file.path.display()
289            )));
290        }
291        let mut source = root.open_regular(&file.path).map_err(|error| {
292            MongrelError::Other(format!("backup file {}: {error}", file.path.display()))
293        })?;
294        let metadata = source.metadata()?;
295        if metadata.len() != file.bytes {
296            return Err(MongrelError::Other(format!(
297                "backup file {} size mismatch: expected {}, got {}",
298                file.path.display(),
299                file.bytes,
300                metadata.len()
301            )));
302        }
303        let actual = sha256_open_file_inner(&mut source, None)?;
304        if actual != file.sha256 {
305            return Err(MongrelError::Other(format!(
306                "backup file {} checksum mismatch",
307                file.path.display()
308            )));
309        }
310    }
311    let mut actual = HashSet::new();
312    root.walk_regular_files(
313        |_, _| Ok(true),
314        |_| Ok(()),
315        |relative, _| {
316            actual.insert(relative.to_path_buf());
317            Ok(())
318        },
319    )?;
320    if actual != expected {
321        let mut missing = expected.difference(&actual).cloned().collect::<Vec<_>>();
322        let mut extra = actual.difference(&expected).cloned().collect::<Vec<_>>();
323        missing.sort();
324        extra.sort();
325        return Err(MongrelError::InvalidArgument(format!(
326            "backup file set differs from manifest (missing: {missing:?}; extra: {extra:?})"
327        )));
328    }
329    Ok((manifest, manifest_sha256))
330}
331
332fn read_backup_manifest_durable(root: &crate::durable_file::DurableRoot) -> Result<Vec<u8>> {
333    let file = root.open_regular(BACKUP_MANIFEST_PATH)?;
334    let bytes = file.metadata()?.len();
335    if bytes > MAX_BACKUP_MANIFEST_BYTES {
336        return Err(MongrelError::ResourceLimitExceeded {
337            resource: "backup manifest bytes",
338            requested: usize::try_from(bytes).unwrap_or(usize::MAX),
339            limit: MAX_BACKUP_MANIFEST_BYTES as usize,
340        });
341    }
342    let mut manifest_bytes = Vec::with_capacity(bytes as usize);
343    file.take(MAX_BACKUP_MANIFEST_BYTES + 1)
344        .read_to_end(&mut manifest_bytes)?;
345    Ok(manifest_bytes)
346}
347
348pub(crate) fn copy_file_synced(source: &Path, destination: &Path) -> Result<u64> {
349    let mut source = crate::durable_file::open_regular_nofollow(source)?;
350    copy_open_file_synced(&mut source, destination)
351}
352
353pub(crate) fn copy_open_file_synced(source: &mut std::fs::File, destination: &Path) -> Result<u64> {
354    let parent = destination
355        .parent()
356        .ok_or_else(|| MongrelError::InvalidArgument("invalid backup file path".into()))?;
357    std::fs::create_dir_all(parent)?;
358    let mut destination = std::fs::File::create(destination)?;
359    let bytes = std::io::copy(source, &mut destination)?;
360    destination.sync_all()?;
361    Ok(bytes)
362}
363
364pub(crate) fn sha256_open_file_inner(
365    file: &mut std::fs::File,
366    control: Option<&crate::ExecutionControl>,
367) -> Result<String> {
368    let mut hasher = Sha256::new();
369    let mut buffer = [0u8; 64 * 1024];
370    loop {
371        if let Some(control) = control {
372            control.checkpoint()?;
373        }
374        let read = file.read(&mut buffer)?;
375        if read == 0 {
376            break;
377        }
378        hasher.update(&buffer[..read]);
379    }
380    Ok(hasher
381        .finalize()
382        .iter()
383        .map(|byte| format!("{byte:02x}"))
384        .collect())
385}
386
387fn validate_relative_path(path: &Path) -> Result<()> {
388    if path.as_os_str().is_empty()
389        || path.is_absolute()
390        || path.components().any(|component| {
391            matches!(
392                component,
393                Component::ParentDir | Component::RootDir | Component::Prefix(_)
394            )
395        })
396    {
397        return Err(MongrelError::InvalidArgument(format!(
398            "invalid backup path {}",
399            path.display()
400        )));
401    }
402    Ok(())
403}
404
405#[cfg(test)]
406mod tests {
407    use super::*;
408
409    #[test]
410    fn pre_1g_manifest_json_decodes_with_unknown_audit_fields() {
411        let json = r#"{"format_version":1,"epoch":7,"created_unix_nanos":42,"files":[]}"#;
412        let manifest: BackupManifest = serde_json::from_str(json).unwrap();
413        assert_eq!(manifest.format_version, 1);
414        assert_eq!(manifest.epoch, 7);
415        assert_eq!(manifest.database_id, None);
416        assert_eq!(manifest.catalog_version, 0);
417        assert_eq!(manifest.snapshot_unix_micros, 0);
418        assert_eq!(manifest.open_generation, 0);
419        assert_eq!(manifest.encryption, None);
420    }
421
422    #[test]
423    fn manifest_serde_round_trip_preserves_audit_fields() {
424        let manifest = BackupManifest {
425            format_version: BACKUP_FORMAT_VERSION,
426            epoch: 9,
427            created_unix_nanos: 1_000,
428            database_id: Some(DatabaseId::from_bytes([7; 16])),
429            catalog_version: 3,
430            snapshot_unix_micros: 1,
431            open_generation: 2,
432            encryption: Some(BackupEncryptionMetadata {
433                kdf: "argon2id-hkdf-sha256".into(),
434                cipher: "aes-256-gcm".into(),
435            }),
436            files: vec![BackupFile {
437                path: PathBuf::from("CATALOG"),
438                bytes: 10,
439                sha256: "ab".repeat(32),
440            }],
441        };
442        let bytes = serde_json::to_vec_pretty(&manifest).unwrap();
443        let decoded: BackupManifest = serde_json::from_slice(&bytes).unwrap();
444        assert_eq!(decoded, manifest);
445    }
446
447    #[test]
448    fn database_id_follows_the_replication_identity_marker() {
449        let directory = tempfile::tempdir().unwrap();
450        let root = crate::durable_file::DurableRoot::open(directory.path()).unwrap();
451        assert_eq!(backup_database_id(&root).unwrap(), None);
452
453        root.create_directory_all("_meta").unwrap();
454        let identity = [7u8; REPLICATION_ID_LEN];
455        root.write_new(REPLICATION_ID_PATH, &identity).unwrap();
456        let id = backup_database_id(&root).unwrap().unwrap();
457        assert_eq!(id.as_bytes(), &[7u8; 16]);
458        // Stable: a second read derives the same identity.
459        assert_eq!(backup_database_id(&root).unwrap(), Some(id));
460    }
461
462    #[test]
463    fn database_id_fails_closed_on_a_malformed_identity_marker() {
464        let directory = tempfile::tempdir().unwrap();
465        let root = crate::durable_file::DurableRoot::open(directory.path()).unwrap();
466        root.create_directory_all("_meta").unwrap();
467        root.write_new(REPLICATION_ID_PATH, &[1, 2, 3]).unwrap();
468        assert!(backup_database_id(&root).is_err());
469
470        let zeroed = tempfile::tempdir().unwrap();
471        let zeroed = crate::durable_file::DurableRoot::open(zeroed.path()).unwrap();
472        zeroed.create_directory_all("_meta").unwrap();
473        zeroed
474            .write_new(REPLICATION_ID_PATH, &[0u8; REPLICATION_ID_LEN])
475            .unwrap();
476        assert!(backup_database_id(&zeroed).is_err());
477    }
478}
mongreldb_core/backup.rs

mongreldb_core/
backup.rs
