mongreldb_core/
backup.rs

1//! Online backup manifest and verification.
2
3use crate::{MongrelError, Result};
4use serde::{Deserialize, Serialize};
5use sha2::{Digest, Sha256};
6use std::collections::HashSet;
7use std::io::Read;
8use std::path::{Component, Path, PathBuf};
9
10pub const BACKUP_FORMAT_VERSION: u16 = 1;
11pub const BACKUP_MANIFEST_PATH: &str = "_meta/backup.json";
12const MAX_BACKUP_MANIFEST_BYTES: u64 = 16 * 1024 * 1024;
13
14#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
15pub struct BackupFile {
16    pub path: PathBuf,
17    pub bytes: u64,
18    pub sha256: String,
19}
20
21#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
22pub struct BackupManifest {
23    pub format_version: u16,
24    pub epoch: u64,
25    pub created_unix_nanos: u64,
26    pub files: Vec<BackupFile>,
27}
28
29#[derive(Debug, Clone, PartialEq, Eq)]
30pub struct BackupReport {
31    pub destination: PathBuf,
32    pub epoch: u64,
33    /// Wall-clock time captured under the same commit boundary as `epoch`.
34    pub boundary_unix_nanos: u64,
35    pub files: usize,
36    pub bytes: u64,
37}
38
39impl BackupManifest {
40    pub(crate) fn create_controlled_durable(
41        root: &crate::durable_file::DurableRoot,
42        epoch: u64,
43        paths: &[PathBuf],
44        control: &crate::ExecutionControl,
45    ) -> Result<Self> {
46        let mut paths = paths.to_vec();
47        paths.sort();
48        paths.dedup();
49        let mut files = Vec::with_capacity(paths.len());
50        for (index, path) in paths.into_iter().enumerate() {
51            if index % 256 == 0 {
52                control.checkpoint()?;
53            }
54            validate_relative_path(&path)?;
55            let mut source = root.open_regular(&path)?;
56            let bytes = source.metadata()?.len();
57            files.push(BackupFile {
58                path,
59                bytes,
60                sha256: sha256_open_file_inner(&mut source, Some(control))?,
61            });
62        }
63        Ok(Self {
64            format_version: BACKUP_FORMAT_VERSION,
65            epoch,
66            created_unix_nanos: std::time::SystemTime::now()
67                .duration_since(std::time::UNIX_EPOCH)
68                .unwrap_or_default()
69                .as_nanos() as u64,
70            files,
71        })
72    }
73
74    pub(crate) fn write_to_durable(&self, root: &crate::durable_file::DurableRoot) -> Result<()> {
75        root.create_directory_all("_meta")?;
76        let bytes = serde_json::to_vec_pretty(self)
77            .map_err(|error| MongrelError::Other(format!("backup manifest encode: {error}")))?;
78        root.write_new(BACKUP_MANIFEST_PATH, &bytes)?;
79        Ok(())
80    }
81
82    pub fn total_bytes(&self) -> u64 {
83        self.files.iter().map(|file| file.bytes).sum()
84    }
85}
86
87/// Verify a backup manifest, every listed file size/hash, and the catalog.
88pub fn verify_backup(root: impl AsRef<Path>) -> Result<BackupManifest> {
89    let root = crate::durable_file::DurableRoot::open(root)?;
90    verify_backup_durable(&root)
91}
92
93pub(crate) fn verify_backup_durable(
94    root: &crate::durable_file::DurableRoot,
95) -> Result<BackupManifest> {
96    verify_backup_durable_with_manifest_sha256(root).map(|(manifest, _)| manifest)
97}
98
99pub(crate) fn verify_backup_durable_with_manifest_sha256(
100    root: &crate::durable_file::DurableRoot,
101) -> Result<(BackupManifest, String)> {
102    let manifest_bytes = read_backup_manifest_durable(root)?;
103    let manifest_sha256 = Sha256::digest(&manifest_bytes)
104        .iter()
105        .map(|byte| format!("{byte:02x}"))
106        .collect();
107    let manifest: BackupManifest = serde_json::from_slice(&manifest_bytes).map_err(|error| {
108        MongrelError::InvalidArgument(format!("invalid backup manifest: {error}"))
109    })?;
110    if manifest.format_version != BACKUP_FORMAT_VERSION {
111        return Err(MongrelError::InvalidArgument(format!(
112            "unsupported backup format version {}",
113            manifest.format_version
114        )));
115    }
116    if root.open_regular(crate::catalog::CATALOG_FILENAME).is_err() {
117        return Err(MongrelError::InvalidArgument(
118            "backup has no catalog".into(),
119        ));
120    }
121    let mut expected = HashSet::with_capacity(manifest.files.len() + 1);
122    expected.insert(PathBuf::from(BACKUP_MANIFEST_PATH));
123    for file in &manifest.files {
124        validate_relative_path(&file.path)?;
125        if !expected.insert(file.path.clone()) {
126            return Err(MongrelError::InvalidArgument(format!(
127                "backup manifest lists {} more than once",
128                file.path.display()
129            )));
130        }
131        let mut source = root.open_regular(&file.path).map_err(|error| {
132            MongrelError::Other(format!("backup file {}: {error}", file.path.display()))
133        })?;
134        let metadata = source.metadata()?;
135        if metadata.len() != file.bytes {
136            return Err(MongrelError::Other(format!(
137                "backup file {} size mismatch: expected {}, got {}",
138                file.path.display(),
139                file.bytes,
140                metadata.len()
141            )));
142        }
143        let actual = sha256_open_file_inner(&mut source, None)?;
144        if actual != file.sha256 {
145            return Err(MongrelError::Other(format!(
146                "backup file {} checksum mismatch",
147                file.path.display()
148            )));
149        }
150    }
151    let mut actual = HashSet::new();
152    root.walk_regular_files(
153        |_, _| Ok(true),
154        |_| Ok(()),
155        |relative, _| {
156            actual.insert(relative.to_path_buf());
157            Ok(())
158        },
159    )?;
160    if actual != expected {
161        let mut missing = expected.difference(&actual).cloned().collect::<Vec<_>>();
162        let mut extra = actual.difference(&expected).cloned().collect::<Vec<_>>();
163        missing.sort();
164        extra.sort();
165        return Err(MongrelError::InvalidArgument(format!(
166            "backup file set differs from manifest (missing: {missing:?}; extra: {extra:?})"
167        )));
168    }
169    Ok((manifest, manifest_sha256))
170}
171
172fn read_backup_manifest_durable(root: &crate::durable_file::DurableRoot) -> Result<Vec<u8>> {
173    let file = root.open_regular(BACKUP_MANIFEST_PATH)?;
174    let bytes = file.metadata()?.len();
175    if bytes > MAX_BACKUP_MANIFEST_BYTES {
176        return Err(MongrelError::ResourceLimitExceeded {
177            resource: "backup manifest bytes",
178            requested: usize::try_from(bytes).unwrap_or(usize::MAX),
179            limit: MAX_BACKUP_MANIFEST_BYTES as usize,
180        });
181    }
182    let mut manifest_bytes = Vec::with_capacity(bytes as usize);
183    file.take(MAX_BACKUP_MANIFEST_BYTES + 1)
184        .read_to_end(&mut manifest_bytes)?;
185    Ok(manifest_bytes)
186}
187
188pub(crate) fn copy_file_synced(source: &Path, destination: &Path) -> Result<u64> {
189    let mut source = crate::durable_file::open_regular_nofollow(source)?;
190    copy_open_file_synced(&mut source, destination)
191}
192
193pub(crate) fn copy_open_file_synced(source: &mut std::fs::File, destination: &Path) -> Result<u64> {
194    let parent = destination
195        .parent()
196        .ok_or_else(|| MongrelError::InvalidArgument("invalid backup file path".into()))?;
197    std::fs::create_dir_all(parent)?;
198    let mut destination = std::fs::File::create(destination)?;
199    let bytes = std::io::copy(source, &mut destination)?;
200    destination.sync_all()?;
201    Ok(bytes)
202}
203
204fn sha256_open_file_inner(
205    file: &mut std::fs::File,
206    control: Option<&crate::ExecutionControl>,
207) -> Result<String> {
208    let mut hasher = Sha256::new();
209    let mut buffer = [0u8; 64 * 1024];
210    loop {
211        if let Some(control) = control {
212            control.checkpoint()?;
213        }
214        let read = file.read(&mut buffer)?;
215        if read == 0 {
216            break;
217        }
218        hasher.update(&buffer[..read]);
219    }
220    Ok(hasher
221        .finalize()
222        .iter()
223        .map(|byte| format!("{byte:02x}"))
224        .collect())
225}
226
227fn validate_relative_path(path: &Path) -> Result<()> {
228    if path.as_os_str().is_empty()
229        || path.is_absolute()
230        || path.components().any(|component| {
231            matches!(
232                component,
233                Component::ParentDir | Component::RootDir | Component::Prefix(_)
234            )
235        })
236    {
237        return Err(MongrelError::InvalidArgument(format!(
238            "invalid backup path {}",
239            path.display()
240        )));
241    }
242    Ok(())
243}
mongreldb_core/backup.rs

mongreldb_core/
backup.rs
