hayson_engine/database/
backup.rs

1use std::collections::BTreeSet;
2use std::fs;
3use std::io;
4use std::path::{Component, Path, PathBuf};
5
6use serde_json::{json, Value};
7
8use crate::atomic::write_atomic;
9use crate::catalog::{parse_catalog, Catalog};
10use crate::errors::HaysonSchemaError;
11use crate::lock::DirLock;
12use crate::storage::{partition_names_in, LOG_SUFFIX};
13
14use super::database::{Database, DatabaseError, HeldLock, LockMode};
15use super::layout::{Layout, LayoutOptions};
16use super::verify::{verify, VerifyOptions};
17
18/// The manifest a backup carries next to the files it copied.
19pub const BACKUP_MANIFEST: &str = "hayson-backup.json";
20const BACKUP_FORMAT: u64 = 1;
21const RESTORE_OLD: &str = ".restore-old";
22const RESTORE_NEW: &str = ".restore-new";
23
24#[derive(Debug, Clone)]
25pub struct BackupOptions {
26    /// How the writer lock is taken for the length of the copy. [`LockMode::Skip`] copies a database
27    /// a writer may be appending to, which can capture a half-written last line (the engine cuts
28    /// such a tail off on the next writable open) — but nothing worse.
29    pub lock: LockMode,
30    /// Also copy the `<table>.idx` snapshots. Off by default: they are caches the engine rebuilds
31    /// from the logs, so leaving them out makes a backup smaller and no less complete.
32    pub indexes: bool,
33}
34
35impl Default for BackupOptions {
36    fn default() -> Self {
37        BackupOptions {
38            lock: LockMode::default(),
39            indexes: false,
40        }
41    }
42}
43
44#[derive(Debug, Clone, PartialEq, Eq)]
45pub struct BackupResult {
46    pub dest: PathBuf,
47    pub files: usize,
48    pub bytes: u64,
49}
50
51#[derive(Debug, Clone, Default)]
52pub struct RestoreOptions {
53    /// How the writer lock on the directory being restored into is taken. As in [`BackupOptions`].
54    pub lock: LockMode,
55}
56
57#[derive(Debug, Clone, PartialEq, Eq)]
58pub struct RestoreResult {
59    pub dir: PathBuf,
60    pub files: usize,
61    pub bytes: u64,
62    /// Files of the database that was there before and were replaced.
63    pub replaced: usize,
64}
65
66fn refuse(message: String) -> DatabaseError {
67    DatabaseError::Schema(HaysonSchemaError(message))
68}
69
70fn read_catalog_of(layout: &Layout) -> Result<Catalog, DatabaseError> {
71    let path = layout.metadata_path();
72    let raw = fs::read_to_string(path).map_err(|error| {
73        if error.kind() == io::ErrorKind::NotFound {
74            refuse(format!(
75                "\"{}\" does not exist — there is no database to back up",
76                path.display()
77            ))
78        } else {
79            DatabaseError::Io(error)
80        }
81    })?;
82    let value: Value = serde_json::from_str(&raw)
83        .map_err(|error| DatabaseError::Catalog(format!("\"{}\" is not valid JSON: {error}", path.display())))?;
84    Ok(parse_catalog(&value)?)
85}
86
87fn relative_safely(root: &Path, path: &Path) -> Result<PathBuf, DatabaseError> {
88    let relative = path
89        .strip_prefix(root)
90        .map_err(|_| refuse(format!("\"{}\" is outside the database directory", path.display())))?;
91    check_relative(relative)?;
92    Ok(relative.to_path_buf())
93}
94
95/// A path a backup or a restore may join onto a root: relative, no `..`, no root, no prefix.
96fn check_relative(path: &Path) -> Result<(), DatabaseError> {
97    let sane = path.components().all(|part| matches!(part, Component::Normal(_)));
98    if sane && path.components().next().is_some() {
99        Ok(())
100    } else {
101        Err(refuse(format!(
102            "\"{}\" is not a path inside the database directory",
103            path.display()
104        )))
105    }
106}
107
108fn files_in(dir: &Path) -> Result<Vec<PathBuf>, DatabaseError> {
109    let mut found = Vec::new();
110    let Ok(entries) = fs::read_dir(dir) else {
111        return Ok(found);
112    };
113    for entry in entries {
114        let entry = entry?;
115        let kind = entry.file_type()?;
116        if kind.is_symlink() {
117            return Err(refuse(format!(
118                "\"{}\" is a symbolic link — refusing to follow it",
119                entry.path().display()
120            )));
121        }
122        if kind.is_file() {
123            found.push(entry.path());
124        }
125    }
126    found.sort();
127    Ok(found)
128}
129
130/// Every file the database at `root` accounts for, as paths relative to `root`: its metadata
131/// document, the logs of its tables (one file, or one per partition), the files under each table's
132/// image directory and, when asked, the index snapshots. Anything else in the directory — another
133/// database in a subfolder, say — is not the database's and is left out.
134fn owned_files(root: &Path, layout: &Layout, catalog: &Catalog, indexes: bool) -> Result<Vec<PathBuf>, DatabaseError> {
135    let mut owned: BTreeSet<PathBuf> = BTreeSet::new();
136    let mut add = |path: &Path| -> Result<(), DatabaseError> {
137        owned.insert(relative_safely(root, path)?);
138        Ok(())
139    };
140    add(layout.metadata_path())?;
141    for (name, entry) in &catalog.tables {
142        let flat = layout.log_path(name);
143        match fs::symlink_metadata(&flat) {
144            Ok(meta) if meta.file_type().is_symlink() => {
145                return Err(refuse(format!(
146                    "\"{}\" is a symbolic link — refusing to follow it",
147                    flat.display()
148                )))
149            }
150            Ok(meta) if meta.is_file() => add(&flat)?,
151            _ => {}
152        }
153        let partitions = layout.tables().join(name);
154        for part in partition_names_in(&partitions) {
155            add(&partitions.join(format!("{part}{LOG_SUFFIX}")))?;
156        }
157        if indexes && layout.index_path(name).is_file() {
158            add(&layout.index_path(name))?;
159        }
160        let image_dir = entry
161            .options
162            .as_ref()
163            .and_then(|options| options.image_dir.clone())
164            .or_else(|| catalog.defaults.as_ref().and_then(|d| d.image_dir.clone()))
165            .map(|dir| root.join(dir))
166            .unwrap_or_else(|| layout.tables().join(format!("{name}.images")));
167        for file in files_in(&image_dir)? {
168            add(&file)?;
169        }
170    }
171    Ok(owned.into_iter().collect())
172}
173
174fn hold(root: &Path, layout: &Layout, lock: &LockMode) -> Result<HeldLock, DatabaseError> {
175    let LockMode::Enabled(options) = lock else {
176        return Ok(HeldLock(None));
177    };
178    if root.is_dir() {
179        fs::create_dir_all(layout.runtime())?;
180    }
181    Ok(HeldLock(Some(DirLock::acquire_as(
182        layout.lock_path(),
183        options.clone(),
184        root.to_path_buf(),
185    )?)))
186}
187
188fn copy_synced(from: &Path, to: &Path) -> io::Result<u64> {
189    if let Some(parent) = to.parent() {
190        fs::create_dir_all(parent)?;
191    }
192    let bytes = fs::copy(from, to)?;
193    fs::File::open(to)?.sync_all()?;
194    Ok(bytes)
195}
196
197/// Copies a database directory into `dest`: the files the database accounts for (see
198/// [`owned_files`]) plus a `hayson-backup.json` manifest listing them with their sizes.
199///
200/// `dest` must not exist or must be an empty directory. The copy is built next to it and renamed
201/// into place, so a backup that fails halfway leaves no half-filled `dest`. The directory is held
202/// under the writer lock for the length of the copy unless `options.lock` is [`LockMode::Skip`];
203/// a database with an unfinished unit of work is refused, because its log and journal disagree
204/// until a writable open finishes it.
205///
206/// History kept in a database of its own is not part of this database's files, so it is not copied.
207pub fn backup_database(dir: &Path, dest: &Path, options: &BackupOptions) -> Result<BackupResult, DatabaseError> {
208    let layout = Database::resolve_layout(dir, &LayoutOptions::default())?;
209    let catalog = read_catalog_of(&layout)?;
210    if dest.exists()
211        && fs::read_dir(dest)
212            .map(|mut entries| entries.next().is_some())
213            .unwrap_or(true)
214    {
215        return Err(refuse(format!(
216            "\"{}\" already exists and is not an empty directory",
217            dest.display()
218        )));
219    }
220    let partial = sibling(dest, ".partial");
221    if partial.exists() {
222        fs::remove_dir_all(&partial)?;
223    }
224
225    let held = hold(dir, &layout, &options.lock)?;
226    let outcome = copy_database(dir, &layout, &catalog, &partial, options);
227    drop(held);
228    match outcome {
229        Ok(result) => {
230            if dest.exists() {
231                fs::remove_dir(dest)?;
232            }
233            fs::rename(&partial, dest)?;
234            Ok(BackupResult {
235                dest: dest.to_path_buf(),
236                ..result
237            })
238        }
239        Err(error) => {
240            let _ = fs::remove_dir_all(&partial);
241            Err(error)
242        }
243    }
244}
245
246fn sibling(path: &Path, suffix: &str) -> PathBuf {
247    let mut name = path.file_name().map(|n| n.to_os_string()).unwrap_or_default();
248    name.push(suffix);
249    path.with_file_name(name)
250}
251
252fn copy_database(
253    root: &Path,
254    layout: &Layout,
255    catalog: &Catalog,
256    partial: &Path,
257    options: &BackupOptions,
258) -> Result<BackupResult, DatabaseError> {
259    if layout.journal_path().exists() {
260        return Err(refuse(format!(
261            "\"{}\" holds an unfinished unit of work — open the database for writing once so it is finished, then back it up",
262            layout.journal_path().display()
263        )));
264    }
265    let files = owned_files(root, layout, catalog, options.indexes)?;
266    fs::create_dir_all(partial)?;
267    let mut listed = Vec::new();
268    let mut total = 0u64;
269    for relative in &files {
270        let bytes = copy_synced(&root.join(relative), &partial.join(relative))?;
271        total += bytes;
272        listed.push(json!({ "path": relative.to_string_lossy().replace('\\', "/"), "bytes": bytes }));
273    }
274    let manifest = json!({
275        "format": BACKUP_FORMAT,
276        "createdAt": chrono::Utc::now().timestamp_millis(),
277        "engine": env!("CARGO_PKG_VERSION"),
278        "metadataFile": layout.metadata_file(),
279        "files": listed,
280    });
281    let mut text = serde_json::to_string_pretty(&manifest).map_err(|e| DatabaseError::Catalog(e.to_string()))?;
282    text.push('\n');
283    write_atomic(&partial.join(BACKUP_MANIFEST), text.as_bytes(), None)?;
284    Ok(BackupResult {
285        dest: partial.to_path_buf(),
286        files: files.len(),
287        bytes: total,
288    })
289}
290
291struct ManifestEntry {
292    relative: PathBuf,
293    bytes: u64,
294}
295
296fn read_manifest(backup: &Path) -> Result<(String, Vec<ManifestEntry>), DatabaseError> {
297    let path = backup.join(BACKUP_MANIFEST);
298    let raw = fs::read_to_string(&path).map_err(|error| {
299        refuse(format!(
300            "\"{}\" is not a readable backup manifest: {error}",
301            path.display()
302        ))
303    })?;
304    let manifest: Value = serde_json::from_str(&raw)
305        .map_err(|error| refuse(format!("\"{}\" is not valid JSON: {error}", path.display())))?;
306    if manifest.get("format").and_then(Value::as_u64) != Some(BACKUP_FORMAT) {
307        return Err(refuse(format!(
308            "\"{}\" is not a backup format this engine reads",
309            path.display()
310        )));
311    }
312    let metadata_file = manifest
313        .get("metadataFile")
314        .and_then(Value::as_str)
315        .ok_or_else(|| refuse("backup manifest names no \"metadataFile\"".to_string()))?
316        .to_string();
317    let listed = manifest
318        .get("files")
319        .and_then(Value::as_array)
320        .ok_or_else(|| refuse("backup manifest lists no \"files\"".to_string()))?;
321    let mut entries = Vec::new();
322    for item in listed {
323        let relative = PathBuf::from(
324            item.get("path")
325                .and_then(Value::as_str)
326                .ok_or_else(|| refuse("a backup manifest entry has no \"path\"".to_string()))?,
327        );
328        check_relative(&relative)?;
329        let bytes = item
330            .get("bytes")
331            .and_then(Value::as_u64)
332            .ok_or_else(|| refuse(format!("manifest entry \"{}\" has no \"bytes\"", relative.display())))?;
333        entries.push(ManifestEntry { relative, bytes });
334    }
335    Ok((metadata_file, entries))
336}
337
338/// Restores a backup made by [`backup_database`] into `dir`, replacing the database that is there.
339///
340/// Recoverable: nothing is deleted until the replacement is in place. The backup is copied into
341/// `<dir>/.restore-new` and checked (sizes against the manifest, then [`verify`] with no error);
342/// the files of the database being replaced are moved into `<dir>/.restore-old`; the new files are
343/// moved in; and only then is `.restore-old` removed. If a move fails, the files already moved are
344/// put back, and when even that fails both errors are reported. A `.restore-old` left by an
345/// interrupted restore is refused, because it may hold the only good copy.
346///
347/// Only the files the database accounts for are replaced, so another database living in a
348/// subfolder of `dir` is untouched.
349pub fn restore_database(backup: &Path, dir: &Path, options: &RestoreOptions) -> Result<RestoreResult, DatabaseError> {
350    let (metadata_file, entries) = read_manifest(backup)?;
351    for entry in &entries {
352        let path = backup.join(&entry.relative);
353        let meta = fs::symlink_metadata(&path).map_err(|error| {
354            refuse(format!(
355                "backup file \"{}\" is missing: {error}",
356                entry.relative.display()
357            ))
358        })?;
359        if !meta.is_file() || meta.len() != entry.bytes {
360            return Err(refuse(format!(
361                "backup file \"{}\" does not match its manifest entry ({} bytes expected)",
362                entry.relative.display(),
363                entry.bytes
364            )));
365        }
366    }
367    let old = dir.join(RESTORE_OLD);
368    if old.exists() {
369        return Err(refuse(format!(
370            "\"{}\" exists — an earlier restore was interrupted and it may hold the only good copy of the database; move its files back (or delete it) before restoring again",
371            old.display()
372        )));
373    }
374    fs::create_dir_all(dir)?;
375    let layout = Database::resolve_layout(
376        dir,
377        &LayoutOptions {
378            metadata_file: Some(metadata_file.clone()),
379            ..LayoutOptions::default()
380        },
381    )?;
382
383    let held = hold(dir, &layout, &options.lock)?;
384    let staged = dir.join(RESTORE_NEW);
385    let outcome = install(dir, &layout, backup, &entries, &staged, &old);
386    let _ = fs::remove_dir_all(&staged);
387    drop(held);
388    outcome
389}
390
391fn install(
392    root: &Path,
393    layout: &Layout,
394    backup: &Path,
395    entries: &[ManifestEntry],
396    staged: &Path,
397    old: &Path,
398) -> Result<RestoreResult, DatabaseError> {
399    if staged.exists() {
400        fs::remove_dir_all(staged)?;
401    }
402    let mut total = 0u64;
403    for entry in entries {
404        total += copy_synced(&backup.join(&entry.relative), &staged.join(&entry.relative))?;
405    }
406    let report = verify(
407        staged,
408        &VerifyOptions {
409            relations: false,
410            ..VerifyOptions::default()
411        },
412    );
413    if let Some(problem) = report
414        .problems
415        .iter()
416        .find(|p| p.severity == super::verify::Severity::Error)
417    {
418        return Err(refuse(format!(
419            "the backup does not verify ({}: {}) — nothing was changed",
420            problem.code, problem.message
421        )));
422    }
423
424    // The files of the database now there: what its catalog accounts for (when it can be read), plus
425    // whatever the backup is about to overwrite.
426    let mut replaced: BTreeSet<PathBuf> = entries.iter().map(|e| e.relative.clone()).collect();
427    if let Ok(catalog) = read_catalog_of(layout) {
428        if let Ok(current) = owned_files(root, layout, &catalog, true) {
429            replaced.extend(current);
430        }
431    }
432    replaced.retain(|relative| root.join(relative).is_file());
433
434    fs::create_dir_all(old)?;
435    let mut moved_out: Vec<PathBuf> = Vec::new();
436    let mut moved_in: Vec<PathBuf> = Vec::new();
437    let swap = (|| -> io::Result<()> {
438        for relative in &replaced {
439            move_file(&root.join(relative), &old.join(relative))?;
440            moved_out.push(relative.clone());
441        }
442        for entry in entries {
443            move_file(&staged.join(&entry.relative), &root.join(&entry.relative))?;
444            moved_in.push(entry.relative.clone());
445        }
446        Ok(())
447    })();
448    match swap {
449        Ok(()) => {
450            fs::remove_dir_all(old)?;
451            Ok(RestoreResult {
452                dir: root.to_path_buf(),
453                files: entries.len(),
454                bytes: total,
455                replaced: replaced.len(),
456            })
457        }
458        Err(error) => {
459            let rollback = (|| -> io::Result<()> {
460                for relative in &moved_in {
461                    fs::remove_file(root.join(relative))?;
462                }
463                for relative in &moved_out {
464                    move_file(&old.join(relative), &root.join(relative))?;
465                }
466                Ok(())
467            })();
468            match rollback {
469                Ok(()) => {
470                    let _ = fs::remove_dir_all(old);
471                    Err(DatabaseError::Io(error))
472                }
473                Err(rollback_error) => Err(refuse(format!(
474                    "restore failed ({error}) and putting the previous files back failed too ({rollback_error}) — the previous database is in \"{}\"",
475                    old.display()
476                ))),
477            }
478        }
479    }
480}
481
482fn move_file(from: &Path, to: &Path) -> io::Result<()> {
483    if let Some(parent) = to.parent() {
484        fs::create_dir_all(parent)?;
485    }
486    fs::rename(from, to)
487}
hayson_engine/database/backup.rs

hayson_engine/database/
backup.rs
