hayson_engine/database/
backup.rs

1use std::collections::BTreeSet;
2use std::fs;
3use std::io;
4use std::path::{Component, Path, PathBuf};
5
6use serde_json::{json, Value};
7
8use crate::atomic::write_atomic;
9use crate::catalog::{parse_catalog, Catalog};
10use crate::errors::HaysonSchemaError;
11use crate::lock::DirLock;
12use crate::storage::{partition_names_in, LOG_SUFFIX};
13
14use super::database::{Database, DatabaseError, HeldLock, LockMode};
15use super::layout::{Layout, LayoutOptions};
16use super::verify::{verify, VerifyOptions};
17
18/// The manifest a backup carries next to the files it copied.
19pub const BACKUP_MANIFEST: &str = "hayson-backup.json";
20const BACKUP_FORMAT: u64 = 1;
21const RESTORE_OLD: &str = ".restore-old";
22const RESTORE_NEW: &str = ".restore-new";
23
24#[derive(Debug, Clone)]
25pub struct BackupOptions {
26    /// How the writer lock is taken for the length of the copy. [`LockMode::Skip`] copies a database
27    /// a writer may be appending to, which can capture a half-written last line (the engine cuts
28    /// such a tail off on the next writable open) — but nothing worse.
29    pub lock: LockMode,
30    /// Also copy the `<table>.idx` snapshots. Off by default: they are caches the engine rebuilds
31    /// from the logs, so leaving them out makes a backup smaller and no less complete.
32    pub indexes: bool,
33}
34
35impl Default for BackupOptions {
36    fn default() -> Self {
37        BackupOptions {
38            lock: LockMode::default(),
39            indexes: false,
40        }
41    }
42}
43
44#[derive(Debug, Clone, PartialEq, Eq)]
45pub struct BackupResult {
46    pub dest: PathBuf,
47    pub files: usize,
48    pub bytes: u64,
49}
50
51#[derive(Debug, Clone, Default)]
52pub struct RestoreOptions {
53    /// How the writer lock on the directory being restored into is taken. As in [`BackupOptions`].
54    pub lock: LockMode,
55}
56
57#[derive(Debug, Clone, PartialEq, Eq)]
58pub struct RestoreResult {
59    pub dir: PathBuf,
60    pub files: usize,
61    pub bytes: u64,
62    /// Files of the database that was there before and were replaced.
63    pub replaced: usize,
64}
65
66pub(super) fn refuse(message: String) -> DatabaseError {
67    DatabaseError::Schema(HaysonSchemaError(message))
68}
69
70pub(super) fn read_catalog_of(layout: &Layout) -> Result<Catalog, DatabaseError> {
71    let path = layout.metadata_path();
72    let raw = fs::read_to_string(path).map_err(|error| {
73        if error.kind() == io::ErrorKind::NotFound {
74            refuse(format!(
75                "\"{}\" does not exist — there is no database to back up",
76                path.display()
77            ))
78        } else {
79            DatabaseError::Io(error)
80        }
81    })?;
82    let value: Value = serde_json::from_str(&raw)
83        .map_err(|error| DatabaseError::Catalog(format!("\"{}\" is not valid JSON: {error}", path.display())))?;
84    Ok(parse_catalog(&value)?)
85}
86
87fn relative_safely(root: &Path, path: &Path) -> Result<PathBuf, DatabaseError> {
88    let relative = path
89        .strip_prefix(root)
90        .map_err(|_| refuse(format!("\"{}\" is outside the database directory", path.display())))?;
91    check_relative(relative)?;
92    Ok(relative.to_path_buf())
93}
94
95/// A path a backup or a restore may join onto a root: relative, no `..`, no root, no prefix.
96pub(super) fn check_relative(path: &Path) -> Result<(), DatabaseError> {
97    let sane = path.components().all(|part| matches!(part, Component::Normal(_)));
98    if sane && path.components().next().is_some() {
99        Ok(())
100    } else {
101        Err(refuse(format!(
102            "\"{}\" is not a path inside the database directory",
103            path.display()
104        )))
105    }
106}
107
108fn files_in(dir: &Path) -> Result<Vec<PathBuf>, DatabaseError> {
109    let mut found = Vec::new();
110    let Ok(entries) = fs::read_dir(dir) else {
111        return Ok(found);
112    };
113    for entry in entries {
114        let entry = entry?;
115        let kind = entry.file_type()?;
116        if kind.is_symlink() {
117            return Err(refuse(format!(
118                "\"{}\" is a symbolic link — refusing to follow it",
119                entry.path().display()
120            )));
121        }
122        if kind.is_file() {
123            found.push(entry.path());
124        }
125    }
126    found.sort();
127    Ok(found)
128}
129
130/// Every file the database at `root` accounts for, as paths relative to `root`: its metadata
131/// document, the logs of its tables (one file, or one per partition), the files under each table's
132/// image directory and, when asked, the index snapshots. Anything else in the directory — another
133/// database in a subfolder, say — is not the database's and is left out.
134pub(super) fn owned_files(
135    root: &Path,
136    layout: &Layout,
137    catalog: &Catalog,
138    indexes: bool,
139) -> Result<Vec<PathBuf>, DatabaseError> {
140    let mut owned: BTreeSet<PathBuf> = BTreeSet::new();
141    let mut add = |path: &Path| -> Result<(), DatabaseError> {
142        owned.insert(relative_safely(root, path)?);
143        Ok(())
144    };
145    add(layout.metadata_path())?;
146    for (name, entry) in &catalog.tables {
147        let flat = layout.log_path(name);
148        match fs::symlink_metadata(&flat) {
149            Ok(meta) if meta.file_type().is_symlink() => {
150                return Err(refuse(format!(
151                    "\"{}\" is a symbolic link — refusing to follow it",
152                    flat.display()
153                )))
154            }
155            Ok(meta) if meta.is_file() => add(&flat)?,
156            _ => {}
157        }
158        let partitions = layout.tables().join(name);
159        for part in partition_names_in(&partitions) {
160            add(&partitions.join(format!("{part}{LOG_SUFFIX}")))?;
161        }
162        if indexes && layout.index_path(name).is_file() {
163            add(&layout.index_path(name))?;
164        }
165        let image_dir = entry
166            .options
167            .as_ref()
168            .and_then(|options| options.image_dir.clone())
169            .or_else(|| catalog.defaults.as_ref().and_then(|d| d.image_dir.clone()))
170            .map(|dir| root.join(dir))
171            .unwrap_or_else(|| layout.tables().join(format!("{name}.images")));
172        for file in files_in(&image_dir)? {
173            add(&file)?;
174        }
175    }
176    Ok(owned.into_iter().collect())
177}
178
179pub(super) fn hold(root: &Path, layout: &Layout, lock: &LockMode) -> Result<HeldLock, DatabaseError> {
180    let LockMode::Enabled(options) = lock else {
181        return Ok(HeldLock(None));
182    };
183    if root.is_dir() {
184        fs::create_dir_all(layout.runtime())?;
185    }
186    Ok(HeldLock(Some(DirLock::acquire_as(
187        layout.lock_path(),
188        options.clone(),
189        root.to_path_buf(),
190    )?)))
191}
192
193fn copy_synced(from: &Path, to: &Path) -> io::Result<u64> {
194    if let Some(parent) = to.parent() {
195        fs::create_dir_all(parent)?;
196    }
197    let bytes = fs::copy(from, to)?;
198    fs::File::open(to)?.sync_all()?;
199    Ok(bytes)
200}
201
202/// Copies a database directory into `dest`: the files the database accounts for (see
203/// [`owned_files`]) plus a `hayson-backup.json` manifest listing them with their sizes.
204///
205/// `dest` must not exist or must be an empty directory. The copy is built next to it and renamed
206/// into place, so a backup that fails halfway leaves no half-filled `dest`. The directory is held
207/// under the writer lock for the length of the copy unless `options.lock` is [`LockMode::Skip`];
208/// a database with an unfinished unit of work is refused, because its log and journal disagree
209/// until a writable open finishes it.
210///
211/// History kept in a database of its own is not part of this database's files, so it is not copied.
212pub fn backup_database(dir: &Path, dest: &Path, options: &BackupOptions) -> Result<BackupResult, DatabaseError> {
213    let layout = Database::resolve_layout(dir, &LayoutOptions::default())?;
214    let catalog = read_catalog_of(&layout)?;
215    if dest.exists()
216        && fs::read_dir(dest)
217            .map(|mut entries| entries.next().is_some())
218            .unwrap_or(true)
219    {
220        return Err(refuse(format!(
221            "\"{}\" already exists and is not an empty directory",
222            dest.display()
223        )));
224    }
225    let partial = sibling(dest, ".partial");
226    if partial.exists() {
227        fs::remove_dir_all(&partial)?;
228    }
229
230    let held = hold(dir, &layout, &options.lock)?;
231    let outcome = copy_database(dir, &layout, &catalog, &partial, options);
232    drop(held);
233    match outcome {
234        Ok(result) => {
235            if dest.exists() {
236                fs::remove_dir(dest)?;
237            }
238            fs::rename(&partial, dest)?;
239            Ok(BackupResult {
240                dest: dest.to_path_buf(),
241                ..result
242            })
243        }
244        Err(error) => {
245            let _ = fs::remove_dir_all(&partial);
246            Err(error)
247        }
248    }
249}
250
251pub(super) fn sibling(path: &Path, suffix: &str) -> PathBuf {
252    let mut name = path.file_name().map(|n| n.to_os_string()).unwrap_or_default();
253    name.push(suffix);
254    path.with_file_name(name)
255}
256
257fn copy_database(
258    root: &Path,
259    layout: &Layout,
260    catalog: &Catalog,
261    partial: &Path,
262    options: &BackupOptions,
263) -> Result<BackupResult, DatabaseError> {
264    if layout.journal_path().exists() {
265        return Err(refuse(format!(
266            "\"{}\" holds an unfinished unit of work — open the database for writing once so it is finished, then back it up",
267            layout.journal_path().display()
268        )));
269    }
270    let files = owned_files(root, layout, catalog, options.indexes)?;
271    fs::create_dir_all(partial)?;
272    let mut listed = Vec::new();
273    let mut total = 0u64;
274    for relative in &files {
275        let bytes = copy_synced(&root.join(relative), &partial.join(relative))?;
276        total += bytes;
277        listed.push(json!({ "path": relative.to_string_lossy().replace('\\', "/"), "bytes": bytes }));
278    }
279    let manifest = json!({
280        "format": BACKUP_FORMAT,
281        "createdAt": chrono::Utc::now().timestamp_millis(),
282        "engine": env!("CARGO_PKG_VERSION"),
283        "metadataFile": layout.metadata_file(),
284        "files": listed,
285    });
286    let mut text = serde_json::to_string_pretty(&manifest).map_err(|e| DatabaseError::Catalog(e.to_string()))?;
287    text.push('\n');
288    write_atomic(&partial.join(BACKUP_MANIFEST), text.as_bytes(), None)?;
289    Ok(BackupResult {
290        dest: partial.to_path_buf(),
291        files: files.len(),
292        bytes: total,
293    })
294}
295
296pub(super) struct ManifestEntry {
297    pub(super) relative: PathBuf,
298    pub(super) bytes: u64,
299}
300
301fn read_manifest(backup: &Path) -> Result<(String, Vec<ManifestEntry>), DatabaseError> {
302    let path = backup.join(BACKUP_MANIFEST);
303    let raw = fs::read_to_string(&path).map_err(|error| {
304        refuse(format!(
305            "\"{}\" is not a readable backup manifest: {error}",
306            path.display()
307        ))
308    })?;
309    parse_manifest(&raw, &path.display().to_string())
310}
311
312pub(super) fn parse_manifest(raw: &str, label: &str) -> Result<(String, Vec<ManifestEntry>), DatabaseError> {
313    let manifest: Value =
314        serde_json::from_str(raw).map_err(|error| refuse(format!("\"{label}\" is not valid JSON: {error}")))?;
315    if manifest.get("format").and_then(Value::as_u64) != Some(BACKUP_FORMAT) {
316        return Err(refuse(format!("\"{label}\" is not a backup format this engine reads")));
317    }
318    let metadata_file = manifest
319        .get("metadataFile")
320        .and_then(Value::as_str)
321        .ok_or_else(|| refuse("backup manifest names no \"metadataFile\"".to_string()))?
322        .to_string();
323    let listed = manifest
324        .get("files")
325        .and_then(Value::as_array)
326        .ok_or_else(|| refuse("backup manifest lists no \"files\"".to_string()))?;
327    let mut entries = Vec::new();
328    for item in listed {
329        let relative = PathBuf::from(
330            item.get("path")
331                .and_then(Value::as_str)
332                .ok_or_else(|| refuse("a backup manifest entry has no \"path\"".to_string()))?,
333        );
334        check_relative(&relative)?;
335        let bytes = item
336            .get("bytes")
337            .and_then(Value::as_u64)
338            .ok_or_else(|| refuse(format!("manifest entry \"{}\" has no \"bytes\"", relative.display())))?;
339        entries.push(ManifestEntry { relative, bytes });
340    }
341    Ok((metadata_file, entries))
342}
343
344/// Restores a backup made by [`backup_database`] into `dir`, replacing the database that is there.
345///
346/// Recoverable: nothing is deleted until the replacement is in place. The backup is copied into
347/// `<dir>/.restore-new` and checked (sizes against the manifest, then [`verify`] with no error);
348/// the files of the database being replaced are moved into `<dir>/.restore-old`; the new files are
349/// moved in; and only then is `.restore-old` removed. If a move fails, the files already moved are
350/// put back, and when even that fails both errors are reported. A `.restore-old` left by an
351/// interrupted restore is refused, because it may hold the only good copy.
352///
353/// Only the files the database accounts for are replaced, so another database living in a
354/// subfolder of `dir` is untouched.
355pub fn restore_database(backup: &Path, dir: &Path, options: &RestoreOptions) -> Result<RestoreResult, DatabaseError> {
356    let (metadata_file, entries) = read_manifest(backup)?;
357    for entry in &entries {
358        let path = backup.join(&entry.relative);
359        let meta = fs::symlink_metadata(&path).map_err(|error| {
360            refuse(format!(
361                "backup file \"{}\" is missing: {error}",
362                entry.relative.display()
363            ))
364        })?;
365        if !meta.is_file() || meta.len() != entry.bytes {
366            return Err(refuse(format!(
367                "backup file \"{}\" does not match its manifest entry ({} bytes expected)",
368                entry.relative.display(),
369                entry.bytes
370            )));
371        }
372    }
373    restore_staged(dir, &metadata_file, &entries, &options.lock, |staged| {
374        let mut total = 0u64;
375        for entry in &entries {
376            total += copy_synced(&backup.join(&entry.relative), &staged.join(&entry.relative))?;
377        }
378        Ok(total)
379    })
380}
381
382/// The recoverable half of a restore, shared with unpacking: `stage` fills the given staging
383/// directory with the `entries` (returning their byte total) while the writer lock is held.
384pub(super) fn restore_staged(
385    dir: &Path,
386    metadata_file: &str,
387    entries: &[ManifestEntry],
388    lock: &LockMode,
389    stage: impl FnOnce(&Path) -> Result<u64, DatabaseError>,
390) -> Result<RestoreResult, DatabaseError> {
391    let old = dir.join(RESTORE_OLD);
392    if old.exists() {
393        return Err(refuse(format!(
394            "\"{}\" exists — an earlier restore was interrupted and it may hold the only good copy of the database; move its files back (or delete it) before restoring again",
395            old.display()
396        )));
397    }
398    fs::create_dir_all(dir)?;
399    let layout = Database::resolve_layout(
400        dir,
401        &LayoutOptions {
402            metadata_file: Some(metadata_file.to_string()),
403            ..LayoutOptions::default()
404        },
405    )?;
406
407    let held = hold(dir, &layout, lock)?;
408    let staged = dir.join(RESTORE_NEW);
409    let outcome = install(dir, &layout, entries, &staged, &old, stage);
410    let _ = fs::remove_dir_all(&staged);
411    drop(held);
412    outcome
413}
414
415fn install(
416    root: &Path,
417    layout: &Layout,
418    entries: &[ManifestEntry],
419    staged: &Path,
420    old: &Path,
421    stage: impl FnOnce(&Path) -> Result<u64, DatabaseError>,
422) -> Result<RestoreResult, DatabaseError> {
423    if staged.exists() {
424        fs::remove_dir_all(staged)?;
425    }
426    let total = stage(staged)?;
427    let report = verify(
428        staged,
429        &VerifyOptions {
430            relations: false,
431            ..VerifyOptions::default()
432        },
433    );
434    if let Some(problem) = report
435        .problems
436        .iter()
437        .find(|p| p.severity == super::verify::Severity::Error)
438    {
439        return Err(refuse(format!(
440            "the backup does not verify ({}: {}) — nothing was changed",
441            problem.code, problem.message
442        )));
443    }
444
445    // The files of the database now there: what its catalog accounts for (when it can be read), plus
446    // whatever the backup is about to overwrite.
447    let mut replaced: BTreeSet<PathBuf> = entries.iter().map(|e| e.relative.clone()).collect();
448    if let Ok(catalog) = read_catalog_of(layout) {
449        if let Ok(current) = owned_files(root, layout, &catalog, true) {
450            replaced.extend(current);
451        }
452    }
453    replaced.retain(|relative| root.join(relative).is_file());
454
455    fs::create_dir_all(old)?;
456    let mut moved_out: Vec<PathBuf> = Vec::new();
457    let mut moved_in: Vec<PathBuf> = Vec::new();
458    let swap = (|| -> io::Result<()> {
459        for relative in &replaced {
460            move_file(&root.join(relative), &old.join(relative))?;
461            moved_out.push(relative.clone());
462        }
463        for entry in entries {
464            move_file(&staged.join(&entry.relative), &root.join(&entry.relative))?;
465            moved_in.push(entry.relative.clone());
466        }
467        Ok(())
468    })();
469    match swap {
470        Ok(()) => {
471            fs::remove_dir_all(old)?;
472            Ok(RestoreResult {
473                dir: root.to_path_buf(),
474                files: entries.len(),
475                bytes: total,
476                replaced: replaced.len(),
477            })
478        }
479        Err(error) => {
480            let rollback = (|| -> io::Result<()> {
481                for relative in &moved_in {
482                    fs::remove_file(root.join(relative))?;
483                }
484                for relative in &moved_out {
485                    move_file(&old.join(relative), &root.join(relative))?;
486                }
487                Ok(())
488            })();
489            match rollback {
490                Ok(()) => {
491                    let _ = fs::remove_dir_all(old);
492                    Err(DatabaseError::Io(error))
493                }
494                Err(rollback_error) => Err(refuse(format!(
495                    "restore failed ({error}) and putting the previous files back failed too ({rollback_error}) — the previous database is in \"{}\"",
496                    old.display()
497                ))),
498            }
499        }
500    }
501}
502
503fn move_file(from: &Path, to: &Path) -> io::Result<()> {
504    if let Some(parent) = to.parent() {
505        fs::create_dir_all(parent)?;
506    }
507    fs::rename(from, to)
508}
hayson_engine/database/backup.rs

hayson_engine/database/
backup.rs
