Skip to main content

second_brain_api/
backup.rs

1use std::fs;
2use std::path::{Path, PathBuf};
3use std::sync::mpsc::{Sender, channel};
4use std::time::{Duration, Instant};
5
6use anyhow::{Context, Result, anyhow, bail, ensure};
7use serde::Serialize;
8
9use second_brain_core::cold;
10use second_brain_core::dir_copy::{
11    copy_dir_stable, copy_file_synced, copy_symlink, is_not_found, sync_dir,
12};
13use second_brain_core::index::raw;
14use second_brain_core::keys::{
15    self, DeviceKeys, WRAPPED_STORE_KEY_FILE, WrapRefresh, every_held_epoch, left_behind,
16    unwrap_store_key_with_epochs,
17};
18use second_brain_core::private_files::create_private_dir;
19use second_brain_core::sqlite_store::encrypt_migration::key_opens_file;
20use second_brain_core::sqlite_store::{AtRestPurpose, SqliteStore};
21use second_brain_core::transcript_archive::TranscriptArchive;
22
23use crate::maintenance::{DrainRequest, PauseCommand};
24use crate::write_gate::WriteGate;
25
26const DB_FILE: &str = "brain.db";
27const TRANSCRIPTS_DIR: &str = "transcripts";
28const MODEL_CACHE_DIR: &str = "fastembed";
29const STALE_MARKER: &str = ".stale-";
30const COLD_RETRY: Duration = Duration::from_secs(5);
31const COLD_WAIT: Duration = Duration::from_secs(30 * 60);
32
33/// What the backup reads: the live store and the threads that write beside
34/// it, each held only while its part is copied.
35pub struct Backup<'a> {
36    pub store: &'a SqliteStore,
37    pub data_dir: &'a Path,
38    pub gate: &'a WriteGate,
39    /// The index drain thread; None when the indexes never opened, and then
40    /// the backup carries no index for the restored daemon to reuse.
41    pub drain: Option<&'a Sender<DrainRequest>>,
42    pub archive: Option<&'a TranscriptArchive>,
43}
44
45#[derive(Debug, Serialize)]
46pub struct BackupSummary {
47    pub path: PathBuf,
48    pub memory_count: u64,
49    pub stored_total: u64,
50    pub bytes: u64,
51    pub outbox_settled: bool,
52    pub indexes: &'static str,
53    pub raw_index: &'static str,
54}
55
56#[derive(Debug)]
57pub enum BackupError {
58    BadDestination(String),
59    Refused(String),
60    Failed(anyhow::Error),
61}
62
63impl std::fmt::Display for BackupError {
64    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
65        match self {
66            Self::BadDestination(m) | Self::Refused(m) => f.write_str(m),
67            Self::Failed(e) => write!(f, "{e:#}"),
68        }
69    }
70}
71
72impl From<anyhow::Error> for BackupError {
73    fn from(e: anyhow::Error) -> Self {
74        Self::Failed(e)
75    }
76}
77
78struct Live {
79    db: PathBuf,
80    indexes: PathBuf,
81    raw_index: PathBuf,
82    cold: PathBuf,
83}
84
85impl Live {
86    fn of(db: PathBuf) -> Self {
87        Self {
88            indexes: db.with_extension("indexes"),
89            raw_index: raw::dir_for_db(&db),
90            cold: cold::dir_for_db(&db),
91            db,
92        }
93    }
94
95    fn handled(&self, name: &str) -> bool {
96        let db = self
97            .db
98            .file_name()
99            .and_then(|n| n.to_str())
100            .unwrap_or(DB_FILE);
101        [&self.indexes, &self.raw_index, &self.cold]
102            .iter()
103            .any(|dir| dir.file_name().and_then(|n| n.to_str()) == Some(name))
104            || name == db
105            || name == format!("{db}-wal")
106            || name == TRANSCRIPTS_DIR
107            || name == WRAPPED_STORE_KEY_FILE
108    }
109}
110
111impl Backup<'_> {
112    /// Writes a copy of the data dir to `dest` that `sb restore` accepts,
113    /// with the store, its indexes, its cold segments and its wrapped key
114    /// all taken at one moment. `dest` appears only once complete.
115    pub fn write(&self, dest: &Path) -> Result<BackupSummary, BackupError> {
116        let _maintenance = self.gate.hold_maintenance();
117        let live = Live::of(self.db_path()?);
118        let (parent, name) = self.check_destination(dest)?;
119        let dest = parent.join(&name);
120        remove_stale_partials(&parent, &name);
121        let partial = parent.join(format!(".{name}.partial-{}", uuid::Uuid::new_v4().simple()));
122        create_private_dir(&partial)?;
123        let published = self.fill(&partial, &live).and_then(|summary| {
124            sync_dir(&partial)?;
125            fs::rename(&partial, &dest)
126                .with_context(|| format!("renaming {} to {}", partial.display(), dest.display()))?;
127            Ok(summary)
128        });
129        match published {
130            Ok(summary) => {
131                if let Err(e) = sync_dir(&parent) {
132                    tracing::warn!("the backup is in place but not yet flushed to disk: {e:#}");
133                }
134                Ok(BackupSummary {
135                    path: dest,
136                    ..summary
137                })
138            }
139            Err(e) => {
140                if let Err(cleanup) = fs::remove_dir_all(&partial) {
141                    tracing::warn!(
142                        "cannot remove the partial backup {}: {cleanup}",
143                        partial.display()
144                    );
145                }
146                Err(e)
147            }
148        }
149    }
150
151    fn db_path(&self) -> Result<PathBuf, BackupError> {
152        let db = self.store.file_path()?.ok_or_else(|| {
153            BackupError::Refused("an in-memory store has no file to back up".into())
154        })?;
155        let dir = db.parent().map(Path::to_path_buf).unwrap_or_default();
156        if fs::canonicalize(&dir).ok() != fs::canonicalize(self.data_dir).ok() {
157            return Err(BackupError::Refused(format!(
158                "the store {} is outside the data dir {}, so restore would not find it",
159                db.display(),
160                self.data_dir.display()
161            )));
162        }
163        Ok(db)
164    }
165
166    fn check_destination(&self, dest: &Path) -> Result<(PathBuf, String), BackupError> {
167        let bad = |m: String| Err(BackupError::BadDestination(m));
168        if !dest.is_absolute() {
169            return bad(format!("{} is not an absolute path", dest.display()));
170        }
171        let (Some(parent), Some(name)) = (dest.parent(), dest.file_name()) else {
172            return bad(format!("{} names no directory", dest.display()));
173        };
174        let parent = fs::canonicalize(parent)
175            .map_err(|e| BackupError::BadDestination(format!("{}: {e}", parent.display())))?;
176        let data_dir = fs::canonicalize(self.data_dir).context("reading the data dir")?;
177        if parent.starts_with(&data_dir) {
178            return bad(format!(
179                "{} is inside the data dir {}",
180                dest.display(),
181                data_dir.display()
182            ));
183        }
184        if fs::symlink_metadata(parent.join(name)).is_ok() {
185            return Err(BackupError::Refused(format!(
186                "{} already exists; a backup never writes over one",
187                dest.display()
188            )));
189        }
190        Ok((parent, name.to_string_lossy().into_owned()))
191    }
192
193    fn fill(&self, partial: &Path, live: &Live) -> Result<BackupSummary, BackupError> {
194        let device = self.fix_keys(partial)?;
195        let copy_db = partial.join(DB_FILE);
196        let indexes = self.snapshot(&copy_db, live)?;
197        let checked = self.store.check_backup_copy(&copy_db)?;
198        if let Some(device) = &device {
199            self.check_wrapped_key(device, partial, &copy_db)?;
200        }
201        let raw_index = if live.raw_index.is_dir()
202            && copy_dir_stable(&live.raw_index, &raw::dir_for_db(&copy_db))?
203        {
204            "copied"
205        } else {
206            "rebuilt at first start"
207        };
208        self.copy_transcripts(&partial.join(TRANSCRIPTS_DIR))?;
209        self.copy_rest(partial, live)?;
210        Ok(BackupSummary {
211            path: partial.to_path_buf(),
212            memory_count: checked.counts.live,
213            stored_total: checked.counts.stored_total,
214            bytes: tree_bytes(partial)?,
215            outbox_settled: checked.outbox_settled,
216            indexes,
217            raw_index,
218        })
219    }
220
221    /// Settles every key the copy depends on before it is taken: the wrapped
222    /// store key restore opens, and the at-rest keys that seal the index and
223    /// transcript files, which the copied store must already hold.
224    fn fix_keys(&self, partial: &Path) -> Result<Option<DeviceKeys>, BackupError> {
225        if !self.store.is_encrypted() {
226            return Ok(None);
227        }
228        let device = keys::load_device_keys(self.data_dir)?.ok_or_else(|| {
229            BackupError::Refused("this device has no device key to wrap the store key with".into())
230        })?;
231        if let WrapRefresh::NoIdentity =
232            keys::refresh_store_key_wrap(self.store, self.data_dir, &device)?
233        {
234            return Err(BackupError::Refused(
235                "this device has no sync identity, so no key could open its backup; \
236                 pair it with `sb pair` first"
237                    .into(),
238            ));
239        }
240        for purpose in [
241            AtRestPurpose::Indexes,
242            AtRestPurpose::RawIndex,
243            AtRestPurpose::TranscriptArchive,
244        ] {
245            self.store.at_rest_keys(purpose)?;
246        }
247        copy_file_synced(
248            &self.data_dir.join(WRAPPED_STORE_KEY_FILE),
249            &partial.join(WRAPPED_STORE_KEY_FILE),
250        )?;
251        Ok(Some(device))
252    }
253
254    /// Takes the store copy while the drain thread and cold passes are held,
255    /// then copies the files they write, so both match the copy. Lock order:
256    /// maintenance, drain pause, cold guard; the drain thread's purge takes
257    /// the cold guard, so it must be paused first.
258    fn snapshot(&self, copy_db: &Path, live: &Live) -> Result<&'static str> {
259        let deadline = Instant::now() + COLD_WAIT;
260        loop {
261            let paused = self.drain.map(Paused::start).transpose()?;
262            let guard = match self.store.cold_tier() {
263                None => None,
264                Some(tier) => match tier.try_hold_passes() {
265                    Some(guard) => Some(guard),
266                    None => {
267                        drop(paused);
268                        ensure!(
269                            Instant::now() < deadline,
270                            "a cold pass held the cold tier for 30 minutes; run the backup again"
271                        );
272                        std::thread::sleep(COLD_RETRY);
273                        continue;
274                    }
275                },
276            };
277            if let Some(paused) = &paused {
278                paused.settle()?;
279            }
280            self.store.vacuum_into(copy_db)?;
281            let indexes = match &paused {
282                None => "unavailable",
283                Some(_) if !live.indexes.is_dir() => "unavailable",
284                Some(_) => {
285                    if copy_dir_stable(&live.indexes, &copy_db.with_extension("indexes"))? {
286                        "copied"
287                    } else {
288                        "rebuilt at first start"
289                    }
290                }
291            };
292            drop(paused);
293            if live.cold.is_dir() {
294                ensure!(
295                    copy_dir_stable(&live.cold, &cold::dir_for_db(copy_db))?,
296                    "the cold segments at {} changed while the cold tier was held",
297                    live.cold.display()
298                );
299            }
300            drop(guard);
301            return Ok(indexes);
302        }
303    }
304
305    /// Unwraps the copied store key the way `sb restore --own-keys` does
306    /// and proves it opens the copied store.
307    fn check_wrapped_key(&self, device: &DeviceKeys, partial: &Path, copy_db: &Path) -> Result<()> {
308        let held = every_held_epoch(self.store, device)?;
309        let key = unwrap_store_key_with_epochs(partial, &held)?
310            .ok_or_else(|| anyhow!("the backup holds no {WRAPPED_STORE_KEY_FILE}"))?;
311        ensure!(
312            key_opens_file(copy_db, &key),
313            "the copied {WRAPPED_STORE_KEY_FILE} does not open the copied store"
314        );
315        Ok(())
316    }
317
318    fn copy_transcripts(&self, dest: &Path) -> Result<()> {
319        match self.archive {
320            Some(archive) => archive.copy_into(dest),
321            None => {
322                let live = self.data_dir.join(TRANSCRIPTS_DIR);
323                if live.is_dir() && !copy_dir_stable(&live, dest)? {
324                    bail!("the transcript archive at {} kept changing", live.display());
325                }
326                Ok(())
327            }
328        }
329    }
330
331    /// Every other top-level entry restore would carry over.
332    fn copy_rest(&self, partial: &Path, live: &Live) -> Result<()> {
333        for entry in fs::read_dir(self.data_dir)
334            .with_context(|| format!("reading {}", self.data_dir.display()))?
335        {
336            let entry = entry?;
337            let Some(name) = entry.file_name().to_str().map(str::to_owned) else {
338                continue;
339            };
340            if left_behind(&name)
341                || live.handled(&name)
342                || name == MODEL_CACHE_DIR
343                || name.contains(STALE_MARKER)
344            {
345                continue;
346            }
347            let (source, target) = (entry.path(), partial.join(&name));
348            let kind = match entry.file_type() {
349                Ok(kind) => kind,
350                Err(e) if e.kind() == std::io::ErrorKind::NotFound => continue,
351                Err(e) => return Err(e).with_context(|| format!("reading {}", source.display())),
352            };
353            if kind.is_dir() {
354                ensure!(
355                    copy_dir_stable(&source, &target)? || !source.exists(),
356                    "{} kept changing",
357                    source.display()
358                );
359            } else {
360                let copied = if kind.is_symlink() {
361                    copy_symlink(&source, &target)
362                } else {
363                    copy_file_synced(&source, &target)
364                };
365                match copied {
366                    Err(e) if is_not_found(&e) => {}
367                    other => other?,
368                }
369            }
370        }
371        Ok(())
372    }
373}
374
375/// The drain thread held between `start` and drop.
376struct Paused(Sender<PauseCommand>);
377
378impl Paused {
379    fn start(drain: &Sender<DrainRequest>) -> Result<Self> {
380        let (commands, held) = channel();
381        drain
382            .send(DrainRequest::Pause(held))
383            .map_err(|_| anyhow!("the index drain thread is not running"))?;
384        let paused = Self(commands);
385        paused.settle()?;
386        Ok(paused)
387    }
388
389    fn settle(&self) -> Result<()> {
390        let (reply, done) = channel();
391        self.0
392            .send(PauseCommand::Settle(reply))
393            .map_err(|_| anyhow!("the index drain thread stopped"))?;
394        done.recv()
395            .map_err(|_| anyhow!("the index drain thread stopped"))?
396    }
397}
398
399/// No other backup runs while the maintenance mutex is held, so any
400/// partial copy beside `dest` is from one that was killed.
401fn remove_stale_partials(parent: &Path, name: &str) {
402    let prefix = format!(".{name}.partial-");
403    let entries = match fs::read_dir(parent) {
404        Ok(entries) => entries,
405        Err(e) => {
406            tracing::warn!(
407                "cannot look for partial backups in {}: {e}",
408                parent.display()
409            );
410            return;
411        }
412    };
413    for entry in entries {
414        let entry = match entry {
415            Ok(entry) => entry,
416            Err(e) => {
417                tracing::warn!(
418                    "cannot look for partial backups in {}: {e}",
419                    parent.display()
420                );
421                continue;
422            }
423        };
424        if entry.file_name().to_string_lossy().starts_with(&prefix)
425            && let Err(e) = fs::remove_dir_all(entry.path())
426        {
427            tracing::warn!(
428                "cannot remove the partial backup {}: {e}",
429                entry.path().display()
430            );
431        }
432    }
433}
434
435fn tree_bytes(dir: &Path) -> Result<u64> {
436    let mut total = 0;
437    for entry in fs::read_dir(dir).with_context(|| format!("reading {}", dir.display()))? {
438        let entry = entry?;
439        let kind = entry.file_type()?;
440        if kind.is_dir() {
441            total += tree_bytes(&entry.path())?;
442        } else if kind.is_file() {
443            total += entry.metadata()?.len();
444        }
445    }
446    Ok(total)
447}