Skip to main content

second_brain_api/
snapshot_merge.rs

1use std::collections::BTreeSet;
2use std::fs;
3use std::io::Write;
4use std::path::{Path, PathBuf};
5use std::sync::{Arc, Mutex};
6use std::time::Duration;
7
8use anyhow::{Context, Result, anyhow, bail};
9use chrono::Utc;
10use second_brain_core::sqlite_store::{MergeOutcome, SnapshotOptions, SqliteStore, replay_build};
11use second_brain_sync::keyring::{IdentityMismatch, Keyring};
12use second_brain_sync::protocol::{Hello, Merging, SYNC_PROTOCOL_VERSION};
13use second_brain_sync::snapshot::{
14    SNAPSHOT_FORMAT, SnapshotHeader, SnapshotKey, SnapshotWriter, spool, spool_rows, verify,
15};
16use serde::{Deserialize, Serialize};
17use tokio::task::JoinHandle;
18
19use crate::health::Health;
20use crate::write_gate::WriteGate;
21
22const SPOOL: &str = "snapshot.spool";
23const SPOOL_INFO: &str = "snapshot.spool.json";
24const SPOOL_FAILING: &str = "snapshot.spool.failing.json";
25
26/// A merge that has failed for this long is dropped, so the exporter sends
27/// a fresh snapshot instead of the device waiting on one that never lands.
28const ABANDON_AFTER_MICROS: i64 = 24 * 3600 * 1_000_000;
29
30/// How often the daemon looks for a spooled snapshot whose merge has not
31/// finished: one interrupted by a restart, a closed clock gate or store
32/// maintenance.
33const RESUME_INTERVAL: Duration = Duration::from_secs(60);
34
35/// A second snapshot is refused while one is being received, or is
36/// spooled and unmerged, so the pusher waits instead of sending it again.
37#[derive(Debug)]
38pub struct MergeRunning(pub Option<Merging>);
39
40impl std::fmt::Display for MergeRunning {
41    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
42        match &self.0 {
43            Some(merging) => write!(
44                f,
45                "a snapshot merge from {} through seq {} has not finished",
46                merging.exporter, merging.seq
47            ),
48            None => write!(f, "another snapshot is being received"),
49        }
50    }
51}
52
53impl std::error::Error for MergeRunning {}
54
55/// Data dirs whose snapshot is being received or merged now: one per store
56/// at a time, so the spool never changes under a merge, and the two
57/// daemons of an in-process test each get their own.
58static RUNNING: Mutex<BTreeSet<PathBuf>> = Mutex::new(BTreeSet::new());
59
60struct Running(PathBuf);
61
62impl Running {
63    fn claim(data_dir: &Path) -> Option<Self> {
64        let mut running = RUNNING.lock().unwrap_or_else(|e| e.into_inner());
65        running
66            .insert(data_dir.to_path_buf())
67            .then(|| Self(data_dir.to_path_buf()))
68    }
69}
70
71impl Drop for Running {
72    fn drop(&mut self) {
73        RUNNING
74            .lock()
75            .unwrap_or_else(|e| e.into_inner())
76            .remove(&self.0);
77    }
78}
79
80/// Streams a snapshot of this store, sealed for `peer`, into `out`, after
81/// `pin_for_snapshot`.
82pub fn write_snapshot<W: Write>(
83    sqlite: &SqliteStore,
84    data_dir: &Path,
85    gate: &WriteGate,
86    peer: &Hello,
87    out: W,
88) -> Result<()> {
89    pin_for_snapshot(sqlite, gate, peer)?;
90    stream_snapshot(sqlite, data_dir, peer, out)
91}
92
93/// Pins `peer` at the head, so the floor holds every row above the seq of
94/// the snapshot that follows until its follow-up pull or push acks it.
95pub fn pin_for_snapshot(sqlite: &SqliteStore, gate: &WriteGate, peer: &Hello) -> Result<()> {
96    let _slot = gate.try_enter().ok_or_else(|| {
97        anyhow!("store maintenance is running; the snapshot is sent on the next sync")
98    })?;
99    sqlite.pin_peer(&peer.machine_id, sqlite.log_head()?)
100}
101
102/// The stream `write_snapshot` sends, for a caller that already pinned.
103pub fn stream_snapshot<W: Write>(
104    sqlite: &SqliteStore,
105    data_dir: &Path,
106    peer: &Hello,
107    out: W,
108) -> Result<()> {
109    let keyring = Keyring::load(sqlite, data_dir)?;
110    let declared = peer.keys.as_ref().map(|k| k.epochs.as_slice());
111    let key = keyring.snapshot_key_for(&peer.machine_id, declared)?;
112    let keys = keyring.handshake_for(&peer.machine_id, declared)?;
113    let header = SnapshotHeader {
114        format: SNAPSHOT_FORMAT,
115        snapshot_id: uuid::Uuid::new_v4().to_string(),
116        exporter: sqlite.machine_id().to_string(),
117        epoch: *key.epoch(),
118        sync_protocol: SYNC_PROTOCOL_VERSION,
119        schema_version: sqlite.schema_version()?,
120        created_at: Utc::now().timestamp_micros(),
121    };
122    let mut writer = SnapshotWriter::new(out, key, header)?;
123    let state = sqlite.export_snapshot(SnapshotOptions { with_chunks: true }, &mut |row| {
124        writer.row(&row)
125    })?;
126    writer.finish(state, Some(keys))?;
127    Ok(())
128}
129
130/// The merge spooled here and not yet finished.
131pub fn merging(data_dir: &Path) -> Result<Option<Merging>> {
132    match fs::read(data_dir.join(SPOOL_INFO)) {
133        Ok(bytes) => Ok(Some(
134            serde_json::from_slice(&bytes).context("reading the spooled snapshot's info")?,
135        )),
136        Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
137        Err(e) => Err(e.into()),
138    }
139}
140
141fn key_for<'a>(keyring: &'a Keyring<'a>) -> impl Fn(&SnapshotHeader) -> Result<SnapshotKey> + 'a {
142    move |header| keyring.snapshot_opening_key(&header.exporter, &header.epoch)
143}
144
145/// Spools a snapshot stream into the data dir and checks it against its
146/// manifest, and against `exporter` when the caller knows who sent it. The
147/// merge itself runs after, from the spool, so a cut or restarted merge
148/// resumes without a second transfer.
149pub fn receive<R: std::io::Read>(
150    sqlite: &SqliteStore,
151    data_dir: &Path,
152    input: R,
153    exporter: Option<&str>,
154) -> Result<Merging> {
155    let Some(_running) = Running::claim(data_dir) else {
156        return Err(MergeRunning(merging(data_dir)?).into());
157    };
158    if let Some(running) = merging(data_dir)? {
159        return Err(MergeRunning(Some(running)).into());
160    }
161    let keyring = Keyring::load(sqlite, data_dir)?;
162    let spool_path = data_dir.join(SPOOL);
163    let (header, manifest) = spool(input, &spool_path, &key_for(&keyring))?;
164    if let Some(expected) = exporter
165        && header.exporter != expected
166    {
167        remove_if_present(&spool_path)?;
168        bail!(
169            "the snapshot from {expected} was exported by {}; nothing was applied",
170            header.exporter
171        );
172    }
173    let merging = Merging {
174        snapshot_id: header.snapshot_id,
175        exporter: header.exporter,
176        seq: manifest.state.seq,
177    };
178    remove_if_present(&data_dir.join(SPOOL_FAILING))?;
179    write_atomically(&data_dir.join(SPOOL_INFO), &serde_json::to_vec(&merging)?)?;
180    Ok(merging)
181}
182
183/// Merges the spooled snapshot, if any, and clears the spool once the
184/// cursors are raised. None when there is nothing to merge, another merge
185/// or store maintenance holds the store, or maintenance closed the gate
186/// mid-merge, so a later call resumes. A spool that will not open, or one
187/// whose merge has failed for a day, is dropped, so the exporter sends
188/// another instead of this device waiting on it.
189pub fn merge_spooled(
190    sqlite: &SqliteStore,
191    data_dir: &Path,
192    gate: &WriteGate,
193) -> Result<Option<MergeOutcome>> {
194    let Some(_running) = Running::claim(data_dir) else {
195        return Ok(None);
196    };
197    let Some(merging) = merging(data_dir)? else {
198        return Ok(None);
199    };
200    let Some(_slot) = gate.try_enter() else {
201        return Ok(None);
202    };
203    match merge_claimed(sqlite, data_dir, gate, &merging) {
204        Ok(Step::Done(outcome)) => {
205            clear_spool(data_dir)?;
206            tracing::info!(
207                exporter = %merging.exporter,
208                seq = merging.seq,
209                rows = outcome.rows,
210                changed = outcome.changed,
211                settled = outcome.settled,
212                "snapshot merged"
213            );
214            Ok(Some(outcome))
215        }
216        Ok(Step::Paused { progressed }) => {
217            if progressed {
218                remove_if_present(&data_dir.join(SPOOL_FAILING))?;
219            }
220            Ok(None)
221        }
222        Err(Failure::Spool(e)) => {
223            abandon(sqlite, data_dir)?;
224            Err(e.context(format!(
225                "the snapshot from {} will not open and was dropped; the next sync gets another",
226                merging.exporter
227            )))
228        }
229        Err(Failure::Merge {
230            error: e,
231            progressed,
232        }) => {
233            let now = Utc::now().timestamp_micros();
234            let failing = note_failure(data_dir, &e, progressed, now)?;
235            if now - failing.since >= ABANDON_AFTER_MICROS {
236                abandon(sqlite, data_dir)?;
237                return Err(e.context(format!(
238                    "the snapshot from {} failed to merge for a day and was dropped; the next \
239                     sync gets another",
240                    merging.exporter
241                )));
242            }
243            Err(e)
244        }
245    }
246}
247
248/// `progressed` says whether the attempt applied rows: only then does a
249/// failure or pause start a new day before the merge is dropped.
250enum Step {
251    Done(MergeOutcome),
252    Paused { progressed: bool },
253}
254
255/// `Spool` is a fault of the spooled snapshot itself, which no retry
256/// fixes; `Merge` is one of this store, which may clear.
257enum Failure {
258    Spool(anyhow::Error),
259    Merge {
260        error: anyhow::Error,
261        progressed: bool,
262    },
263}
264
265impl Failure {
266    fn merge(error: anyhow::Error) -> Self {
267        Failure::Merge {
268            error,
269            progressed: false,
270        }
271    }
272}
273
274fn identity_or_merge(e: anyhow::Error) -> Failure {
275    if e.downcast_ref::<IdentityMismatch>().is_some() {
276        Failure::Spool(e)
277    } else {
278        Failure::merge(e)
279    }
280}
281
282fn merge_claimed(
283    sqlite: &SqliteStore,
284    data_dir: &Path,
285    gate: &WriteGate,
286    merging: &Merging,
287) -> Result<Step, Failure> {
288    let spool_path = data_dir.join(SPOOL);
289    let keyring = Keyring::load(sqlite, data_dir).map_err(Failure::merge)?;
290    let key_for = key_for(&keyring);
291    let (header, manifest) = verify(&spool_path, &key_for).map_err(Failure::Spool)?;
292    if header.snapshot_id != merging.snapshot_id {
293        return Err(Failure::Spool(anyhow!(
294            "the spool holds snapshot {}, not {}",
295            header.snapshot_id,
296            merging.snapshot_id
297        )));
298    }
299    if let Some(keys) = &manifest.keys {
300        keyring
301            .check_peer(keys, &header.exporter)
302            .map_err(identity_or_merge)?;
303    }
304    keyring
305        .opened_snapshot(&header.exporter, &header.epoch)
306        .map_err(identity_or_merge)?;
307    let mut merge = sqlite
308        .begin_snapshot_merge(&header.snapshot_id, &header.exporter, merging.seq)
309        .map_err(Failure::merge)?;
310    let rows_before = merge.progress().rows_done;
311    let (mut push_failed, mut paused) = (false, false);
312    let streamed = spool_rows(&spool_path, &key_for, &mut |row| {
313        if let Err(e) = merge.push(row) {
314            push_failed = true;
315            return Err(e);
316        }
317        if gate.is_closed() {
318            paused = true;
319            bail!("store maintenance closed the write gate");
320        }
321        Ok(())
322    });
323    let progressed = merge.progress().rows_done > rows_before;
324    match streamed {
325        Ok(()) => {}
326        Err(_) if paused => return Ok(Step::Paused { progressed }),
327        Err(error) if push_failed => return Err(Failure::Merge { error, progressed }),
328        Err(e) => return Err(Failure::Spool(e)),
329    }
330    if let Some(keys) = &manifest.keys {
331        keyring
332            .accept(&header.exporter, keys, Utc::now().timestamp_micros())
333            .map_err(identity_or_merge)?;
334    }
335    let outcome = merge
336        .finish(&manifest.state.cursors)
337        .map_err(Failure::merge)?;
338    Ok(Step::Done(outcome))
339}
340
341/// How long the spooled merge has been failing, kept beside the spool so a
342/// restart does not reset the day it gets before it is dropped.
343#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
344pub struct MergeFailing {
345    pub since: i64,
346    pub attempts: u32,
347    pub error: String,
348}
349
350/// The spooled merge, for `/health`.
351#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
352pub struct MergeStatus {
353    pub merging: Merging,
354    pub failing: Option<MergeFailing>,
355}
356
357pub fn status(data_dir: &Path) -> Result<Option<MergeStatus>> {
358    let Some(merging) = merging(data_dir)? else {
359        return Ok(None);
360    };
361    Ok(Some(MergeStatus {
362        merging,
363        failing: failing(data_dir)?,
364    }))
365}
366
367fn failing(data_dir: &Path) -> Result<Option<MergeFailing>> {
368    match fs::read(data_dir.join(SPOOL_FAILING)) {
369        Ok(bytes) => Ok(Some(
370            serde_json::from_slice(&bytes).context("reading the snapshot merge's failures")?,
371        )),
372        Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
373        Err(e) => Err(e.into()),
374    }
375}
376
377fn note_failure(
378    data_dir: &Path,
379    error: &anyhow::Error,
380    progressed: bool,
381    now: i64,
382) -> Result<MergeFailing> {
383    let failing = next_failing(failing(data_dir)?, progressed, error, now);
384    write_atomically(
385        &data_dir.join(SPOOL_FAILING),
386        &serde_json::to_vec(&failing)?,
387    )?;
388    Ok(failing)
389}
390
391fn next_failing(
392    before: Option<MergeFailing>,
393    progressed: bool,
394    error: &anyhow::Error,
395    now: i64,
396) -> MergeFailing {
397    match before.filter(|_| !progressed) {
398        Some(before) => MergeFailing {
399            since: before.since.min(now),
400            attempts: before.attempts.saturating_add(1),
401            error: format!("{error:#}"),
402        },
403        None => MergeFailing {
404            since: now,
405            attempts: 1,
406            error: format!("{error:#}"),
407        },
408    }
409}
410
411fn write_atomically(path: &Path, bytes: &[u8]) -> Result<()> {
412    let mut name = path.file_name().unwrap_or_default().to_os_string();
413    name.push(".tmp");
414    let tmp = path.with_file_name(name);
415    fs::write(&tmp, bytes)?;
416    fs::rename(&tmp, path)?;
417    Ok(())
418}
419
420fn remove_if_present(path: &Path) -> Result<()> {
421    match fs::remove_file(path) {
422        Ok(()) => Ok(()),
423        Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
424        Err(e) => Err(e).with_context(|| format!("removing {}", path.display())),
425    }
426}
427
428/// The info file goes last, so a crash part way leaves a spool that the
429/// next attempt finds and clears again.
430fn clear_spool(data_dir: &Path) -> Result<()> {
431    remove_if_present(&data_dir.join(SPOOL_FAILING))?;
432    remove_if_present(&data_dir.join(SPOOL))?;
433    remove_if_present(&data_dir.join(SPOOL_INFO))
434}
435
436fn abandon(sqlite: &SqliteStore, data_dir: &Path) -> Result<()> {
437    sqlite.abandon_snapshot_merge()?;
438    clear_spool(data_dir)
439}
440
441/// Each step runs whatever the one before returned, and the first error
442/// is the one reported.
443fn upkeep(sqlite: &SqliteStore, data_dir: &Path, gate: &WriteGate, health: &Health) -> Result<()> {
444    let merged = merge_spooled(sqlite, data_dir, gate).map(|_| ());
445    let reported = status(data_dir).map(|merge| health.snapshot_merge_checked(merge, Utc::now()));
446    let replayed = replay(sqlite, gate);
447    merged.and(reported).and(replayed)
448}
449
450/// Settled records replay once per binary and schema, so a fix shipped
451/// later recovers them, and are dropped by age or past the cap.
452fn replay(sqlite: &SqliteStore, gate: &WriteGate) -> Result<()> {
453    let Some(_slot) = gate.try_enter() else {
454        return Ok(());
455    };
456    if let Some(replayed) = sqlite.replay_settled_records(&replay_build())?
457        && replayed.recovered + replayed.still_failing > 0
458    {
459        tracing::info!(
460            recovered = replayed.recovered,
461            still_failing = replayed.still_failing,
462            "replayed settled sync records"
463        );
464    }
465    let pruned = sqlite.prune_settled_records(Utc::now().timestamp_micros())?;
466    if pruned.aged + pruned.capped > 0 {
467        tracing::info!(
468            aged = pruned.aged,
469            capped = pruned.capped,
470            "dropped settled sync records"
471        );
472    }
473    Ok(())
474}
475
476/// Retries an unfinished merge every minute until it clears or is dropped,
477/// reports it to `/health`, and replays settled records after an upgrade.
478pub fn spawn_resumer(
479    sqlite: Arc<SqliteStore>,
480    data_dir: PathBuf,
481    gate: Arc<WriteGate>,
482    health: Arc<Health>,
483) -> JoinHandle<()> {
484    tokio::spawn(async move {
485        let mut interval = tokio::time::interval(RESUME_INTERVAL);
486        loop {
487            interval.tick().await;
488            let (sqlite, data_dir, gate, health) = (
489                sqlite.clone(),
490                data_dir.clone(),
491                gate.clone(),
492                health.clone(),
493            );
494            let done =
495                tokio::task::spawn_blocking(move || upkeep(&sqlite, &data_dir, &gate, &health))
496                    .await;
497            match done {
498                Ok(Ok(())) => {}
499                Ok(Err(e)) => tracing::warn!("snapshot merge or settled replay failed: {e:#}"),
500                Err(e) => tracing::warn!("snapshot merge or settled replay panicked: {e}"),
501            }
502        }
503    })
504}
505
506#[cfg(test)]
507mod tests {
508    use super::*;
509
510    #[test]
511    fn failures_keep_their_first_time_across_attempts_and_clear_with_the_spool() {
512        let dir = tempfile::tempdir().unwrap();
513        let first = note_failure(dir.path(), &anyhow!("disk full"), false, 100).unwrap();
514        let second = note_failure(dir.path(), &anyhow!("still full"), false, 250).unwrap();
515
516        assert_eq!((first.since, first.attempts), (100, 1));
517        assert_eq!((second.since, second.attempts), (100, 2));
518        assert_eq!(second.error, "still full");
519        assert_eq!(failing(dir.path()).unwrap(), Some(second));
520
521        clear_spool(dir.path()).unwrap();
522        assert_eq!(failing(dir.path()).unwrap(), None);
523    }
524
525    #[test]
526    fn a_failure_after_rows_were_applied_starts_a_new_day() {
527        let before = MergeFailing {
528            since: 100,
529            attempts: 4,
530            error: "disk full".to_string(),
531        };
532        let stuck = next_failing(Some(before.clone()), false, &anyhow!("again"), 900);
533        let moved = next_failing(Some(before), true, &anyhow!("again"), 900);
534
535        assert_eq!((stuck.since, stuck.attempts), (100, 5));
536        assert_eq!((moved.since, moved.attempts), (900, 1));
537    }
538}