1use std::collections::BTreeSet;
2use std::fs;
3use std::io::Write;
4use std::path::{Path, PathBuf};
5use std::sync::{Arc, Mutex};
6use std::time::Duration;
7
8use anyhow::{Context, Result, anyhow, bail};
9use chrono::Utc;
10use second_brain_core::sqlite_store::{MergeOutcome, SnapshotOptions, SqliteStore, replay_build};
11use second_brain_sync::keyring::{IdentityMismatch, Keyring};
12use second_brain_sync::protocol::{Hello, Merging, SYNC_PROTOCOL_VERSION};
13use second_brain_sync::snapshot::{
14 SNAPSHOT_FORMAT, SnapshotHeader, SnapshotKey, SnapshotWriter, spool, spool_rows, verify,
15};
16use serde::{Deserialize, Serialize};
17use tokio::task::JoinHandle;
18
19use crate::health::Health;
20use crate::write_gate::WriteGate;
21
22const SPOOL: &str = "snapshot.spool";
23const SPOOL_INFO: &str = "snapshot.spool.json";
24const SPOOL_FAILING: &str = "snapshot.spool.failing.json";
25
26const ABANDON_AFTER_MICROS: i64 = 24 * 3600 * 1_000_000;
29
30const RESUME_INTERVAL: Duration = Duration::from_secs(60);
34
35#[derive(Debug)]
38pub struct MergeRunning(pub Option<Merging>);
39
40impl std::fmt::Display for MergeRunning {
41 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
42 match &self.0 {
43 Some(merging) => write!(
44 f,
45 "a snapshot merge from {} through seq {} has not finished",
46 merging.exporter, merging.seq
47 ),
48 None => write!(f, "another snapshot is being received"),
49 }
50 }
51}
52
53impl std::error::Error for MergeRunning {}
54
55static RUNNING: Mutex<BTreeSet<PathBuf>> = Mutex::new(BTreeSet::new());
59
60struct Running(PathBuf);
61
62impl Running {
63 fn claim(data_dir: &Path) -> Option<Self> {
64 let mut running = RUNNING.lock().unwrap_or_else(|e| e.into_inner());
65 running
66 .insert(data_dir.to_path_buf())
67 .then(|| Self(data_dir.to_path_buf()))
68 }
69}
70
71impl Drop for Running {
72 fn drop(&mut self) {
73 RUNNING
74 .lock()
75 .unwrap_or_else(|e| e.into_inner())
76 .remove(&self.0);
77 }
78}
79
80pub fn write_snapshot<W: Write>(
83 sqlite: &SqliteStore,
84 data_dir: &Path,
85 gate: &WriteGate,
86 peer: &Hello,
87 out: W,
88) -> Result<()> {
89 pin_for_snapshot(sqlite, gate, peer)?;
90 stream_snapshot(sqlite, data_dir, peer, out)
91}
92
93pub fn pin_for_snapshot(sqlite: &SqliteStore, gate: &WriteGate, peer: &Hello) -> Result<()> {
96 let _slot = gate.try_enter().ok_or_else(|| {
97 anyhow!("store maintenance is running; the snapshot is sent on the next sync")
98 })?;
99 sqlite.pin_peer(&peer.machine_id, sqlite.log_head()?)
100}
101
102pub fn stream_snapshot<W: Write>(
104 sqlite: &SqliteStore,
105 data_dir: &Path,
106 peer: &Hello,
107 out: W,
108) -> Result<()> {
109 let keyring = Keyring::load(sqlite, data_dir)?;
110 let declared = peer.keys.as_ref().map(|k| k.epochs.as_slice());
111 let key = keyring.snapshot_key_for(&peer.machine_id, declared)?;
112 let keys = keyring.handshake_for(&peer.machine_id, declared)?;
113 let header = SnapshotHeader {
114 format: SNAPSHOT_FORMAT,
115 snapshot_id: uuid::Uuid::new_v4().to_string(),
116 exporter: sqlite.machine_id().to_string(),
117 epoch: *key.epoch(),
118 sync_protocol: SYNC_PROTOCOL_VERSION,
119 schema_version: sqlite.schema_version()?,
120 created_at: Utc::now().timestamp_micros(),
121 };
122 let mut writer = SnapshotWriter::new(out, key, header)?;
123 let state = sqlite.export_snapshot(SnapshotOptions { with_chunks: true }, &mut |row| {
124 writer.row(&row)
125 })?;
126 writer.finish(state, Some(keys))?;
127 Ok(())
128}
129
130pub fn merging(data_dir: &Path) -> Result<Option<Merging>> {
132 match fs::read(data_dir.join(SPOOL_INFO)) {
133 Ok(bytes) => Ok(Some(
134 serde_json::from_slice(&bytes).context("reading the spooled snapshot's info")?,
135 )),
136 Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
137 Err(e) => Err(e.into()),
138 }
139}
140
141fn key_for<'a>(keyring: &'a Keyring<'a>) -> impl Fn(&SnapshotHeader) -> Result<SnapshotKey> + 'a {
142 move |header| keyring.snapshot_opening_key(&header.exporter, &header.epoch)
143}
144
145pub fn receive<R: std::io::Read>(
150 sqlite: &SqliteStore,
151 data_dir: &Path,
152 input: R,
153 exporter: Option<&str>,
154) -> Result<Merging> {
155 let Some(_running) = Running::claim(data_dir) else {
156 return Err(MergeRunning(merging(data_dir)?).into());
157 };
158 if let Some(running) = merging(data_dir)? {
159 return Err(MergeRunning(Some(running)).into());
160 }
161 let keyring = Keyring::load(sqlite, data_dir)?;
162 let spool_path = data_dir.join(SPOOL);
163 let (header, manifest) = spool(input, &spool_path, &key_for(&keyring))?;
164 if let Some(expected) = exporter
165 && header.exporter != expected
166 {
167 remove_if_present(&spool_path)?;
168 bail!(
169 "the snapshot from {expected} was exported by {}; nothing was applied",
170 header.exporter
171 );
172 }
173 let merging = Merging {
174 snapshot_id: header.snapshot_id,
175 exporter: header.exporter,
176 seq: manifest.state.seq,
177 };
178 remove_if_present(&data_dir.join(SPOOL_FAILING))?;
179 write_atomically(&data_dir.join(SPOOL_INFO), &serde_json::to_vec(&merging)?)?;
180 Ok(merging)
181}
182
183pub fn merge_spooled(
190 sqlite: &SqliteStore,
191 data_dir: &Path,
192 gate: &WriteGate,
193) -> Result<Option<MergeOutcome>> {
194 let Some(_running) = Running::claim(data_dir) else {
195 return Ok(None);
196 };
197 let Some(merging) = merging(data_dir)? else {
198 return Ok(None);
199 };
200 let Some(_slot) = gate.try_enter() else {
201 return Ok(None);
202 };
203 match merge_claimed(sqlite, data_dir, gate, &merging) {
204 Ok(Step::Done(outcome)) => {
205 clear_spool(data_dir)?;
206 tracing::info!(
207 exporter = %merging.exporter,
208 seq = merging.seq,
209 rows = outcome.rows,
210 changed = outcome.changed,
211 settled = outcome.settled,
212 "snapshot merged"
213 );
214 Ok(Some(outcome))
215 }
216 Ok(Step::Paused { progressed }) => {
217 if progressed {
218 remove_if_present(&data_dir.join(SPOOL_FAILING))?;
219 }
220 Ok(None)
221 }
222 Err(Failure::Spool(e)) => {
223 abandon(sqlite, data_dir)?;
224 Err(e.context(format!(
225 "the snapshot from {} will not open and was dropped; the next sync gets another",
226 merging.exporter
227 )))
228 }
229 Err(Failure::Merge {
230 error: e,
231 progressed,
232 }) => {
233 let now = Utc::now().timestamp_micros();
234 let failing = note_failure(data_dir, &e, progressed, now)?;
235 if now - failing.since >= ABANDON_AFTER_MICROS {
236 abandon(sqlite, data_dir)?;
237 return Err(e.context(format!(
238 "the snapshot from {} failed to merge for a day and was dropped; the next \
239 sync gets another",
240 merging.exporter
241 )));
242 }
243 Err(e)
244 }
245 }
246}
247
248enum Step {
251 Done(MergeOutcome),
252 Paused { progressed: bool },
253}
254
255enum Failure {
258 Spool(anyhow::Error),
259 Merge {
260 error: anyhow::Error,
261 progressed: bool,
262 },
263}
264
265impl Failure {
266 fn merge(error: anyhow::Error) -> Self {
267 Failure::Merge {
268 error,
269 progressed: false,
270 }
271 }
272}
273
274fn identity_or_merge(e: anyhow::Error) -> Failure {
275 if e.downcast_ref::<IdentityMismatch>().is_some() {
276 Failure::Spool(e)
277 } else {
278 Failure::merge(e)
279 }
280}
281
282fn merge_claimed(
283 sqlite: &SqliteStore,
284 data_dir: &Path,
285 gate: &WriteGate,
286 merging: &Merging,
287) -> Result<Step, Failure> {
288 let spool_path = data_dir.join(SPOOL);
289 let keyring = Keyring::load(sqlite, data_dir).map_err(Failure::merge)?;
290 let key_for = key_for(&keyring);
291 let (header, manifest) = verify(&spool_path, &key_for).map_err(Failure::Spool)?;
292 if header.snapshot_id != merging.snapshot_id {
293 return Err(Failure::Spool(anyhow!(
294 "the spool holds snapshot {}, not {}",
295 header.snapshot_id,
296 merging.snapshot_id
297 )));
298 }
299 if let Some(keys) = &manifest.keys {
300 keyring
301 .check_peer(keys, &header.exporter)
302 .map_err(identity_or_merge)?;
303 }
304 keyring
305 .opened_snapshot(&header.exporter, &header.epoch)
306 .map_err(identity_or_merge)?;
307 let mut merge = sqlite
308 .begin_snapshot_merge(&header.snapshot_id, &header.exporter, merging.seq)
309 .map_err(Failure::merge)?;
310 let rows_before = merge.progress().rows_done;
311 let (mut push_failed, mut paused) = (false, false);
312 let streamed = spool_rows(&spool_path, &key_for, &mut |row| {
313 if let Err(e) = merge.push(row) {
314 push_failed = true;
315 return Err(e);
316 }
317 if gate.is_closed() {
318 paused = true;
319 bail!("store maintenance closed the write gate");
320 }
321 Ok(())
322 });
323 let progressed = merge.progress().rows_done > rows_before;
324 match streamed {
325 Ok(()) => {}
326 Err(_) if paused => return Ok(Step::Paused { progressed }),
327 Err(error) if push_failed => return Err(Failure::Merge { error, progressed }),
328 Err(e) => return Err(Failure::Spool(e)),
329 }
330 if let Some(keys) = &manifest.keys {
331 keyring
332 .accept(&header.exporter, keys, Utc::now().timestamp_micros())
333 .map_err(identity_or_merge)?;
334 }
335 let outcome = merge
336 .finish(&manifest.state.cursors)
337 .map_err(Failure::merge)?;
338 Ok(Step::Done(outcome))
339}
340
341#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
344pub struct MergeFailing {
345 pub since: i64,
346 pub attempts: u32,
347 pub error: String,
348}
349
350#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
352pub struct MergeStatus {
353 pub merging: Merging,
354 pub failing: Option<MergeFailing>,
355}
356
357pub fn status(data_dir: &Path) -> Result<Option<MergeStatus>> {
358 let Some(merging) = merging(data_dir)? else {
359 return Ok(None);
360 };
361 Ok(Some(MergeStatus {
362 merging,
363 failing: failing(data_dir)?,
364 }))
365}
366
367fn failing(data_dir: &Path) -> Result<Option<MergeFailing>> {
368 match fs::read(data_dir.join(SPOOL_FAILING)) {
369 Ok(bytes) => Ok(Some(
370 serde_json::from_slice(&bytes).context("reading the snapshot merge's failures")?,
371 )),
372 Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
373 Err(e) => Err(e.into()),
374 }
375}
376
377fn note_failure(
378 data_dir: &Path,
379 error: &anyhow::Error,
380 progressed: bool,
381 now: i64,
382) -> Result<MergeFailing> {
383 let failing = next_failing(failing(data_dir)?, progressed, error, now);
384 write_atomically(
385 &data_dir.join(SPOOL_FAILING),
386 &serde_json::to_vec(&failing)?,
387 )?;
388 Ok(failing)
389}
390
391fn next_failing(
392 before: Option<MergeFailing>,
393 progressed: bool,
394 error: &anyhow::Error,
395 now: i64,
396) -> MergeFailing {
397 match before.filter(|_| !progressed) {
398 Some(before) => MergeFailing {
399 since: before.since.min(now),
400 attempts: before.attempts.saturating_add(1),
401 error: format!("{error:#}"),
402 },
403 None => MergeFailing {
404 since: now,
405 attempts: 1,
406 error: format!("{error:#}"),
407 },
408 }
409}
410
411fn write_atomically(path: &Path, bytes: &[u8]) -> Result<()> {
412 let mut name = path.file_name().unwrap_or_default().to_os_string();
413 name.push(".tmp");
414 let tmp = path.with_file_name(name);
415 fs::write(&tmp, bytes)?;
416 fs::rename(&tmp, path)?;
417 Ok(())
418}
419
420fn remove_if_present(path: &Path) -> Result<()> {
421 match fs::remove_file(path) {
422 Ok(()) => Ok(()),
423 Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
424 Err(e) => Err(e).with_context(|| format!("removing {}", path.display())),
425 }
426}
427
428fn clear_spool(data_dir: &Path) -> Result<()> {
431 remove_if_present(&data_dir.join(SPOOL_FAILING))?;
432 remove_if_present(&data_dir.join(SPOOL))?;
433 remove_if_present(&data_dir.join(SPOOL_INFO))
434}
435
436fn abandon(sqlite: &SqliteStore, data_dir: &Path) -> Result<()> {
437 sqlite.abandon_snapshot_merge()?;
438 clear_spool(data_dir)
439}
440
441fn upkeep(sqlite: &SqliteStore, data_dir: &Path, gate: &WriteGate, health: &Health) -> Result<()> {
444 let merged = merge_spooled(sqlite, data_dir, gate).map(|_| ());
445 let reported = status(data_dir).map(|merge| health.snapshot_merge_checked(merge, Utc::now()));
446 let replayed = replay(sqlite, gate);
447 merged.and(reported).and(replayed)
448}
449
450fn replay(sqlite: &SqliteStore, gate: &WriteGate) -> Result<()> {
453 let Some(_slot) = gate.try_enter() else {
454 return Ok(());
455 };
456 if let Some(replayed) = sqlite.replay_settled_records(&replay_build())?
457 && replayed.recovered + replayed.still_failing > 0
458 {
459 tracing::info!(
460 recovered = replayed.recovered,
461 still_failing = replayed.still_failing,
462 "replayed settled sync records"
463 );
464 }
465 let pruned = sqlite.prune_settled_records(Utc::now().timestamp_micros())?;
466 if pruned.aged + pruned.capped > 0 {
467 tracing::info!(
468 aged = pruned.aged,
469 capped = pruned.capped,
470 "dropped settled sync records"
471 );
472 }
473 Ok(())
474}
475
476pub fn spawn_resumer(
479 sqlite: Arc<SqliteStore>,
480 data_dir: PathBuf,
481 gate: Arc<WriteGate>,
482 health: Arc<Health>,
483) -> JoinHandle<()> {
484 tokio::spawn(async move {
485 let mut interval = tokio::time::interval(RESUME_INTERVAL);
486 loop {
487 interval.tick().await;
488 let (sqlite, data_dir, gate, health) = (
489 sqlite.clone(),
490 data_dir.clone(),
491 gate.clone(),
492 health.clone(),
493 );
494 let done =
495 tokio::task::spawn_blocking(move || upkeep(&sqlite, &data_dir, &gate, &health))
496 .await;
497 match done {
498 Ok(Ok(())) => {}
499 Ok(Err(e)) => tracing::warn!("snapshot merge or settled replay failed: {e:#}"),
500 Err(e) => tracing::warn!("snapshot merge or settled replay panicked: {e}"),
501 }
502 }
503 })
504}
505
506#[cfg(test)]
507mod tests {
508 use super::*;
509
510 #[test]
511 fn failures_keep_their_first_time_across_attempts_and_clear_with_the_spool() {
512 let dir = tempfile::tempdir().unwrap();
513 let first = note_failure(dir.path(), &anyhow!("disk full"), false, 100).unwrap();
514 let second = note_failure(dir.path(), &anyhow!("still full"), false, 250).unwrap();
515
516 assert_eq!((first.since, first.attempts), (100, 1));
517 assert_eq!((second.since, second.attempts), (100, 2));
518 assert_eq!(second.error, "still full");
519 assert_eq!(failing(dir.path()).unwrap(), Some(second));
520
521 clear_spool(dir.path()).unwrap();
522 assert_eq!(failing(dir.path()).unwrap(), None);
523 }
524
525 #[test]
526 fn a_failure_after_rows_were_applied_starts_a_new_day() {
527 let before = MergeFailing {
528 since: 100,
529 attempts: 4,
530 error: "disk full".to_string(),
531 };
532 let stuck = next_failing(Some(before.clone()), false, &anyhow!("again"), 900);
533 let moved = next_failing(Some(before), true, &anyhow!("again"), 900);
534
535 assert_eq!((stuck.since, stuck.attempts), (100, 5));
536 assert_eq!((moved.since, moved.attempts), (900, 1));
537 }
538}