use std::collections::BTreeSet;
use std::fs;
use std::io::Write;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
use std::time::Duration;
use anyhow::{Context, Result, anyhow, bail};
use chrono::Utc;
use second_brain_core::sqlite_store::{MergeOutcome, SnapshotOptions, SqliteStore, replay_build};
use second_brain_sync::keyring::{IdentityMismatch, Keyring};
use second_brain_sync::protocol::{Hello, Merging, SYNC_PROTOCOL_VERSION};
use second_brain_sync::snapshot::{
SNAPSHOT_FORMAT, SnapshotHeader, SnapshotKey, SnapshotWriter, spool, spool_rows, verify,
};
use serde::{Deserialize, Serialize};
use tokio::task::JoinHandle;
use crate::health::Health;
use crate::write_gate::WriteGate;
const SPOOL: &str = "snapshot.spool";
const SPOOL_INFO: &str = "snapshot.spool.json";
const SPOOL_FAILING: &str = "snapshot.spool.failing.json";
const ABANDON_AFTER_MICROS: i64 = 24 * 3600 * 1_000_000;
const RESUME_INTERVAL: Duration = Duration::from_secs(60);
#[derive(Debug)]
pub struct MergeRunning(pub Option<Merging>);
impl std::fmt::Display for MergeRunning {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match &self.0 {
Some(merging) => write!(
f,
"a snapshot merge from {} through seq {} has not finished",
merging.exporter, merging.seq
),
None => write!(f, "another snapshot is being received"),
}
}
}
impl std::error::Error for MergeRunning {}
static RUNNING: Mutex<BTreeSet<PathBuf>> = Mutex::new(BTreeSet::new());
struct Running(PathBuf);
impl Running {
fn claim(data_dir: &Path) -> Option<Self> {
let mut running = RUNNING.lock().unwrap_or_else(|e| e.into_inner());
running
.insert(data_dir.to_path_buf())
.then(|| Self(data_dir.to_path_buf()))
}
}
impl Drop for Running {
fn drop(&mut self) {
RUNNING
.lock()
.unwrap_or_else(|e| e.into_inner())
.remove(&self.0);
}
}
pub fn write_snapshot<W: Write>(
sqlite: &SqliteStore,
data_dir: &Path,
gate: &WriteGate,
peer: &Hello,
out: W,
) -> Result<()> {
pin_for_snapshot(sqlite, gate, peer)?;
stream_snapshot(sqlite, data_dir, peer, out)
}
pub fn pin_for_snapshot(sqlite: &SqliteStore, gate: &WriteGate, peer: &Hello) -> Result<()> {
let _slot = gate.try_enter().ok_or_else(|| {
anyhow!("store maintenance is running; the snapshot is sent on the next sync")
})?;
sqlite.pin_peer(&peer.machine_id, sqlite.log_head()?)
}
pub fn stream_snapshot<W: Write>(
sqlite: &SqliteStore,
data_dir: &Path,
peer: &Hello,
out: W,
) -> Result<()> {
let keyring = Keyring::load(sqlite, data_dir)?;
let declared = peer.keys.as_ref().map(|k| k.epochs.as_slice());
let key = keyring.snapshot_key_for(&peer.machine_id, declared)?;
let keys = keyring.handshake_for(&peer.machine_id, declared)?;
let header = SnapshotHeader {
format: SNAPSHOT_FORMAT,
snapshot_id: uuid::Uuid::new_v4().to_string(),
exporter: sqlite.machine_id().to_string(),
epoch: *key.epoch(),
sync_protocol: SYNC_PROTOCOL_VERSION,
schema_version: sqlite.schema_version()?,
created_at: Utc::now().timestamp_micros(),
};
let mut writer = SnapshotWriter::new(out, key, header)?;
let state = sqlite.export_snapshot(SnapshotOptions { with_chunks: true }, &mut |row| {
writer.row(&row)
})?;
writer.finish(state, Some(keys))?;
Ok(())
}
pub fn merging(data_dir: &Path) -> Result<Option<Merging>> {
match fs::read(data_dir.join(SPOOL_INFO)) {
Ok(bytes) => Ok(Some(
serde_json::from_slice(&bytes).context("reading the spooled snapshot's info")?,
)),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
Err(e) => Err(e.into()),
}
}
fn key_for<'a>(keyring: &'a Keyring<'a>) -> impl Fn(&SnapshotHeader) -> Result<SnapshotKey> + 'a {
move |header| keyring.snapshot_opening_key(&header.exporter, &header.epoch)
}
pub fn receive<R: std::io::Read>(
sqlite: &SqliteStore,
data_dir: &Path,
input: R,
exporter: Option<&str>,
) -> Result<Merging> {
let Some(_running) = Running::claim(data_dir) else {
return Err(MergeRunning(merging(data_dir)?).into());
};
if let Some(running) = merging(data_dir)? {
return Err(MergeRunning(Some(running)).into());
}
let keyring = Keyring::load(sqlite, data_dir)?;
let spool_path = data_dir.join(SPOOL);
let (header, manifest) = spool(input, &spool_path, &key_for(&keyring))?;
if let Some(expected) = exporter
&& header.exporter != expected
{
remove_if_present(&spool_path)?;
bail!(
"the snapshot from {expected} was exported by {}; nothing was applied",
header.exporter
);
}
let merging = Merging {
snapshot_id: header.snapshot_id,
exporter: header.exporter,
seq: manifest.state.seq,
};
remove_if_present(&data_dir.join(SPOOL_FAILING))?;
write_atomically(&data_dir.join(SPOOL_INFO), &serde_json::to_vec(&merging)?)?;
Ok(merging)
}
pub fn merge_spooled(
sqlite: &SqliteStore,
data_dir: &Path,
gate: &WriteGate,
) -> Result<Option<MergeOutcome>> {
let Some(_running) = Running::claim(data_dir) else {
return Ok(None);
};
let Some(merging) = merging(data_dir)? else {
return Ok(None);
};
let Some(_slot) = gate.try_enter() else {
return Ok(None);
};
match merge_claimed(sqlite, data_dir, gate, &merging) {
Ok(Step::Done(outcome)) => {
clear_spool(data_dir)?;
tracing::info!(
exporter = %merging.exporter,
seq = merging.seq,
rows = outcome.rows,
changed = outcome.changed,
settled = outcome.settled,
"snapshot merged"
);
Ok(Some(outcome))
}
Ok(Step::Paused { progressed }) => {
if progressed {
remove_if_present(&data_dir.join(SPOOL_FAILING))?;
}
Ok(None)
}
Err(Failure::Spool(e)) => {
abandon(sqlite, data_dir)?;
Err(e.context(format!(
"the snapshot from {} will not open and was dropped; the next sync gets another",
merging.exporter
)))
}
Err(Failure::Merge {
error: e,
progressed,
}) => {
let now = Utc::now().timestamp_micros();
let failing = note_failure(data_dir, &e, progressed, now)?;
if now - failing.since >= ABANDON_AFTER_MICROS {
abandon(sqlite, data_dir)?;
return Err(e.context(format!(
"the snapshot from {} failed to merge for a day and was dropped; the next \
sync gets another",
merging.exporter
)));
}
Err(e)
}
}
}
enum Step {
Done(MergeOutcome),
Paused { progressed: bool },
}
enum Failure {
Spool(anyhow::Error),
Merge {
error: anyhow::Error,
progressed: bool,
},
}
impl Failure {
fn merge(error: anyhow::Error) -> Self {
Failure::Merge {
error,
progressed: false,
}
}
}
fn identity_or_merge(e: anyhow::Error) -> Failure {
if e.downcast_ref::<IdentityMismatch>().is_some() {
Failure::Spool(e)
} else {
Failure::merge(e)
}
}
fn merge_claimed(
sqlite: &SqliteStore,
data_dir: &Path,
gate: &WriteGate,
merging: &Merging,
) -> Result<Step, Failure> {
let spool_path = data_dir.join(SPOOL);
let keyring = Keyring::load(sqlite, data_dir).map_err(Failure::merge)?;
let key_for = key_for(&keyring);
let (header, manifest) = verify(&spool_path, &key_for).map_err(Failure::Spool)?;
if header.snapshot_id != merging.snapshot_id {
return Err(Failure::Spool(anyhow!(
"the spool holds snapshot {}, not {}",
header.snapshot_id,
merging.snapshot_id
)));
}
if let Some(keys) = &manifest.keys {
keyring
.check_peer(keys, &header.exporter)
.map_err(identity_or_merge)?;
}
keyring
.opened_snapshot(&header.exporter, &header.epoch)
.map_err(identity_or_merge)?;
let mut merge = sqlite
.begin_snapshot_merge(&header.snapshot_id, &header.exporter, merging.seq)
.map_err(Failure::merge)?;
let rows_before = merge.progress().rows_done;
let (mut push_failed, mut paused) = (false, false);
let streamed = spool_rows(&spool_path, &key_for, &mut |row| {
if let Err(e) = merge.push(row) {
push_failed = true;
return Err(e);
}
if gate.is_closed() {
paused = true;
bail!("store maintenance closed the write gate");
}
Ok(())
});
let progressed = merge.progress().rows_done > rows_before;
match streamed {
Ok(()) => {}
Err(_) if paused => return Ok(Step::Paused { progressed }),
Err(error) if push_failed => return Err(Failure::Merge { error, progressed }),
Err(e) => return Err(Failure::Spool(e)),
}
if let Some(keys) = &manifest.keys {
keyring
.accept(&header.exporter, keys, Utc::now().timestamp_micros())
.map_err(identity_or_merge)?;
}
let outcome = merge
.finish(&manifest.state.cursors)
.map_err(Failure::merge)?;
Ok(Step::Done(outcome))
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct MergeFailing {
pub since: i64,
pub attempts: u32,
pub error: String,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct MergeStatus {
pub merging: Merging,
pub failing: Option<MergeFailing>,
}
pub fn status(data_dir: &Path) -> Result<Option<MergeStatus>> {
let Some(merging) = merging(data_dir)? else {
return Ok(None);
};
Ok(Some(MergeStatus {
merging,
failing: failing(data_dir)?,
}))
}
fn failing(data_dir: &Path) -> Result<Option<MergeFailing>> {
match fs::read(data_dir.join(SPOOL_FAILING)) {
Ok(bytes) => Ok(Some(
serde_json::from_slice(&bytes).context("reading the snapshot merge's failures")?,
)),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
Err(e) => Err(e.into()),
}
}
fn note_failure(
data_dir: &Path,
error: &anyhow::Error,
progressed: bool,
now: i64,
) -> Result<MergeFailing> {
let failing = next_failing(failing(data_dir)?, progressed, error, now);
write_atomically(
&data_dir.join(SPOOL_FAILING),
&serde_json::to_vec(&failing)?,
)?;
Ok(failing)
}
fn next_failing(
before: Option<MergeFailing>,
progressed: bool,
error: &anyhow::Error,
now: i64,
) -> MergeFailing {
match before.filter(|_| !progressed) {
Some(before) => MergeFailing {
since: before.since.min(now),
attempts: before.attempts.saturating_add(1),
error: format!("{error:#}"),
},
None => MergeFailing {
since: now,
attempts: 1,
error: format!("{error:#}"),
},
}
}
fn write_atomically(path: &Path, bytes: &[u8]) -> Result<()> {
let mut name = path.file_name().unwrap_or_default().to_os_string();
name.push(".tmp");
let tmp = path.with_file_name(name);
fs::write(&tmp, bytes)?;
fs::rename(&tmp, path)?;
Ok(())
}
fn remove_if_present(path: &Path) -> Result<()> {
match fs::remove_file(path) {
Ok(()) => Ok(()),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(e) => Err(e).with_context(|| format!("removing {}", path.display())),
}
}
fn clear_spool(data_dir: &Path) -> Result<()> {
remove_if_present(&data_dir.join(SPOOL_FAILING))?;
remove_if_present(&data_dir.join(SPOOL))?;
remove_if_present(&data_dir.join(SPOOL_INFO))
}
fn abandon(sqlite: &SqliteStore, data_dir: &Path) -> Result<()> {
sqlite.abandon_snapshot_merge()?;
clear_spool(data_dir)
}
fn upkeep(sqlite: &SqliteStore, data_dir: &Path, gate: &WriteGate, health: &Health) -> Result<()> {
let merged = merge_spooled(sqlite, data_dir, gate).map(|_| ());
let reported = status(data_dir).map(|merge| health.snapshot_merge_checked(merge, Utc::now()));
let replayed = replay(sqlite, gate);
merged.and(reported).and(replayed)
}
fn replay(sqlite: &SqliteStore, gate: &WriteGate) -> Result<()> {
let Some(_slot) = gate.try_enter() else {
return Ok(());
};
if let Some(replayed) = sqlite.replay_settled_records(&replay_build())?
&& replayed.recovered + replayed.still_failing > 0
{
tracing::info!(
recovered = replayed.recovered,
still_failing = replayed.still_failing,
"replayed settled sync records"
);
}
let pruned = sqlite.prune_settled_records(Utc::now().timestamp_micros())?;
if pruned.aged + pruned.capped > 0 {
tracing::info!(
aged = pruned.aged,
capped = pruned.capped,
"dropped settled sync records"
);
}
Ok(())
}
pub fn spawn_resumer(
sqlite: Arc<SqliteStore>,
data_dir: PathBuf,
gate: Arc<WriteGate>,
health: Arc<Health>,
) -> JoinHandle<()> {
tokio::spawn(async move {
let mut interval = tokio::time::interval(RESUME_INTERVAL);
loop {
interval.tick().await;
let (sqlite, data_dir, gate, health) = (
sqlite.clone(),
data_dir.clone(),
gate.clone(),
health.clone(),
);
let done =
tokio::task::spawn_blocking(move || upkeep(&sqlite, &data_dir, &gate, &health))
.await;
match done {
Ok(Ok(())) => {}
Ok(Err(e)) => tracing::warn!("snapshot merge or settled replay failed: {e:#}"),
Err(e) => tracing::warn!("snapshot merge or settled replay panicked: {e}"),
}
}
})
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn failures_keep_their_first_time_across_attempts_and_clear_with_the_spool() {
let dir = tempfile::tempdir().unwrap();
let first = note_failure(dir.path(), &anyhow!("disk full"), false, 100).unwrap();
let second = note_failure(dir.path(), &anyhow!("still full"), false, 250).unwrap();
assert_eq!((first.since, first.attempts), (100, 1));
assert_eq!((second.since, second.attempts), (100, 2));
assert_eq!(second.error, "still full");
assert_eq!(failing(dir.path()).unwrap(), Some(second));
clear_spool(dir.path()).unwrap();
assert_eq!(failing(dir.path()).unwrap(), None);
}
#[test]
fn a_failure_after_rows_were_applied_starts_a_new_day() {
let before = MergeFailing {
since: 100,
attempts: 4,
error: "disk full".to_string(),
};
let stuck = next_failing(Some(before.clone()), false, &anyhow!("again"), 900);
let moved = next_failing(Some(before), true, &anyhow!("again"), 900);
assert_eq!((stuck.since, stuck.attempts), (100, 5));
assert_eq!((moved.since, moved.attempts), (900, 1));
}
}