second-brain-api 0.13.2

HTTP API server for second-brain: REST endpoints for recall, remember, and ingest
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
use std::collections::BTreeSet;
use std::fs;
use std::io::Write;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
use std::time::Duration;

use anyhow::{Context, Result, anyhow, bail};
use chrono::Utc;
use second_brain_core::sqlite_store::{MergeOutcome, SnapshotOptions, SqliteStore, replay_build};
use second_brain_sync::keyring::{IdentityMismatch, Keyring};
use second_brain_sync::protocol::{Hello, Merging, SYNC_PROTOCOL_VERSION};
use second_brain_sync::snapshot::{
    SNAPSHOT_FORMAT, SnapshotHeader, SnapshotKey, SnapshotWriter, spool, spool_rows, verify,
};
use serde::{Deserialize, Serialize};
use tokio::task::JoinHandle;

use crate::health::Health;
use crate::write_gate::WriteGate;

const SPOOL: &str = "snapshot.spool";
const SPOOL_INFO: &str = "snapshot.spool.json";
const SPOOL_FAILING: &str = "snapshot.spool.failing.json";

/// A merge that has failed for this long is dropped, so the exporter sends
/// a fresh snapshot instead of the device waiting on one that never lands.
const ABANDON_AFTER_MICROS: i64 = 24 * 3600 * 1_000_000;

/// How often the daemon looks for a spooled snapshot whose merge has not
/// finished: one interrupted by a restart, a closed clock gate or store
/// maintenance.
const RESUME_INTERVAL: Duration = Duration::from_secs(60);

/// A second snapshot is refused while one is being received, or is
/// spooled and unmerged, so the pusher waits instead of sending it again.
#[derive(Debug)]
pub struct MergeRunning(pub Option<Merging>);

impl std::fmt::Display for MergeRunning {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match &self.0 {
            Some(merging) => write!(
                f,
                "a snapshot merge from {} through seq {} has not finished",
                merging.exporter, merging.seq
            ),
            None => write!(f, "another snapshot is being received"),
        }
    }
}

impl std::error::Error for MergeRunning {}

/// Data dirs whose snapshot is being received or merged now: one per store
/// at a time, so the spool never changes under a merge, and the two
/// daemons of an in-process test each get their own.
static RUNNING: Mutex<BTreeSet<PathBuf>> = Mutex::new(BTreeSet::new());

struct Running(PathBuf);

impl Running {
    fn claim(data_dir: &Path) -> Option<Self> {
        let mut running = RUNNING.lock().unwrap_or_else(|e| e.into_inner());
        running
            .insert(data_dir.to_path_buf())
            .then(|| Self(data_dir.to_path_buf()))
    }
}

impl Drop for Running {
    fn drop(&mut self) {
        RUNNING
            .lock()
            .unwrap_or_else(|e| e.into_inner())
            .remove(&self.0);
    }
}

/// Streams a snapshot of this store, sealed for `peer`, into `out`, after
/// `pin_for_snapshot`.
pub fn write_snapshot<W: Write>(
    sqlite: &SqliteStore,
    data_dir: &Path,
    gate: &WriteGate,
    peer: &Hello,
    out: W,
) -> Result<()> {
    pin_for_snapshot(sqlite, gate, peer)?;
    stream_snapshot(sqlite, data_dir, peer, out)
}

/// Pins `peer` at the head, so the floor holds every row above the seq of
/// the snapshot that follows until its follow-up pull or push acks it.
pub fn pin_for_snapshot(sqlite: &SqliteStore, gate: &WriteGate, peer: &Hello) -> Result<()> {
    let _slot = gate.try_enter().ok_or_else(|| {
        anyhow!("store maintenance is running; the snapshot is sent on the next sync")
    })?;
    sqlite.pin_peer(&peer.machine_id, sqlite.log_head()?)
}

/// The stream `write_snapshot` sends, for a caller that already pinned.
pub fn stream_snapshot<W: Write>(
    sqlite: &SqliteStore,
    data_dir: &Path,
    peer: &Hello,
    out: W,
) -> Result<()> {
    let keyring = Keyring::load(sqlite, data_dir)?;
    let declared = peer.keys.as_ref().map(|k| k.epochs.as_slice());
    let key = keyring.snapshot_key_for(&peer.machine_id, declared)?;
    let keys = keyring.handshake_for(&peer.machine_id, declared)?;
    let header = SnapshotHeader {
        format: SNAPSHOT_FORMAT,
        snapshot_id: uuid::Uuid::new_v4().to_string(),
        exporter: sqlite.machine_id().to_string(),
        epoch: *key.epoch(),
        sync_protocol: SYNC_PROTOCOL_VERSION,
        schema_version: sqlite.schema_version()?,
        created_at: Utc::now().timestamp_micros(),
    };
    let mut writer = SnapshotWriter::new(out, key, header)?;
    let state = sqlite.export_snapshot(SnapshotOptions { with_chunks: true }, &mut |row| {
        writer.row(&row)
    })?;
    writer.finish(state, Some(keys))?;
    Ok(())
}

/// The merge spooled here and not yet finished.
pub fn merging(data_dir: &Path) -> Result<Option<Merging>> {
    match fs::read(data_dir.join(SPOOL_INFO)) {
        Ok(bytes) => Ok(Some(
            serde_json::from_slice(&bytes).context("reading the spooled snapshot's info")?,
        )),
        Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
        Err(e) => Err(e.into()),
    }
}

fn key_for<'a>(keyring: &'a Keyring<'a>) -> impl Fn(&SnapshotHeader) -> Result<SnapshotKey> + 'a {
    move |header| keyring.snapshot_opening_key(&header.exporter, &header.epoch)
}

/// Spools a snapshot stream into the data dir and checks it against its
/// manifest, and against `exporter` when the caller knows who sent it. The
/// merge itself runs after, from the spool, so a cut or restarted merge
/// resumes without a second transfer.
pub fn receive<R: std::io::Read>(
    sqlite: &SqliteStore,
    data_dir: &Path,
    input: R,
    exporter: Option<&str>,
) -> Result<Merging> {
    let Some(_running) = Running::claim(data_dir) else {
        return Err(MergeRunning(merging(data_dir)?).into());
    };
    if let Some(running) = merging(data_dir)? {
        return Err(MergeRunning(Some(running)).into());
    }
    let keyring = Keyring::load(sqlite, data_dir)?;
    let spool_path = data_dir.join(SPOOL);
    let (header, manifest) = spool(input, &spool_path, &key_for(&keyring))?;
    if let Some(expected) = exporter
        && header.exporter != expected
    {
        remove_if_present(&spool_path)?;
        bail!(
            "the snapshot from {expected} was exported by {}; nothing was applied",
            header.exporter
        );
    }
    let merging = Merging {
        snapshot_id: header.snapshot_id,
        exporter: header.exporter,
        seq: manifest.state.seq,
    };
    remove_if_present(&data_dir.join(SPOOL_FAILING))?;
    write_atomically(&data_dir.join(SPOOL_INFO), &serde_json::to_vec(&merging)?)?;
    Ok(merging)
}

/// Merges the spooled snapshot, if any, and clears the spool once the
/// cursors are raised. None when there is nothing to merge, another merge
/// or store maintenance holds the store, or maintenance closed the gate
/// mid-merge, so a later call resumes. A spool that will not open, or one
/// whose merge has failed for a day, is dropped, so the exporter sends
/// another instead of this device waiting on it.
pub fn merge_spooled(
    sqlite: &SqliteStore,
    data_dir: &Path,
    gate: &WriteGate,
) -> Result<Option<MergeOutcome>> {
    let Some(_running) = Running::claim(data_dir) else {
        return Ok(None);
    };
    let Some(merging) = merging(data_dir)? else {
        return Ok(None);
    };
    let Some(_slot) = gate.try_enter() else {
        return Ok(None);
    };
    match merge_claimed(sqlite, data_dir, gate, &merging) {
        Ok(Step::Done(outcome)) => {
            clear_spool(data_dir)?;
            tracing::info!(
                exporter = %merging.exporter,
                seq = merging.seq,
                rows = outcome.rows,
                changed = outcome.changed,
                settled = outcome.settled,
                "snapshot merged"
            );
            Ok(Some(outcome))
        }
        Ok(Step::Paused { progressed }) => {
            if progressed {
                remove_if_present(&data_dir.join(SPOOL_FAILING))?;
            }
            Ok(None)
        }
        Err(Failure::Spool(e)) => {
            abandon(sqlite, data_dir)?;
            Err(e.context(format!(
                "the snapshot from {} will not open and was dropped; the next sync gets another",
                merging.exporter
            )))
        }
        Err(Failure::Merge {
            error: e,
            progressed,
        }) => {
            let now = Utc::now().timestamp_micros();
            let failing = note_failure(data_dir, &e, progressed, now)?;
            if now - failing.since >= ABANDON_AFTER_MICROS {
                abandon(sqlite, data_dir)?;
                return Err(e.context(format!(
                    "the snapshot from {} failed to merge for a day and was dropped; the next \
                     sync gets another",
                    merging.exporter
                )));
            }
            Err(e)
        }
    }
}

/// `progressed` says whether the attempt applied rows: only then does a
/// failure or pause start a new day before the merge is dropped.
enum Step {
    Done(MergeOutcome),
    Paused { progressed: bool },
}

/// `Spool` is a fault of the spooled snapshot itself, which no retry
/// fixes; `Merge` is one of this store, which may clear.
enum Failure {
    Spool(anyhow::Error),
    Merge {
        error: anyhow::Error,
        progressed: bool,
    },
}

impl Failure {
    fn merge(error: anyhow::Error) -> Self {
        Failure::Merge {
            error,
            progressed: false,
        }
    }
}

fn identity_or_merge(e: anyhow::Error) -> Failure {
    if e.downcast_ref::<IdentityMismatch>().is_some() {
        Failure::Spool(e)
    } else {
        Failure::merge(e)
    }
}

fn merge_claimed(
    sqlite: &SqliteStore,
    data_dir: &Path,
    gate: &WriteGate,
    merging: &Merging,
) -> Result<Step, Failure> {
    let spool_path = data_dir.join(SPOOL);
    let keyring = Keyring::load(sqlite, data_dir).map_err(Failure::merge)?;
    let key_for = key_for(&keyring);
    let (header, manifest) = verify(&spool_path, &key_for).map_err(Failure::Spool)?;
    if header.snapshot_id != merging.snapshot_id {
        return Err(Failure::Spool(anyhow!(
            "the spool holds snapshot {}, not {}",
            header.snapshot_id,
            merging.snapshot_id
        )));
    }
    if let Some(keys) = &manifest.keys {
        keyring
            .check_peer(keys, &header.exporter)
            .map_err(identity_or_merge)?;
    }
    keyring
        .opened_snapshot(&header.exporter, &header.epoch)
        .map_err(identity_or_merge)?;
    let mut merge = sqlite
        .begin_snapshot_merge(&header.snapshot_id, &header.exporter, merging.seq)
        .map_err(Failure::merge)?;
    let rows_before = merge.progress().rows_done;
    let (mut push_failed, mut paused) = (false, false);
    let streamed = spool_rows(&spool_path, &key_for, &mut |row| {
        if let Err(e) = merge.push(row) {
            push_failed = true;
            return Err(e);
        }
        if gate.is_closed() {
            paused = true;
            bail!("store maintenance closed the write gate");
        }
        Ok(())
    });
    let progressed = merge.progress().rows_done > rows_before;
    match streamed {
        Ok(()) => {}
        Err(_) if paused => return Ok(Step::Paused { progressed }),
        Err(error) if push_failed => return Err(Failure::Merge { error, progressed }),
        Err(e) => return Err(Failure::Spool(e)),
    }
    if let Some(keys) = &manifest.keys {
        keyring
            .accept(&header.exporter, keys, Utc::now().timestamp_micros())
            .map_err(identity_or_merge)?;
    }
    let outcome = merge
        .finish(&manifest.state.cursors)
        .map_err(Failure::merge)?;
    Ok(Step::Done(outcome))
}

/// How long the spooled merge has been failing, kept beside the spool so a
/// restart does not reset the day it gets before it is dropped.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct MergeFailing {
    pub since: i64,
    pub attempts: u32,
    pub error: String,
}

/// The spooled merge, for `/health`.
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct MergeStatus {
    pub merging: Merging,
    pub failing: Option<MergeFailing>,
}

pub fn status(data_dir: &Path) -> Result<Option<MergeStatus>> {
    let Some(merging) = merging(data_dir)? else {
        return Ok(None);
    };
    Ok(Some(MergeStatus {
        merging,
        failing: failing(data_dir)?,
    }))
}

fn failing(data_dir: &Path) -> Result<Option<MergeFailing>> {
    match fs::read(data_dir.join(SPOOL_FAILING)) {
        Ok(bytes) => Ok(Some(
            serde_json::from_slice(&bytes).context("reading the snapshot merge's failures")?,
        )),
        Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
        Err(e) => Err(e.into()),
    }
}

fn note_failure(
    data_dir: &Path,
    error: &anyhow::Error,
    progressed: bool,
    now: i64,
) -> Result<MergeFailing> {
    let failing = next_failing(failing(data_dir)?, progressed, error, now);
    write_atomically(
        &data_dir.join(SPOOL_FAILING),
        &serde_json::to_vec(&failing)?,
    )?;
    Ok(failing)
}

fn next_failing(
    before: Option<MergeFailing>,
    progressed: bool,
    error: &anyhow::Error,
    now: i64,
) -> MergeFailing {
    match before.filter(|_| !progressed) {
        Some(before) => MergeFailing {
            since: before.since.min(now),
            attempts: before.attempts.saturating_add(1),
            error: format!("{error:#}"),
        },
        None => MergeFailing {
            since: now,
            attempts: 1,
            error: format!("{error:#}"),
        },
    }
}

fn write_atomically(path: &Path, bytes: &[u8]) -> Result<()> {
    let mut name = path.file_name().unwrap_or_default().to_os_string();
    name.push(".tmp");
    let tmp = path.with_file_name(name);
    fs::write(&tmp, bytes)?;
    fs::rename(&tmp, path)?;
    Ok(())
}

fn remove_if_present(path: &Path) -> Result<()> {
    match fs::remove_file(path) {
        Ok(()) => Ok(()),
        Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
        Err(e) => Err(e).with_context(|| format!("removing {}", path.display())),
    }
}

/// The info file goes last, so a crash part way leaves a spool that the
/// next attempt finds and clears again.
fn clear_spool(data_dir: &Path) -> Result<()> {
    remove_if_present(&data_dir.join(SPOOL_FAILING))?;
    remove_if_present(&data_dir.join(SPOOL))?;
    remove_if_present(&data_dir.join(SPOOL_INFO))
}

fn abandon(sqlite: &SqliteStore, data_dir: &Path) -> Result<()> {
    sqlite.abandon_snapshot_merge()?;
    clear_spool(data_dir)
}

/// Each step runs whatever the one before returned, and the first error
/// is the one reported.
fn upkeep(sqlite: &SqliteStore, data_dir: &Path, gate: &WriteGate, health: &Health) -> Result<()> {
    let merged = merge_spooled(sqlite, data_dir, gate).map(|_| ());
    let reported = status(data_dir).map(|merge| health.snapshot_merge_checked(merge, Utc::now()));
    let replayed = replay(sqlite, gate);
    merged.and(reported).and(replayed)
}

/// Settled records replay once per binary and schema, so a fix shipped
/// later recovers them, and are dropped by age or past the cap.
fn replay(sqlite: &SqliteStore, gate: &WriteGate) -> Result<()> {
    let Some(_slot) = gate.try_enter() else {
        return Ok(());
    };
    if let Some(replayed) = sqlite.replay_settled_records(&replay_build())?
        && replayed.recovered + replayed.still_failing > 0
    {
        tracing::info!(
            recovered = replayed.recovered,
            still_failing = replayed.still_failing,
            "replayed settled sync records"
        );
    }
    let pruned = sqlite.prune_settled_records(Utc::now().timestamp_micros())?;
    if pruned.aged + pruned.capped > 0 {
        tracing::info!(
            aged = pruned.aged,
            capped = pruned.capped,
            "dropped settled sync records"
        );
    }
    Ok(())
}

/// Retries an unfinished merge every minute until it clears or is dropped,
/// reports it to `/health`, and replays settled records after an upgrade.
pub fn spawn_resumer(
    sqlite: Arc<SqliteStore>,
    data_dir: PathBuf,
    gate: Arc<WriteGate>,
    health: Arc<Health>,
) -> JoinHandle<()> {
    tokio::spawn(async move {
        let mut interval = tokio::time::interval(RESUME_INTERVAL);
        loop {
            interval.tick().await;
            let (sqlite, data_dir, gate, health) = (
                sqlite.clone(),
                data_dir.clone(),
                gate.clone(),
                health.clone(),
            );
            let done =
                tokio::task::spawn_blocking(move || upkeep(&sqlite, &data_dir, &gate, &health))
                    .await;
            match done {
                Ok(Ok(())) => {}
                Ok(Err(e)) => tracing::warn!("snapshot merge or settled replay failed: {e:#}"),
                Err(e) => tracing::warn!("snapshot merge or settled replay panicked: {e}"),
            }
        }
    })
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn failures_keep_their_first_time_across_attempts_and_clear_with_the_spool() {
        let dir = tempfile::tempdir().unwrap();
        let first = note_failure(dir.path(), &anyhow!("disk full"), false, 100).unwrap();
        let second = note_failure(dir.path(), &anyhow!("still full"), false, 250).unwrap();

        assert_eq!((first.since, first.attempts), (100, 1));
        assert_eq!((second.since, second.attempts), (100, 2));
        assert_eq!(second.error, "still full");
        assert_eq!(failing(dir.path()).unwrap(), Some(second));

        clear_spool(dir.path()).unwrap();
        assert_eq!(failing(dir.path()).unwrap(), None);
    }

    #[test]
    fn a_failure_after_rows_were_applied_starts_a_new_day() {
        let before = MergeFailing {
            since: 100,
            attempts: 4,
            error: "disk full".to_string(),
        };
        let stuck = next_failing(Some(before.clone()), false, &anyhow!("again"), 900);
        let moved = next_failing(Some(before), true, &anyhow!("again"), 900);

        assert_eq!((stuck.since, stuck.attempts), (100, 5));
        assert_eq!((moved.since, moved.attempts), (900, 1));
    }
}