Skip to main content

second_brain_core/sqlite_store/
settled.rs

1use anyhow::{Context, Result};
2use rusqlite::{Connection, ErrorCode, OptionalExtension, Transaction, params};
3use serde::{Deserialize, Serialize};
4
5use super::clock::is_new_layout;
6use super::floor::DAY_MICROS;
7use super::{SqliteStore, sync_dedup};
8use crate::schema::SyncRecord;
9
10/// A content-deterministic failure settles only after this many attempts
11/// spread over at least `SETTLE_AFTER_MICROS`.
12pub(crate) const SETTLE_AFTER_ATTEMPTS: u32 = 10;
13pub(crate) const SETTLE_AFTER_MICROS: i64 = DAY_MICROS;
14
15const FAILED: &str = "sync_failed:";
16const REPLAYED_FOR: &str = "sync_settled_replayed_for";
17const REPLAY_PAGE: usize = 100;
18const DROPPED: &str = "sync_settled_dropped";
19
20/// A settled record this old that a newer build has replayed without
21/// recovering is dropped.
22pub(crate) const SETTLED_KEEP_MICROS: i64 = 90 * DAY_MICROS;
23/// Settled records past this many are dropped oldest first, so a device
24/// that never upgrades stays bounded.
25pub(crate) const SETTLED_CAP: u64 = 10_000;
26
27/// The binary and schema a replay runs under, recorded on each record as
28/// it settles so only a replay by a later build counts towards dropping it.
29pub fn replay_build() -> String {
30    format!(
31        "{} schema {}",
32        env!("CARGO_PKG_VERSION"),
33        super::ddl::SCHEMA_VERSION
34    )
35}
36
37/// A record that failed to apply, kept with its content so it is retried
38/// here rather than pulled again, and so a later binary can replay it.
39#[derive(Debug, Clone, Serialize, Deserialize)]
40pub struct FailedRecord {
41    /// The peer whose log the record came from, or `snap:<id>` for a
42    /// snapshot row.
43    pub source: String,
44    /// The record's position in `source`'s numbering.
45    pub seq: u64,
46    pub record: SyncRecord,
47    pub error: String,
48    pub attempts: u32,
49    pub first_failed_at: i64,
50    /// Settled records no longer hold the source's ack back; they wait for
51    /// the replay after an upgrade.
52    pub settled: bool,
53    #[serde(default)]
54    pub settled_at: Option<i64>,
55    /// `replay_build` when the record settled.
56    #[serde(default)]
57    pub settled_build: Option<String>,
58    /// A replay by a build other than `settled_build` has failed on it.
59    #[serde(default)]
60    pub replayed_newer: bool,
61}
62
63#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
64pub struct FailedCounts {
65    pub failing: u64,
66    pub settled: u64,
67    /// Settled records dropped over the cap, ever.
68    pub dropped: u64,
69}
70
71#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
72pub struct PruneCounts {
73    pub aged: u64,
74    pub capped: u64,
75}
76
77#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
78pub struct RetryCounts {
79    pub applied: u64,
80    pub settled: u64,
81    pub failing: u64,
82}
83
84#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
85pub struct ReplayCounts {
86    pub recovered: u64,
87    pub still_failing: u64,
88}
89
90/// A busy writer or an I/O fault says nothing about the record, so it never
91/// settles; anything else (a parse failure, a constraint, an unsupported
92/// shape) fails the same way on every attempt. Parsers in this crate wrap
93/// their failures in `io::Error::other`, so those kinds are content.
94pub fn is_transient(error: &anyhow::Error) -> bool {
95    error.chain().any(|cause| {
96        if let Some(io) = cause.downcast_ref::<std::io::Error>() {
97            return !matches!(
98                io.kind(),
99                std::io::ErrorKind::Other
100                    | std::io::ErrorKind::InvalidData
101                    | std::io::ErrorKind::InvalidInput
102            );
103        }
104        match cause.downcast_ref::<rusqlite::Error>() {
105            Some(rusqlite::Error::SqliteFailure(e, _)) => matches!(
106                e.code,
107                ErrorCode::DatabaseBusy
108                    | ErrorCode::DatabaseLocked
109                    | ErrorCode::SystemIoFailure
110                    | ErrorCode::DiskFull
111                    | ErrorCode::OutOfMemory
112                    | ErrorCode::CannotOpen
113            ),
114            _ => false,
115        }
116    })
117}
118
119/// An error a caller applying records must stop on rather than settle: the
120/// clock, or the store itself.
121pub(super) fn is_outer(error: &anyhow::Error) -> bool {
122    error.chain().any(|c| c.is::<super::ClockUntrusted>()) || is_transient(error)
123}
124
125fn prefix(source: &str) -> String {
126    format!("{FAILED}{source}:")
127}
128
129fn key(source: &str, seq: u64) -> String {
130    format!("{}{seq:020}", prefix(source))
131}
132
133/// The prefix's upper bound: `;` sorts right after `:`.
134fn prefix_end(source: &str) -> String {
135    format!("{FAILED}{source};")
136}
137
138pub(super) fn put(conn: &Connection, failed: &FailedRecord) -> Result<()> {
139    conn.execute(
140        "INSERT OR REPLACE INTO index_meta (key, value) VALUES (?1, ?2)",
141        params![
142            key(&failed.source, failed.seq),
143            serde_json::to_string(failed)?
144        ],
145    )?;
146    Ok(())
147}
148
149fn get(conn: &Connection, source: &str, seq: u64) -> Result<Option<FailedRecord>> {
150    let value: Option<String> = conn
151        .query_row(
152            "SELECT value FROM index_meta WHERE key = ?1",
153            params![key(source, seq)],
154            |r| r.get(0),
155        )
156        .optional()?;
157    value
158        .map(|v| serde_json::from_str(&v).context("reading a failed sync record"))
159        .transpose()
160}
161
162fn remove(conn: &Connection, source: &str, seq: u64) -> Result<()> {
163    conn.execute(
164        "DELETE FROM index_meta WHERE key = ?1",
165        params![key(source, seq)],
166    )?;
167    Ok(())
168}
169
170fn key_range(source: Option<&str>) -> (String, String) {
171    match source {
172        Some(source) => (prefix(source), prefix_end(source)),
173        None => (
174            FAILED.to_string(),
175            format!("{};", FAILED.trim_end_matches(':')),
176        ),
177    }
178}
179
180/// Drops every kept record of a deleted node, whose content must not
181/// outlive the delete; applying one again would be skipped anyway.
182pub(super) fn drop_failed_of(conn: &Connection, node_id: &str) -> Result<()> {
183    let (lo, hi) = key_range(None);
184    conn.execute(
185        "DELETE FROM index_meta WHERE key >= ?1 AND key < ?2 \
186         AND json_extract(value, '$.record.node_id') = ?3",
187        params![lo, hi, node_id],
188    )?;
189    Ok(())
190}
191
192fn all_failed(conn: &Connection, source: Option<&str>) -> Result<Vec<FailedRecord>> {
193    let (lo, hi) = key_range(source);
194    let mut stmt =
195        conn.prepare("SELECT value FROM index_meta WHERE key >= ?1 AND key < ?2 ORDER BY key")?;
196    let rows = stmt.query_map(params![lo, hi], |r| r.get::<_, String>(0))?;
197    rows.map(|v| serde_json::from_str(&v?).context("reading a failed sync record"))
198        .collect()
199}
200
201fn settled_page(conn: &Connection, after: &str) -> Result<Vec<(String, FailedRecord)>> {
202    let (_, hi) = key_range(None);
203    let mut stmt = conn.prepare(
204        "SELECT key, value FROM index_meta WHERE key > ?1 AND key < ?2 \
205         AND json_extract(value, '$.settled') IS 1 ORDER BY key LIMIT ?3",
206    )?;
207    let rows = stmt.query_map(params![after, hi, REPLAY_PAGE as i64], |r| {
208        Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?))
209    })?;
210    rows.map(|row| {
211        let (key, value) = row?;
212        let failed = serde_json::from_str(&value).context("reading a failed sync record")?;
213        Ok((key, failed))
214    })
215    .collect()
216}
217
218/// A settled snapshot row: settled at once, since a merge never comes back
219/// to a row.
220pub(super) fn settle_on_tx(
221    tx: &Transaction<'_>,
222    source: &str,
223    seq: u64,
224    record: &SyncRecord,
225    error: &anyhow::Error,
226    now_micros: i64,
227) -> Result<()> {
228    put(
229        tx,
230        &FailedRecord {
231            source: source.to_string(),
232            seq,
233            record: record.clone(),
234            error: format!("{error:#}"),
235            attempts: 1,
236            first_failed_at: now_micros,
237            settled: true,
238            settled_at: Some(now_micros),
239            settled_build: Some(replay_build()),
240            replayed_newer: false,
241        },
242    )
243}
244
245/// Drops settled records that are past `SETTLED_KEEP_MICROS` and were
246/// replayed by a newer build, then the oldest settled records past `cap`.
247fn prune_on(tx: &Transaction<'_>, now_micros: i64, cap: u64) -> Result<PruneCounts> {
248    let (lo, hi) = key_range(None);
249    let aged = tx.execute(
250        "DELETE FROM index_meta WHERE key >= ?1 AND key < ?2 \
251         AND json_extract(value, '$.settled') IS 1 \
252         AND json_extract(value, '$.replayed_newer') IS 1 \
253         AND COALESCE(json_extract(value, '$.settled_at'), \
254                      json_extract(value, '$.first_failed_at')) <= ?3",
255        params![lo, hi, now_micros - SETTLED_KEEP_MICROS],
256    )?;
257    let settled: i64 = tx.query_row(
258        "SELECT COUNT(*) FROM index_meta WHERE key >= ?1 AND key < ?2 \
259         AND json_extract(value, '$.settled') IS 1",
260        params![lo, hi],
261        |r| r.get(0),
262    )?;
263    let over = u64::try_from(settled)?.saturating_sub(cap);
264    let capped = if over == 0 {
265        0
266    } else {
267        tx.execute(
268            "DELETE FROM index_meta WHERE key IN (SELECT key FROM index_meta \
269             WHERE key >= ?1 AND key < ?2 AND json_extract(value, '$.settled') IS 1 \
270             ORDER BY COALESCE(json_extract(value, '$.settled_at'), \
271                               json_extract(value, '$.first_failed_at')), key \
272             LIMIT ?3)",
273            params![lo, hi, i64::try_from(over)?],
274        )?
275    };
276    if capped > 0 {
277        tx.execute(
278            "INSERT INTO index_meta (key, value) VALUES (?1, ?2) \
279             ON CONFLICT(key) DO UPDATE SET value = CAST(value AS INTEGER) + excluded.value",
280            params![DROPPED, capped as i64],
281        )?;
282    }
283    Ok(PruneCounts {
284        aged: aged as u64,
285        capped: capped as u64,
286    })
287}
288
289impl SqliteStore {
290    /// Keeps a record from `source`'s log that failed to apply, or counts
291    /// one more attempt on one already kept. A content-deterministic
292    /// failure settles after `SETTLE_AFTER_ATTEMPTS` attempts spread over a
293    /// day; a transient one never does.
294    pub fn note_failed_record(
295        &self,
296        source: &str,
297        record: &SyncRecord,
298        error: &anyhow::Error,
299        now_micros: i64,
300    ) -> Result<()> {
301        let conn = self.conn();
302        let tx = conn.unchecked_transaction()?;
303        let seq = record.local_seq;
304        let mut failed = get(&tx, source, seq)?.unwrap_or(FailedRecord {
305            source: source.to_string(),
306            seq,
307            record: record.clone(),
308            error: String::new(),
309            attempts: 0,
310            first_failed_at: now_micros,
311            settled: false,
312            settled_at: None,
313            settled_build: None,
314            replayed_newer: false,
315        });
316        failed.attempts = failed.attempts.saturating_add(1);
317        failed.error = format!("{error:#}");
318        let was_settled = failed.settled;
319        failed.settled = !is_transient(error)
320            && failed.attempts >= SETTLE_AFTER_ATTEMPTS
321            && now_micros - failed.first_failed_at >= SETTLE_AFTER_MICROS;
322        if failed.settled && !was_settled {
323            failed.settled_at = Some(now_micros);
324            failed.settled_build = Some(replay_build());
325        }
326        put(&tx, &failed)?;
327        tx.commit()?;
328        Ok(())
329    }
330
331    /// Applies again every unsettled record kept from `source`; one that
332    /// now applies or settles stops holding `source`'s ack back.
333    pub fn retry_failed_records(&self, source: &str, now_micros: i64) -> Result<RetryCounts> {
334        let pending: Vec<FailedRecord> = all_failed(&self.reader(), Some(source))?
335            .into_iter()
336            .filter(|f| !f.settled)
337            .collect();
338        let mut counts = RetryCounts::default();
339        for failed in pending {
340            match crate::store::StoreAdmin::apply_record(self, &failed.record) {
341                Ok(_) => {
342                    remove(&self.conn(), source, failed.seq)?;
343                    counts.applied += 1;
344                }
345                Err(e) if e.chain().any(|c| c.is::<super::ClockUntrusted>()) => return Err(e),
346                Err(e) => {
347                    self.note_failed_record(source, &failed.record, &e, now_micros)?;
348                    let settled =
349                        get(&self.reader(), source, failed.seq)?.is_some_and(|f| f.settled);
350                    if settled {
351                        counts.settled += 1;
352                    } else {
353                        counts.failing += 1;
354                    }
355                }
356            }
357        }
358        Ok(counts)
359    }
360
361    /// The highest seq of `source`'s log applied here without a gap: below
362    /// the first record from it that still fails and has not settled.
363    pub fn applied_through(&self, source: &str, reached: u64) -> Result<u64> {
364        let first_failing: Option<i64> = self
365            .reader()
366            .query_row(
367                "SELECT json_extract(value, '$.seq') FROM index_meta \
368                 WHERE key >= ?1 AND key < ?2 AND json_extract(value, '$.settled') IS NOT 1 \
369                 ORDER BY key LIMIT 1",
370                params![prefix(source), prefix_end(source)],
371                |r| r.get(0),
372            )
373            .optional()?;
374        Ok(match first_failing {
375            Some(seq) => reached.min(u64::try_from(seq)?.saturating_sub(1)),
376            None => reached,
377        })
378    }
379
380    pub fn failed_records(&self) -> Result<Vec<FailedRecord>> {
381        all_failed(&self.reader(), None)
382    }
383
384    pub fn failed_record_counts(&self) -> Result<FailedCounts> {
385        let (lo, hi) = key_range(None);
386        let (total, settled): (i64, i64) = self.reader().query_row(
387            "SELECT COUNT(*), COALESCE(SUM(json_extract(value, '$.settled') IS 1), 0) \
388             FROM index_meta WHERE key >= ?1 AND key < ?2",
389            params![lo, hi],
390            |r| Ok((r.get(0)?, r.get(1)?)),
391        )?;
392        let dropped: Option<i64> = self
393            .reader()
394            .query_row(
395                "SELECT CAST(value AS INTEGER) FROM index_meta WHERE key = ?1",
396                params![DROPPED],
397                |r| r.get(0),
398            )
399            .optional()?;
400        Ok(FailedCounts {
401            failing: u64::try_from(total - settled)?,
402            settled: u64::try_from(settled)?,
403            dropped: u64::try_from(dropped.unwrap_or(0))?,
404        })
405    }
406
407    /// Applies every settled record again once per `build`, so a bug fixed
408    /// in a later binary or schema recovers records whose source log may
409    /// no longer hold them. A record that now lands is logged as a relay
410    /// under `replay:<id>:<n>`, numbered densely, so it reaches this store's
411    /// pullers. Each page of records is its own transaction and the build is
412    /// marked done after the last page, so a retry after a crash replays
413    /// what is left under a new id.
414    pub fn replay_settled_records(&self, build: &str) -> Result<Option<ReplayCounts>> {
415        let done: Option<String> = self
416            .reader()
417            .query_row(
418                "SELECT value FROM index_meta WHERE key = ?1",
419                params![REPLAYED_FOR],
420                |r| r.get(0),
421            )
422            .optional()?;
423        if done.as_deref() == Some(build) {
424            return Ok(None);
425        }
426        let replay_id = uuid::Uuid::new_v4().to_string();
427        let origin = format!("replay:{replay_id}");
428        let mut counts = ReplayCounts::default();
429        let mut n = 0u64;
430        let mut after = FAILED.to_string();
431        loop {
432            let page = self.with_txn(|tx| {
433                let page = settled_page(tx, &after)?;
434                for (_, failed) in &page {
435                    self.replay_one(tx, failed, build, &origin, &mut n, &mut counts)?;
436                }
437                Ok(page)
438            })?;
439            match page.last() {
440                Some((key, _)) if page.len() == REPLAY_PAGE => after.clone_from(key),
441                _ => break,
442            }
443        }
444        self.conn().execute(
445            "INSERT OR REPLACE INTO index_meta (key, value) VALUES (?1, ?2)",
446            params![REPLAYED_FOR, build],
447        )?;
448        Ok(Some(counts))
449    }
450
451    /// Drops settled records that a newer build could not recover once
452    /// they are 90 days old, and the oldest past `SETTLED_CAP`.
453    pub fn prune_settled_records(&self, now_micros: i64) -> Result<PruneCounts> {
454        self.with_txn(|tx| prune_on(tx, now_micros, SETTLED_CAP))
455    }
456
457    fn replay_one(
458        &self,
459        tx: &Transaction<'_>,
460        failed: &FailedRecord,
461        build: &str,
462        origin: &str,
463        n: &mut u64,
464        counts: &mut ReplayCounts,
465    ) -> Result<()> {
466        let relay = relay_record(&failed.record, origin, *n);
467        match self.apply_logged_on_tx(tx, &relay)? {
468            Ok(logged) => {
469                if logged {
470                    *n += 1;
471                }
472                let original = &failed.record;
473                if is_new_layout(&original.origin_machine_id, original.origin_seq as i64) {
474                    sync_dedup::record(
475                        tx,
476                        &original.origin_machine_id,
477                        original.origin_seq as i64,
478                    )?;
479                }
480                remove(tx, &failed.source, failed.seq)?;
481                counts.recovered += 1;
482            }
483            Err(_) => {
484                counts.still_failing += 1;
485                if !failed.replayed_newer && failed.settled_build.as_deref() != Some(build) {
486                    put(
487                        tx,
488                        &FailedRecord {
489                            replayed_newer: true,
490                            ..failed.clone()
491                        },
492                    )?;
493                }
494            }
495        }
496        Ok(())
497    }
498
499    /// Applies `record` inside a savepoint. The outer error is one the
500    /// caller must stop on (the clock, the store itself); the inner one is
501    /// the record's own failure, already rolled back. `Ok(true)` means the
502    /// record was logged under its key.
503    pub(super) fn apply_logged_on_tx(
504        &self,
505        tx: &Transaction<'_>,
506        record: &SyncRecord,
507    ) -> Result<std::result::Result<bool, anyhow::Error>> {
508        tx.execute_batch("SAVEPOINT apply_one")?;
509        match self.apply_on_tx(tx, record) {
510            Ok(_) => {
511                let logged = logged(tx, &record.origin_machine_id, record.origin_seq)?;
512                if logged {
513                    tx.execute_batch("RELEASE apply_one")?;
514                } else {
515                    tx.execute_batch("ROLLBACK TO apply_one; RELEASE apply_one")?;
516                }
517                Ok(Ok(logged))
518            }
519            Err(e) => {
520                tx.execute_batch("ROLLBACK TO apply_one; RELEASE apply_one")?;
521                if is_outer(&e) {
522                    return Err(e);
523                }
524                Ok(Err(e))
525            }
526        }
527    }
528}
529
530fn logged(conn: &Connection, origin: &str, origin_seq: u64) -> Result<bool> {
531    Ok(conn
532        .query_row(
533            "SELECT 1 FROM sync_log WHERE id = ?1",
534            params![format!("{origin}:{origin_seq}")],
535            |_| Ok(()),
536        )
537        .optional()?
538        .is_some())
539}
540
541/// The relay keeps the record's authored time and author: `updated_by`
542/// travels in `data`, so a relay under another origin orders the same.
543pub(super) fn relay_record(record: &SyncRecord, origin: &str, n: u64) -> SyncRecord {
544    let mut relay = record.clone();
545    if let Some(object) = relay.data.as_mut().and_then(|d| d.as_object_mut()) {
546        object
547            .entry(super::authored_order::UPDATED_BY.to_string())
548            .or_insert_with(|| record.origin_machine_id.clone().into());
549    }
550    relay.origin_machine_id = origin.to_string();
551    relay.origin_seq = n;
552    relay
553}
554
555#[cfg(test)]
556mod tests {
557    use std::io::{Error, ErrorKind};
558
559    use super::*;
560
561    #[test]
562    fn an_io_fault_is_transient() {
563        for kind in [
564            ErrorKind::StorageFull,
565            ErrorKind::PermissionDenied,
566            ErrorKind::TimedOut,
567        ] {
568            assert!(is_transient(&anyhow::Error::new(Error::new(kind, "x"))));
569        }
570    }
571
572    #[test]
573    fn an_io_error_that_describes_bad_content_is_not_transient() {
574        for kind in [
575            ErrorKind::Other,
576            ErrorKind::InvalidData,
577            ErrorKind::InvalidInput,
578        ] {
579            assert!(!is_transient(&anyhow::Error::new(Error::new(kind, "x"))));
580        }
581        let parse = super::super::parse_relation_type("not_a_relation_type").unwrap_err();
582        assert!(!is_transient(&anyhow::Error::new(parse)));
583    }
584
585    fn settled_at(source: &str, seq: u64, at: i64) -> FailedRecord {
586        FailedRecord {
587            source: source.to_string(),
588            seq,
589            record: SyncRecord {
590                local_seq: seq,
591                origin_machine_id: source.to_string(),
592                origin_seq: seq,
593                op: crate::schema::SyncOp::Create,
594                node_type: crate::schema::SyncNodeType::Memory,
595                node_id: format!("node-{seq}"),
596                timestamp: chrono::Utc::now(),
597                data: None,
598            },
599            error: "bad".to_string(),
600            attempts: 10,
601            first_failed_at: at,
602            settled: true,
603            settled_at: Some(at),
604            settled_build: Some(replay_build()),
605            replayed_newer: false,
606        }
607    }
608
609    #[test]
610    fn past_the_cap_the_oldest_settled_records_go_and_are_counted() {
611        let store = SqliteStore::in_memory("self".to_string()).unwrap();
612        let conn = store.conn();
613        for (seq, at) in [(1, 500), (2, 100), (3, 400), (4, 200), (5, 300)] {
614            put(&conn, &settled_at("peer", seq, at)).unwrap();
615        }
616        let mut failing = settled_at("peer", 6, 50);
617        failing.settled = false;
618        put(&conn, &failing).unwrap();
619        drop(conn);
620
621        let pruned = store.with_txn(|tx| prune_on(tx, 1_000, 3)).unwrap();
622
623        assert_eq!(pruned, PruneCounts { aged: 0, capped: 2 });
624        let left: Vec<u64> = store
625            .failed_records()
626            .unwrap()
627            .into_iter()
628            .map(|f| f.seq)
629            .collect();
630        assert_eq!(left, vec![1, 3, 5, 6]);
631        let counts = store.failed_record_counts().unwrap();
632        assert_eq!((counts.settled, counts.failing, counts.dropped), (3, 1, 2));
633    }
634
635    #[test]
636    fn counts_read_the_settled_flag_without_decoding_the_record() {
637        let store =
638            SqliteStore::in_memory("00000000-0000-4000-8000-000000000000".to_string()).unwrap();
639        for (seq, settled) in [(1u64, true), (2, false), (3, true)] {
640            let value = serde_json::json!({
641                "settled": settled,
642                "record": { "op": "written by a later build" },
643            });
644            store
645                .conn()
646                .execute(
647                    "INSERT INTO index_meta (key, value) VALUES (?1, ?2)",
648                    params![key("peer", seq), value.to_string()],
649                )
650                .unwrap();
651        }
652
653        assert_eq!(
654            store.failed_record_counts().unwrap(),
655            FailedCounts {
656                failing: 1,
657                settled: 2,
658                dropped: 0,
659            }
660        );
661    }
662}