Skip to main content

second_brain_sync/
import.rs

1use std::collections::BTreeMap;
2use std::io::BufRead;
3
4use anyhow::Result;
5
6use second_brain_core::schema::SyncOp;
7use second_brain_core::sqlite_store::ClockUntrusted;
8use second_brain_core::store::{ApplyOutcome, StoreAdmin};
9
10use crate::export::SyncRecord;
11
12pub struct ImportStats {
13    pub created: u64,
14    pub updated: u64,
15    pub deleted: u64,
16    pub skipped: u64,
17    pub errors: u64,
18    /// Skips broken out by cause. `dangling_endpoint` is the one that means data
19    /// loss rather than convergence: the edge can never apply without operator
20    /// action, so a non-zero count here needs looking at.
21    pub skipped_by_reason: BTreeMap<String, u64>,
22    /// The highest seq the stream carried, past failed records.
23    pub reached: u64,
24    /// Records that failed every in-call attempt, with their last error, for
25    /// the store to keep and retry.
26    pub failed: Vec<(SyncRecord, anyhow::Error)>,
27}
28
29pub fn import_changes<R: BufRead>(store: &dyn StoreAdmin, reader: R) -> Result<(ImportStats, u64)> {
30    let mut stats = ImportStats {
31        created: 0,
32        updated: 0,
33        deleted: 0,
34        skipped: 0,
35        errors: 0,
36        skipped_by_reason: BTreeMap::new(),
37        reached: 0,
38        failed: Vec::new(),
39    };
40    let mut max_seq = 0u64;
41    let mut errored_seqs: Vec<u64> = Vec::new();
42
43    let mut records: Vec<SyncRecord> = Vec::new();
44    for line in reader.lines() {
45        let line = line?;
46        if line.trim().is_empty() {
47            continue;
48        }
49        match serde_json::from_str::<SyncRecord>(&line) {
50            Ok(r) => records.push(r),
51            Err(e) => {
52                tracing::warn!("skipping malformed sync record: {e}");
53                stats.errors += 1;
54            }
55        }
56    }
57
58    for record in &records {
59        if record.local_seq > max_seq {
60            max_seq = record.local_seq;
61        }
62    }
63
64    // Apply runs of consecutive Create records through the atomic batch path so a
65    // whole run commits with one fsync instead of one per record (per-record commits
66    // throttle import to a few records/sec on slow storage). Non-create ops (update,
67    // delete) flush the open create run and apply per-record through apply_record
68    // to preserve their conflict-resolution semantics unchanged.
69    const TX_CHUNK: usize = 500;
70    let mut i = 0;
71    while i < records.len() {
72        let record = &records[i];
73        if matches!(record.op, SyncOp::Create) {
74            let end = (i + TX_CHUNK).min(records.len());
75            let mut j = i;
76            while j < end && matches!(records[j].op, SyncOp::Create) {
77                j += 1;
78            }
79            let chunk = &records[i..j];
80            match store.apply_batch(chunk) {
81                Ok(outcomes) => {
82                    for outcome in outcomes {
83                        count_outcome(outcome, &mut stats);
84                    }
85                }
86                // apply_batch is all-or-nothing, so on error nothing from the
87                // chunk landed and each record retries individually.
88                Err(_) => {
89                    for rec in chunk {
90                        record_outcome(store, rec, &mut stats, &mut errored_seqs)?;
91                    }
92                }
93            }
94            i = j;
95        } else {
96            record_outcome(store, record, &mut stats, &mut errored_seqs)?;
97            i += 1;
98        }
99    }
100
101    stats.reached = max_seq;
102    let watermark = errored_seqs
103        .iter()
104        .min()
105        .map(|m| m.saturating_sub(1))
106        .unwrap_or(max_seq);
107
108    Ok((stats, watermark))
109}
110
111pub fn import_values(
112    store: &dyn StoreAdmin,
113    records: &[serde_json::Value],
114) -> Result<(ImportStats, u64)> {
115    let mut jsonl = String::new();
116    for record in records {
117        jsonl.push_str(&serde_json::to_string(record)?);
118        jsonl.push('\n');
119    }
120    let reader = std::io::BufReader::new(jsonl.as_bytes());
121    import_changes(store, reader)
122}
123
124fn count_outcome(outcome: ApplyOutcome, stats: &mut ImportStats) {
125    match outcome {
126        ApplyOutcome::Created => stats.created += 1,
127        ApplyOutcome::Updated => stats.updated += 1,
128        ApplyOutcome::Deleted => stats.deleted += 1,
129        ApplyOutcome::Skipped(reason) => {
130            stats.skipped += 1;
131            *stats
132                .skipped_by_reason
133                .entry(reason.as_str().to_string())
134                .or_insert(0) += 1;
135        }
136    }
137}
138
139/// A store that cannot mint a seq fails the whole import rather than counting
140/// an error: every later record would fail the same way, and the sender must
141/// keep its cursor to resend them once the clock is trusted.
142fn record_outcome(
143    store: &dyn StoreAdmin,
144    record: &SyncRecord,
145    stats: &mut ImportStats,
146    errored_seqs: &mut Vec<u64>,
147) -> Result<()> {
148    match apply_with_retry(store, record) {
149        Ok(outcome) => count_outcome(outcome, stats),
150        Err(e) if clock_untrusted(&e) => return Err(e),
151        Err(e) => {
152            tracing::warn!(
153                "error applying sync record local_seq={}: {e}",
154                record.local_seq
155            );
156            stats.errors += 1;
157            errored_seqs.push(record.local_seq);
158            stats.failed.push((record.clone(), e));
159        }
160    }
161    Ok(())
162}
163
164fn clock_untrusted(error: &anyhow::Error) -> bool {
165    error.chain().any(|cause| cause.is::<ClockUntrusted>())
166}
167
168fn apply_with_retry(store: &dyn StoreAdmin, record: &SyncRecord) -> Result<ApplyOutcome> {
169    const MAX_ATTEMPTS: u32 = 5;
170    let mut last_err = None;
171    for attempt in 0..MAX_ATTEMPTS {
172        match store.apply_record(record) {
173            Ok(outcome) => return Ok(outcome),
174            Err(e) if clock_untrusted(&e) => return Err(e),
175            Err(e) => {
176                tracing::debug!(
177                    "transient error on attempt {} for local_seq={}: {e}",
178                    attempt + 1,
179                    record.local_seq,
180                );
181                last_err = Some(e);
182                std::thread::sleep(std::time::Duration::from_millis(50 * (1 << attempt)));
183            }
184        }
185    }
186    Err(last_err.unwrap())
187}