Skip to main content

second_brain_sync/
import.rs

1use std::collections::BTreeMap;
2use std::io::BufRead;
3
4use anyhow::Result;
5
6use second_brain_core::schema::SyncOp;
7use second_brain_core::store::{ApplyOutcome, StoreAdmin};
8
9use crate::export::SyncRecord;
10
11pub struct ImportStats {
12    pub created: u64,
13    pub updated: u64,
14    pub deleted: u64,
15    pub skipped: u64,
16    pub errors: u64,
17    /// Skips broken out by cause. `dangling_endpoint` is the one that means data
18    /// loss rather than convergence: the edge can never apply without operator
19    /// action, so a non-zero count here needs looking at.
20    pub skipped_by_reason: BTreeMap<String, u64>,
21}
22
23pub fn import_changes<R: BufRead>(store: &dyn StoreAdmin, reader: R) -> Result<(ImportStats, u64)> {
24    let mut stats = ImportStats {
25        created: 0,
26        updated: 0,
27        deleted: 0,
28        skipped: 0,
29        errors: 0,
30        skipped_by_reason: BTreeMap::new(),
31    };
32    let mut max_seq = 0u64;
33    let mut errored_seqs: Vec<u64> = Vec::new();
34
35    let mut records: Vec<SyncRecord> = Vec::new();
36    for line in reader.lines() {
37        let line = line?;
38        if line.trim().is_empty() {
39            continue;
40        }
41        match serde_json::from_str::<SyncRecord>(&line) {
42            Ok(r) => records.push(r),
43            Err(e) => {
44                tracing::warn!("skipping malformed sync record: {e}");
45                stats.errors += 1;
46            }
47        }
48    }
49
50    for record in &records {
51        if record.local_seq > max_seq {
52            max_seq = record.local_seq;
53        }
54    }
55
56    // Apply runs of consecutive Create records through the atomic batch path so a
57    // whole run commits with one fsync instead of one per record (per-record commits
58    // throttle import to a few records/sec on slow storage). Non-create ops (update,
59    // delete) flush the open create run and apply per-record through apply_record
60    // to preserve their conflict-resolution semantics unchanged.
61    const TX_CHUNK: usize = 500;
62    let mut i = 0;
63    while i < records.len() {
64        let record = &records[i];
65        if matches!(record.op, SyncOp::Create) {
66            let end = (i + TX_CHUNK).min(records.len());
67            let mut j = i;
68            while j < end && matches!(records[j].op, SyncOp::Create) {
69                j += 1;
70            }
71            let chunk = &records[i..j];
72            match store.apply_batch(chunk) {
73                Ok(outcomes) => {
74                    for outcome in outcomes {
75                        count_outcome(outcome, &mut stats);
76                    }
77                }
78                // apply_batch is all-or-nothing, so on error nothing from the
79                // chunk landed and each record retries individually.
80                Err(_) => {
81                    for rec in chunk {
82                        record_outcome(store, rec, &mut stats, &mut errored_seqs);
83                    }
84                }
85            }
86            i = j;
87        } else {
88            record_outcome(store, record, &mut stats, &mut errored_seqs);
89            i += 1;
90        }
91    }
92
93    let watermark = errored_seqs
94        .iter()
95        .min()
96        .map(|m| m.saturating_sub(1))
97        .unwrap_or(max_seq);
98
99    Ok((stats, watermark))
100}
101
102fn count_outcome(outcome: ApplyOutcome, stats: &mut ImportStats) {
103    match outcome {
104        ApplyOutcome::Created => stats.created += 1,
105        ApplyOutcome::Updated => stats.updated += 1,
106        ApplyOutcome::Deleted => stats.deleted += 1,
107        ApplyOutcome::Skipped(reason) => {
108            stats.skipped += 1;
109            *stats
110                .skipped_by_reason
111                .entry(reason.as_str().to_string())
112                .or_insert(0) += 1;
113        }
114    }
115}
116
117fn record_outcome(
118    store: &dyn StoreAdmin,
119    record: &SyncRecord,
120    stats: &mut ImportStats,
121    errored_seqs: &mut Vec<u64>,
122) {
123    match apply_with_retry(store, record) {
124        Ok(outcome) => count_outcome(outcome, stats),
125        Err(e) => {
126            tracing::warn!(
127                "error applying sync record local_seq={}: {e}",
128                record.local_seq
129            );
130            stats.errors += 1;
131            errored_seqs.push(record.local_seq);
132        }
133    }
134}
135
136fn apply_with_retry(store: &dyn StoreAdmin, record: &SyncRecord) -> Result<ApplyOutcome> {
137    const MAX_ATTEMPTS: u32 = 5;
138    let mut last_err = None;
139    for attempt in 0..MAX_ATTEMPTS {
140        match store.apply_record(record) {
141            Ok(outcome) => return Ok(outcome),
142            Err(e) => {
143                tracing::debug!(
144                    "transient error on attempt {} for local_seq={}: {e}",
145                    attempt + 1,
146                    record.local_seq,
147                );
148                last_err = Some(e);
149                std::thread::sleep(std::time::Duration::from_millis(50 * (1 << attempt)));
150            }
151        }
152    }
153    Err(last_err.unwrap())
154}