Skip to main content

second_brain_sync/
import.rs

1use std::collections::BTreeMap;
2use std::io::BufRead;
3
4use anyhow::Result;
5
6use second_brain_core::schema::SyncOp;
7use second_brain_core::store::{ApplyOutcome, StoreAdmin};
8
9use crate::export::SyncRecord;
10
11pub struct ImportStats {
12    pub created: u64,
13    pub updated: u64,
14    pub deleted: u64,
15    pub skipped: u64,
16    pub errors: u64,
17    /// Skips broken out by cause. `dangling_endpoint` is the one that means data
18    /// loss rather than convergence: the edge can never apply without operator
19    /// action, so a non-zero count here needs looking at.
20    pub skipped_by_reason: BTreeMap<String, u64>,
21}
22
23pub fn import_changes<R: BufRead>(store: &dyn StoreAdmin, reader: R) -> Result<(ImportStats, u64)> {
24    let mut stats = ImportStats {
25        created: 0,
26        updated: 0,
27        deleted: 0,
28        skipped: 0,
29        errors: 0,
30        skipped_by_reason: BTreeMap::new(),
31    };
32    let mut max_seq = 0u64;
33    let mut errored_seqs: Vec<u64> = Vec::new();
34
35    let mut records: Vec<SyncRecord> = Vec::new();
36    for line in reader.lines() {
37        let line = line?;
38        if line.trim().is_empty() {
39            continue;
40        }
41        match serde_json::from_str::<SyncRecord>(&line) {
42            Ok(r) => records.push(r),
43            Err(e) => {
44                tracing::warn!("skipping malformed sync record: {e}");
45                stats.errors += 1;
46            }
47        }
48    }
49
50    for record in &records {
51        if record.local_seq > max_seq {
52            max_seq = record.local_seq;
53        }
54    }
55
56    // Apply runs of consecutive Create records through the atomic batch path so a
57    // whole run commits with one fsync instead of one per record (per-record commits
58    // throttle import to a few records/sec on slow storage). Non-create ops (update,
59    // delete) flush the open create run and apply per-record through apply_record
60    // to preserve their conflict-resolution semantics unchanged.
61    const TX_CHUNK: usize = 500;
62    let mut i = 0;
63    while i < records.len() {
64        let record = &records[i];
65        if matches!(record.op, SyncOp::Create) {
66            let end = (i + TX_CHUNK).min(records.len());
67            let mut j = i;
68            while j < end && matches!(records[j].op, SyncOp::Create) {
69                j += 1;
70            }
71            let chunk = &records[i..j];
72            match store.apply_batch(chunk) {
73                Ok(outcomes) => {
74                    for outcome in outcomes {
75                        count_outcome(outcome, &mut stats);
76                    }
77                }
78                // apply_batch is all-or-nothing, so on error nothing from the
79                // chunk landed and each record retries individually.
80                Err(_) => {
81                    for rec in chunk {
82                        record_outcome(store, rec, &mut stats, &mut errored_seqs);
83                    }
84                }
85            }
86            i = j;
87        } else {
88            record_outcome(store, record, &mut stats, &mut errored_seqs);
89            i += 1;
90        }
91    }
92
93    let watermark = errored_seqs
94        .iter()
95        .min()
96        .map(|m| m.saturating_sub(1))
97        .unwrap_or(max_seq);
98
99    Ok((stats, watermark))
100}
101
102pub fn import_values(
103    store: &dyn StoreAdmin,
104    records: &[serde_json::Value],
105) -> Result<(ImportStats, u64)> {
106    let mut jsonl = String::new();
107    for record in records {
108        jsonl.push_str(&serde_json::to_string(record)?);
109        jsonl.push('\n');
110    }
111    let reader = std::io::BufReader::new(jsonl.as_bytes());
112    import_changes(store, reader)
113}
114
115fn count_outcome(outcome: ApplyOutcome, stats: &mut ImportStats) {
116    match outcome {
117        ApplyOutcome::Created => stats.created += 1,
118        ApplyOutcome::Updated => stats.updated += 1,
119        ApplyOutcome::Deleted => stats.deleted += 1,
120        ApplyOutcome::Skipped(reason) => {
121            stats.skipped += 1;
122            *stats
123                .skipped_by_reason
124                .entry(reason.as_str().to_string())
125                .or_insert(0) += 1;
126        }
127    }
128}
129
130fn record_outcome(
131    store: &dyn StoreAdmin,
132    record: &SyncRecord,
133    stats: &mut ImportStats,
134    errored_seqs: &mut Vec<u64>,
135) {
136    match apply_with_retry(store, record) {
137        Ok(outcome) => count_outcome(outcome, stats),
138        Err(e) => {
139            tracing::warn!(
140                "error applying sync record local_seq={}: {e}",
141                record.local_seq
142            );
143            stats.errors += 1;
144            errored_seqs.push(record.local_seq);
145        }
146    }
147}
148
149fn apply_with_retry(store: &dyn StoreAdmin, record: &SyncRecord) -> Result<ApplyOutcome> {
150    const MAX_ATTEMPTS: u32 = 5;
151    let mut last_err = None;
152    for attempt in 0..MAX_ATTEMPTS {
153        match store.apply_record(record) {
154            Ok(outcome) => return Ok(outcome),
155            Err(e) => {
156                tracing::debug!(
157                    "transient error on attempt {} for local_seq={}: {e}",
158                    attempt + 1,
159                    record.local_seq,
160                );
161                last_err = Some(e);
162                std::thread::sleep(std::time::Duration::from_millis(50 * (1 << attempt)));
163            }
164        }
165    }
166    Err(last_err.unwrap())
167}