second_brain_sync/
import.rs1use std::collections::BTreeMap;
2use std::io::BufRead;
3
4use anyhow::Result;
5
6use second_brain_core::schema::SyncOp;
7use second_brain_core::sqlite_store::ClockUntrusted;
8use second_brain_core::store::{ApplyOutcome, StoreAdmin};
9
10use crate::export::SyncRecord;
11
12pub struct ImportStats {
13 pub created: u64,
14 pub updated: u64,
15 pub deleted: u64,
16 pub skipped: u64,
17 pub errors: u64,
18 pub skipped_by_reason: BTreeMap<String, u64>,
22 pub reached: u64,
24 pub failed: Vec<(SyncRecord, anyhow::Error)>,
27}
28
29pub fn import_changes<R: BufRead>(store: &dyn StoreAdmin, reader: R) -> Result<(ImportStats, u64)> {
30 let mut stats = ImportStats {
31 created: 0,
32 updated: 0,
33 deleted: 0,
34 skipped: 0,
35 errors: 0,
36 skipped_by_reason: BTreeMap::new(),
37 reached: 0,
38 failed: Vec::new(),
39 };
40 let mut max_seq = 0u64;
41 let mut errored_seqs: Vec<u64> = Vec::new();
42
43 let mut records: Vec<SyncRecord> = Vec::new();
44 for line in reader.lines() {
45 let line = line?;
46 if line.trim().is_empty() {
47 continue;
48 }
49 match serde_json::from_str::<SyncRecord>(&line) {
50 Ok(r) => records.push(r),
51 Err(e) => {
52 tracing::warn!("skipping malformed sync record: {e}");
53 stats.errors += 1;
54 }
55 }
56 }
57
58 for record in &records {
59 if record.local_seq > max_seq {
60 max_seq = record.local_seq;
61 }
62 }
63
64 const TX_CHUNK: usize = 500;
70 let mut i = 0;
71 while i < records.len() {
72 let record = &records[i];
73 if matches!(record.op, SyncOp::Create) {
74 let end = (i + TX_CHUNK).min(records.len());
75 let mut j = i;
76 while j < end && matches!(records[j].op, SyncOp::Create) {
77 j += 1;
78 }
79 let chunk = &records[i..j];
80 match store.apply_batch(chunk) {
81 Ok(outcomes) => {
82 for outcome in outcomes {
83 count_outcome(outcome, &mut stats);
84 }
85 }
86 Err(_) => {
89 for rec in chunk {
90 record_outcome(store, rec, &mut stats, &mut errored_seqs)?;
91 }
92 }
93 }
94 i = j;
95 } else {
96 record_outcome(store, record, &mut stats, &mut errored_seqs)?;
97 i += 1;
98 }
99 }
100
101 stats.reached = max_seq;
102 let watermark = errored_seqs
103 .iter()
104 .min()
105 .map(|m| m.saturating_sub(1))
106 .unwrap_or(max_seq);
107
108 Ok((stats, watermark))
109}
110
111pub fn import_values(
112 store: &dyn StoreAdmin,
113 records: &[serde_json::Value],
114) -> Result<(ImportStats, u64)> {
115 let mut jsonl = String::new();
116 for record in records {
117 jsonl.push_str(&serde_json::to_string(record)?);
118 jsonl.push('\n');
119 }
120 let reader = std::io::BufReader::new(jsonl.as_bytes());
121 import_changes(store, reader)
122}
123
124fn count_outcome(outcome: ApplyOutcome, stats: &mut ImportStats) {
125 match outcome {
126 ApplyOutcome::Created => stats.created += 1,
127 ApplyOutcome::Updated => stats.updated += 1,
128 ApplyOutcome::Deleted => stats.deleted += 1,
129 ApplyOutcome::Skipped(reason) => {
130 stats.skipped += 1;
131 *stats
132 .skipped_by_reason
133 .entry(reason.as_str().to_string())
134 .or_insert(0) += 1;
135 }
136 }
137}
138
139fn record_outcome(
143 store: &dyn StoreAdmin,
144 record: &SyncRecord,
145 stats: &mut ImportStats,
146 errored_seqs: &mut Vec<u64>,
147) -> Result<()> {
148 match apply_with_retry(store, record) {
149 Ok(outcome) => count_outcome(outcome, stats),
150 Err(e) if clock_untrusted(&e) => return Err(e),
151 Err(e) => {
152 tracing::warn!(
153 "error applying sync record local_seq={}: {e}",
154 record.local_seq
155 );
156 stats.errors += 1;
157 errored_seqs.push(record.local_seq);
158 stats.failed.push((record.clone(), e));
159 }
160 }
161 Ok(())
162}
163
164fn clock_untrusted(error: &anyhow::Error) -> bool {
165 error.chain().any(|cause| cause.is::<ClockUntrusted>())
166}
167
168fn apply_with_retry(store: &dyn StoreAdmin, record: &SyncRecord) -> Result<ApplyOutcome> {
169 const MAX_ATTEMPTS: u32 = 5;
170 let mut last_err = None;
171 for attempt in 0..MAX_ATTEMPTS {
172 match store.apply_record(record) {
173 Ok(outcome) => return Ok(outcome),
174 Err(e) if clock_untrusted(&e) => return Err(e),
175 Err(e) => {
176 tracing::debug!(
177 "transient error on attempt {} for local_seq={}: {e}",
178 attempt + 1,
179 record.local_seq,
180 );
181 last_err = Some(e);
182 std::thread::sleep(std::time::Duration::from_millis(50 * (1 << attempt)));
183 }
184 }
185 }
186 Err(last_err.unwrap())
187}