second_brain_sync/
import.rs1use std::collections::BTreeMap;
2use std::io::BufRead;
3
4use anyhow::Result;
5
6use second_brain_core::schema::SyncOp;
7use second_brain_core::store::{ApplyOutcome, StoreAdmin};
8
9use crate::export::SyncRecord;
10
11pub struct ImportStats {
12 pub created: u64,
13 pub updated: u64,
14 pub deleted: u64,
15 pub skipped: u64,
16 pub errors: u64,
17 pub skipped_by_reason: BTreeMap<String, u64>,
21}
22
23pub fn import_changes<R: BufRead>(store: &dyn StoreAdmin, reader: R) -> Result<(ImportStats, u64)> {
24 let mut stats = ImportStats {
25 created: 0,
26 updated: 0,
27 deleted: 0,
28 skipped: 0,
29 errors: 0,
30 skipped_by_reason: BTreeMap::new(),
31 };
32 let mut max_seq = 0u64;
33 let mut errored_seqs: Vec<u64> = Vec::new();
34
35 let mut records: Vec<SyncRecord> = Vec::new();
36 for line in reader.lines() {
37 let line = line?;
38 if line.trim().is_empty() {
39 continue;
40 }
41 match serde_json::from_str::<SyncRecord>(&line) {
42 Ok(r) => records.push(r),
43 Err(e) => {
44 tracing::warn!("skipping malformed sync record: {e}");
45 stats.errors += 1;
46 }
47 }
48 }
49
50 for record in &records {
51 if record.local_seq > max_seq {
52 max_seq = record.local_seq;
53 }
54 }
55
56 const TX_CHUNK: usize = 500;
62 let mut i = 0;
63 while i < records.len() {
64 let record = &records[i];
65 if matches!(record.op, SyncOp::Create) {
66 let end = (i + TX_CHUNK).min(records.len());
67 let mut j = i;
68 while j < end && matches!(records[j].op, SyncOp::Create) {
69 j += 1;
70 }
71 let chunk = &records[i..j];
72 match store.apply_batch(chunk) {
73 Ok(outcomes) => {
74 for outcome in outcomes {
75 count_outcome(outcome, &mut stats);
76 }
77 }
78 Err(_) => {
81 for rec in chunk {
82 record_outcome(store, rec, &mut stats, &mut errored_seqs);
83 }
84 }
85 }
86 i = j;
87 } else {
88 record_outcome(store, record, &mut stats, &mut errored_seqs);
89 i += 1;
90 }
91 }
92
93 let watermark = errored_seqs
94 .iter()
95 .min()
96 .map(|m| m.saturating_sub(1))
97 .unwrap_or(max_seq);
98
99 Ok((stats, watermark))
100}
101
102fn count_outcome(outcome: ApplyOutcome, stats: &mut ImportStats) {
103 match outcome {
104 ApplyOutcome::Created => stats.created += 1,
105 ApplyOutcome::Updated => stats.updated += 1,
106 ApplyOutcome::Deleted => stats.deleted += 1,
107 ApplyOutcome::Skipped(reason) => {
108 stats.skipped += 1;
109 *stats
110 .skipped_by_reason
111 .entry(reason.as_str().to_string())
112 .or_insert(0) += 1;
113 }
114 }
115}
116
117fn record_outcome(
118 store: &dyn StoreAdmin,
119 record: &SyncRecord,
120 stats: &mut ImportStats,
121 errored_seqs: &mut Vec<u64>,
122) {
123 match apply_with_retry(store, record) {
124 Ok(outcome) => count_outcome(outcome, stats),
125 Err(e) => {
126 tracing::warn!(
127 "error applying sync record local_seq={}: {e}",
128 record.local_seq
129 );
130 stats.errors += 1;
131 errored_seqs.push(record.local_seq);
132 }
133 }
134}
135
136fn apply_with_retry(store: &dyn StoreAdmin, record: &SyncRecord) -> Result<ApplyOutcome> {
137 const MAX_ATTEMPTS: u32 = 5;
138 let mut last_err = None;
139 for attempt in 0..MAX_ATTEMPTS {
140 match store.apply_record(record) {
141 Ok(outcome) => return Ok(outcome),
142 Err(e) => {
143 tracing::debug!(
144 "transient error on attempt {} for local_seq={}: {e}",
145 attempt + 1,
146 record.local_seq,
147 );
148 last_err = Some(e);
149 std::thread::sleep(std::time::Duration::from_millis(50 * (1 << attempt)));
150 }
151 }
152 }
153 Err(last_err.unwrap())
154}