second_brain_sync/
import.rs1use std::collections::BTreeMap;
2use std::io::BufRead;
3
4use anyhow::Result;
5
6use second_brain_core::schema::SyncOp;
7use second_brain_core::store::{ApplyOutcome, StoreAdmin};
8
9use crate::export::SyncRecord;
10
11pub struct ImportStats {
12 pub created: u64,
13 pub updated: u64,
14 pub deleted: u64,
15 pub skipped: u64,
16 pub errors: u64,
17 pub skipped_by_reason: BTreeMap<String, u64>,
21}
22
23pub fn import_changes<R: BufRead>(store: &dyn StoreAdmin, reader: R) -> Result<(ImportStats, u64)> {
24 let mut stats = ImportStats {
25 created: 0,
26 updated: 0,
27 deleted: 0,
28 skipped: 0,
29 errors: 0,
30 skipped_by_reason: BTreeMap::new(),
31 };
32 let mut max_seq = 0u64;
33 let mut errored_seqs: Vec<u64> = Vec::new();
34
35 let mut records: Vec<SyncRecord> = Vec::new();
36 for line in reader.lines() {
37 let line = line?;
38 if line.trim().is_empty() {
39 continue;
40 }
41 match serde_json::from_str::<SyncRecord>(&line) {
42 Ok(r) => records.push(r),
43 Err(e) => {
44 tracing::warn!("skipping malformed sync record: {e}");
45 stats.errors += 1;
46 }
47 }
48 }
49
50 for record in &records {
51 if record.local_seq > max_seq {
52 max_seq = record.local_seq;
53 }
54 }
55
56 const TX_CHUNK: usize = 500;
62 let mut i = 0;
63 while i < records.len() {
64 let record = &records[i];
65 if matches!(record.op, SyncOp::Create) {
66 let end = (i + TX_CHUNK).min(records.len());
67 let mut j = i;
68 while j < end && matches!(records[j].op, SyncOp::Create) {
69 j += 1;
70 }
71 let chunk = &records[i..j];
72 match store.apply_batch(chunk) {
73 Ok(outcomes) => {
74 for outcome in outcomes {
75 count_outcome(outcome, &mut stats);
76 }
77 }
78 Err(_) => {
81 for rec in chunk {
82 record_outcome(store, rec, &mut stats, &mut errored_seqs);
83 }
84 }
85 }
86 i = j;
87 } else {
88 record_outcome(store, record, &mut stats, &mut errored_seqs);
89 i += 1;
90 }
91 }
92
93 let watermark = errored_seqs
94 .iter()
95 .min()
96 .map(|m| m.saturating_sub(1))
97 .unwrap_or(max_seq);
98
99 Ok((stats, watermark))
100}
101
102pub fn import_values(
103 store: &dyn StoreAdmin,
104 records: &[serde_json::Value],
105) -> Result<(ImportStats, u64)> {
106 let mut jsonl = String::new();
107 for record in records {
108 jsonl.push_str(&serde_json::to_string(record)?);
109 jsonl.push('\n');
110 }
111 let reader = std::io::BufReader::new(jsonl.as_bytes());
112 import_changes(store, reader)
113}
114
115fn count_outcome(outcome: ApplyOutcome, stats: &mut ImportStats) {
116 match outcome {
117 ApplyOutcome::Created => stats.created += 1,
118 ApplyOutcome::Updated => stats.updated += 1,
119 ApplyOutcome::Deleted => stats.deleted += 1,
120 ApplyOutcome::Skipped(reason) => {
121 stats.skipped += 1;
122 *stats
123 .skipped_by_reason
124 .entry(reason.as_str().to_string())
125 .or_insert(0) += 1;
126 }
127 }
128}
129
130fn record_outcome(
131 store: &dyn StoreAdmin,
132 record: &SyncRecord,
133 stats: &mut ImportStats,
134 errored_seqs: &mut Vec<u64>,
135) {
136 match apply_with_retry(store, record) {
137 Ok(outcome) => count_outcome(outcome, stats),
138 Err(e) => {
139 tracing::warn!(
140 "error applying sync record local_seq={}: {e}",
141 record.local_seq
142 );
143 stats.errors += 1;
144 errored_seqs.push(record.local_seq);
145 }
146 }
147}
148
149fn apply_with_retry(store: &dyn StoreAdmin, record: &SyncRecord) -> Result<ApplyOutcome> {
150 const MAX_ATTEMPTS: u32 = 5;
151 let mut last_err = None;
152 for attempt in 0..MAX_ATTEMPTS {
153 match store.apply_record(record) {
154 Ok(outcome) => return Ok(outcome),
155 Err(e) => {
156 tracing::debug!(
157 "transient error on attempt {} for local_seq={}: {e}",
158 attempt + 1,
159 record.local_seq,
160 );
161 last_err = Some(e);
162 std::thread::sleep(std::time::Duration::from_millis(50 * (1 << attempt)));
163 }
164 }
165 }
166 Err(last_err.unwrap())
167}