use std::collections::BTreeMap;
use std::io::BufRead;
use anyhow::Result;
use second_brain_core::schema::SyncOp;
use second_brain_core::store::{ApplyOutcome, StoreAdmin};
use crate::export::SyncRecord;
pub struct ImportStats {
pub created: u64,
pub updated: u64,
pub deleted: u64,
pub skipped: u64,
pub errors: u64,
pub skipped_by_reason: BTreeMap<String, u64>,
}
pub fn import_changes<R: BufRead>(store: &dyn StoreAdmin, reader: R) -> Result<(ImportStats, u64)> {
let mut stats = ImportStats {
created: 0,
updated: 0,
deleted: 0,
skipped: 0,
errors: 0,
skipped_by_reason: BTreeMap::new(),
};
let mut max_seq = 0u64;
let mut errored_seqs: Vec<u64> = Vec::new();
let mut records: Vec<SyncRecord> = Vec::new();
for line in reader.lines() {
let line = line?;
if line.trim().is_empty() {
continue;
}
match serde_json::from_str::<SyncRecord>(&line) {
Ok(r) => records.push(r),
Err(e) => {
tracing::warn!("skipping malformed sync record: {e}");
stats.errors += 1;
}
}
}
for record in &records {
if record.local_seq > max_seq {
max_seq = record.local_seq;
}
}
const TX_CHUNK: usize = 500;
let mut i = 0;
while i < records.len() {
let record = &records[i];
if matches!(record.op, SyncOp::Create) {
let end = (i + TX_CHUNK).min(records.len());
let mut j = i;
while j < end && matches!(records[j].op, SyncOp::Create) {
j += 1;
}
let chunk = &records[i..j];
match store.apply_batch(chunk) {
Ok(outcomes) => {
for outcome in outcomes {
count_outcome(outcome, &mut stats);
}
}
Err(_) => {
for rec in chunk {
record_outcome(store, rec, &mut stats, &mut errored_seqs);
}
}
}
i = j;
} else {
record_outcome(store, record, &mut stats, &mut errored_seqs);
i += 1;
}
}
let watermark = errored_seqs
.iter()
.min()
.map(|m| m.saturating_sub(1))
.unwrap_or(max_seq);
Ok((stats, watermark))
}
fn count_outcome(outcome: ApplyOutcome, stats: &mut ImportStats) {
match outcome {
ApplyOutcome::Created => stats.created += 1,
ApplyOutcome::Updated => stats.updated += 1,
ApplyOutcome::Deleted => stats.deleted += 1,
ApplyOutcome::Skipped(reason) => {
stats.skipped += 1;
*stats
.skipped_by_reason
.entry(reason.as_str().to_string())
.or_insert(0) += 1;
}
}
}
fn record_outcome(
store: &dyn StoreAdmin,
record: &SyncRecord,
stats: &mut ImportStats,
errored_seqs: &mut Vec<u64>,
) {
match apply_with_retry(store, record) {
Ok(outcome) => count_outcome(outcome, stats),
Err(e) => {
tracing::warn!(
"error applying sync record local_seq={}: {e}",
record.local_seq
);
stats.errors += 1;
errored_seqs.push(record.local_seq);
}
}
}
fn apply_with_retry(store: &dyn StoreAdmin, record: &SyncRecord) -> Result<ApplyOutcome> {
const MAX_ATTEMPTS: u32 = 5;
let mut last_err = None;
for attempt in 0..MAX_ATTEMPTS {
match store.apply_record(record) {
Ok(outcome) => return Ok(outcome),
Err(e) => {
tracing::debug!(
"transient error on attempt {} for local_seq={}: {e}",
attempt + 1,
record.local_seq,
);
last_err = Some(e);
std::thread::sleep(std::time::Duration::from_millis(50 * (1 << attempt)));
}
}
}
Err(last_err.unwrap())
}