second-brain-sync 0.13.4

Bidirectional sync for second-brain: SSH transport, JSONL change log, conflict resolution
Documentation
use std::collections::BTreeMap;
use std::io::BufRead;

use anyhow::Result;

use second_brain_core::schema::SyncOp;
use second_brain_core::sqlite_store::ClockUntrusted;
use second_brain_core::store::{ApplyOutcome, StoreAdmin};

use crate::export::SyncRecord;

pub struct ImportStats {
    pub created: u64,
    pub updated: u64,
    pub deleted: u64,
    pub skipped: u64,
    pub errors: u64,
    /// Skips broken out by cause. `dangling_endpoint` is the one that means data
    /// loss rather than convergence: the edge can never apply without operator
    /// action, so a non-zero count here needs looking at.
    pub skipped_by_reason: BTreeMap<String, u64>,
    /// The highest seq the stream carried, past failed records.
    pub reached: u64,
    /// Records that failed every in-call attempt, with their last error, for
    /// the store to keep and retry.
    pub failed: Vec<(SyncRecord, anyhow::Error)>,
}

pub fn import_changes<R: BufRead>(store: &dyn StoreAdmin, reader: R) -> Result<(ImportStats, u64)> {
    let mut stats = ImportStats {
        created: 0,
        updated: 0,
        deleted: 0,
        skipped: 0,
        errors: 0,
        skipped_by_reason: BTreeMap::new(),
        reached: 0,
        failed: Vec::new(),
    };
    let mut max_seq = 0u64;
    let mut errored_seqs: Vec<u64> = Vec::new();

    let mut records: Vec<SyncRecord> = Vec::new();
    for line in reader.lines() {
        let line = line?;
        if line.trim().is_empty() {
            continue;
        }
        match serde_json::from_str::<SyncRecord>(&line) {
            Ok(r) => records.push(r),
            Err(e) => {
                tracing::warn!("skipping malformed sync record: {e}");
                stats.errors += 1;
            }
        }
    }

    for record in &records {
        if record.local_seq > max_seq {
            max_seq = record.local_seq;
        }
    }

    // Apply runs of consecutive Create records through the atomic batch path so a
    // whole run commits with one fsync instead of one per record (per-record commits
    // throttle import to a few records/sec on slow storage). Non-create ops (update,
    // delete) flush the open create run and apply per-record through apply_record
    // to preserve their conflict-resolution semantics unchanged.
    const TX_CHUNK: usize = 500;
    let mut i = 0;
    while i < records.len() {
        let record = &records[i];
        if matches!(record.op, SyncOp::Create) {
            let end = (i + TX_CHUNK).min(records.len());
            let mut j = i;
            while j < end && matches!(records[j].op, SyncOp::Create) {
                j += 1;
            }
            let chunk = &records[i..j];
            match store.apply_batch(chunk) {
                Ok(outcomes) => {
                    for outcome in outcomes {
                        count_outcome(outcome, &mut stats);
                    }
                }
                // apply_batch is all-or-nothing, so on error nothing from the
                // chunk landed and each record retries individually.
                Err(_) => {
                    for rec in chunk {
                        record_outcome(store, rec, &mut stats, &mut errored_seqs)?;
                    }
                }
            }
            i = j;
        } else {
            record_outcome(store, record, &mut stats, &mut errored_seqs)?;
            i += 1;
        }
    }

    stats.reached = max_seq;
    let watermark = errored_seqs
        .iter()
        .min()
        .map(|m| m.saturating_sub(1))
        .unwrap_or(max_seq);

    Ok((stats, watermark))
}

pub fn import_values(
    store: &dyn StoreAdmin,
    records: &[serde_json::Value],
) -> Result<(ImportStats, u64)> {
    let mut jsonl = String::new();
    for record in records {
        jsonl.push_str(&serde_json::to_string(record)?);
        jsonl.push('\n');
    }
    let reader = std::io::BufReader::new(jsonl.as_bytes());
    import_changes(store, reader)
}

fn count_outcome(outcome: ApplyOutcome, stats: &mut ImportStats) {
    match outcome {
        ApplyOutcome::Created => stats.created += 1,
        ApplyOutcome::Updated => stats.updated += 1,
        ApplyOutcome::Deleted => stats.deleted += 1,
        ApplyOutcome::Skipped(reason) => {
            stats.skipped += 1;
            *stats
                .skipped_by_reason
                .entry(reason.as_str().to_string())
                .or_insert(0) += 1;
        }
    }
}

/// A store that cannot mint a seq fails the whole import rather than counting
/// an error: every later record would fail the same way, and the sender must
/// keep its cursor to resend them once the clock is trusted.
fn record_outcome(
    store: &dyn StoreAdmin,
    record: &SyncRecord,
    stats: &mut ImportStats,
    errored_seqs: &mut Vec<u64>,
) -> Result<()> {
    match apply_with_retry(store, record) {
        Ok(outcome) => count_outcome(outcome, stats),
        Err(e) if clock_untrusted(&e) => return Err(e),
        Err(e) => {
            tracing::warn!(
                "error applying sync record local_seq={}: {e}",
                record.local_seq
            );
            stats.errors += 1;
            errored_seqs.push(record.local_seq);
            stats.failed.push((record.clone(), e));
        }
    }
    Ok(())
}

fn clock_untrusted(error: &anyhow::Error) -> bool {
    error.chain().any(|cause| cause.is::<ClockUntrusted>())
}

fn apply_with_retry(store: &dyn StoreAdmin, record: &SyncRecord) -> Result<ApplyOutcome> {
    const MAX_ATTEMPTS: u32 = 5;
    let mut last_err = None;
    for attempt in 0..MAX_ATTEMPTS {
        match store.apply_record(record) {
            Ok(outcome) => return Ok(outcome),
            Err(e) if clock_untrusted(&e) => return Err(e),
            Err(e) => {
                tracing::debug!(
                    "transient error on attempt {} for local_seq={}: {e}",
                    attempt + 1,
                    record.local_seq,
                );
                last_err = Some(e);
                std::thread::sleep(std::time::Duration::from_millis(50 * (1 << attempt)));
            }
        }
    }
    Err(last_err.unwrap())
}