Skip to main content

kimetsu_brain/
sync.rs

1/// Epic S3 — Personal brain sync (event-log replication).
2///
3/// Design insight: `events` is the durable source of truth and the projector
4/// rebuilds everything from it.  Sync = EVENT-LOG REPLICATION, not SQLite file
5/// copying.  We export durable events and import them through the projector with
6/// per-event idempotency.  No server, no merge daemon.
7///
8/// # Allowed (durable memory-lifecycle) kinds — exported
9/// - `memory.accepted`
10/// - `memory.proposed`
11/// - `memory.rejected`
12/// - `memory.invalidated`
13/// - `memory.restored`
14/// - `memory.corrected`
15/// - `memory.cited`
16/// - `memory.superseded`
17///
18/// # Excluded kinds — never exported
19/// - `work.episode`         — episodes are LOCAL-ONLY (Flagship 1)
20/// - `context.served`       — local telemetry
21/// - `retrieval.regret`     — local telemetry
22/// - `digest_served`        — local telemetry
23/// - `resume_served`        — local telemetry
24/// - `context.injected`     — raw query bearing
25/// - `run.started`          — local run metadata
26/// - `run.finished`         — local run metadata
27/// - `run.failed`           — local run metadata
28/// - `run.aborted`          — local run metadata
29///
30/// Everything else that is not on the allowlist is also excluded by default.
31///
32/// # Cursor
33/// The monotonic ordering column is `rowid` (the implicit SQLite integer
34/// primary key alias).  A cursor is the last exported `rowid`.  The next
35/// export picks up WHERE rowid > cursor.  Cursor 0 means "from the beginning".
36///
37/// # Idempotency
38/// Import checks `event_id` (ULID) against the local `events` table.
39/// `INSERT OR IGNORE` in `insert_event` already provides this, but we also
40/// count skipped events so the caller can report applied/skipped.
41///
42/// # Directory protocol (3.2)
43/// `<sync_dir>/<machine_id>/<cursor>.jsonl` — each batch file is atomically
44/// written (temp + rename).  A per-source-cursor registry lives at
45/// `.kimetsu/sync-cursors.json`.  `kimetsu brain sync` (no args):
46///   1. Write this machine's new events under `<sync_dir>/<machine_id>/`.
47///   2. For every OTHER subdirectory (= other machine), read batches after
48///      the locally stored cursor for that machine, import them (idempotent),
49///      and advance the cursor.
50use std::collections::{BTreeMap, BTreeSet};
51use std::fs;
52use std::io::{BufRead, BufReader, Write as IoWrite};
53use std::path::{Path, PathBuf};
54
55use kimetsu_core::KimetsuResult;
56use kimetsu_core::event::Event;
57use kimetsu_core::ids::{EventId, RunId};
58use rusqlite::Connection;
59use serde::{Deserialize, Serialize};
60use time::OffsetDateTime;
61use time::format_description::well_known::Rfc3339;
62use ulid::Ulid;
63
64use crate::projector;
65use crate::redact;
66
67// ---------------------------------------------------------------------------
68// Allowlist
69// ---------------------------------------------------------------------------
70
71/// Kinds that carry durable memory-lifecycle meaning and SHOULD be replicated.
72const SYNC_ALLOWED_KINDS: &[&str] = &[
73    "memory.accepted",
74    "memory.proposed",
75    "memory.rejected",
76    "memory.invalidated",
77    "memory.restored",
78    "memory.corrected",
79    "memory.cited",
80    "memory.superseded",
81];
82
83/// Returns `true` when `kind` is allowed in a sync batch.
84pub fn is_sync_allowed(kind: &str) -> bool {
85    SYNC_ALLOWED_KINDS.contains(&kind)
86}
87
88// ---------------------------------------------------------------------------
89// Event wire format
90// ---------------------------------------------------------------------------
91
92/// One line in a sync JSONL batch.  Carries the full event so the remote
93/// projector can replay it.  `payload` is the redacted payload (same
94/// redaction the projector applies at ingest).
95#[derive(Debug, Clone, Serialize, Deserialize)]
96pub struct SyncEvent {
97    pub event_id: String,
98    pub run_id: String,
99    #[serde(with = "time::serde::rfc3339")]
100    pub ts: OffsetDateTime,
101    pub kind: String,
102    pub schema_version: u32,
103    pub payload: serde_json::Value,
104    /// v2.6 #3: who/where wrote this event (`<machine_id>/<agent>`). Carried so a
105    /// replicated/team brain can attribute each event. `#[serde(default)]` keeps
106    /// pre-v8 sync batches (no `origin`) importable.
107    #[serde(default)]
108    pub origin: Option<String>,
109    /// v2.6 #3 Slice B: the event's HLC (canonical string) for convergent
110    /// total-order replay. `#[serde(default)]` keeps pre-v9 batches importable
111    /// (the importer synthesizes a local HLC for those).
112    #[serde(default)]
113    pub hlc: Option<String>,
114}
115
116impl From<&Event> for SyncEvent {
117    fn from(e: &Event) -> Self {
118        // Apply the same payload redaction the projector does so sync batches
119        // are never a second secret store (matches projector::redact_memory_event).
120        let payload = redact_event_payload(e);
121        Self {
122            event_id: e.event_id.to_string(),
123            run_id: e.run_id.to_string(),
124            ts: e.ts,
125            kind: e.kind.clone(),
126            schema_version: e.schema_version,
127            payload,
128            origin: e.origin.clone(),
129            hlc: e.hlc.clone(),
130        }
131    }
132}
133
134impl TryFrom<SyncEvent> for Event {
135    type Error = Box<dyn std::error::Error + Send + Sync>;
136
137    fn try_from(s: SyncEvent) -> Result<Self, Self::Error> {
138        let event_id = EventId(
139            Ulid::from_string(&s.event_id)
140                .map_err(|e| format!("invalid event_id {:?}: {e}", s.event_id))?,
141        );
142        let run_id = RunId(
143            Ulid::from_string(&s.run_id)
144                .map_err(|e| format!("invalid run_id {:?}: {e}", s.run_id))?,
145        );
146        // Preserve the REMOTE HLC; advance the local clock past it so subsequent
147        // LOCAL events sort after everything imported (causality). A pre-v9 peer
148        // sends no HLC → synthesize a current local one so the event still sorts.
149        let hlc = match s.hlc {
150            Some(h) => {
151                if let Some(parsed) = kimetsu_core::clock::Hlc::parse(&h) {
152                    kimetsu_core::clock::observe(&parsed);
153                }
154                Some(h)
155            }
156            None => Some(kimetsu_core::clock::now().to_canonical()),
157        };
158        Ok(Event {
159            event_id,
160            run_id,
161            ts: s.ts,
162            parent_event_id: None,
163            kind: s.kind,
164            schema_version: s.schema_version,
165            payload: s.payload,
166            // Preserve the REMOTE origin — do NOT stamp the local process origin.
167            origin: s.origin,
168            hlc,
169        })
170    }
171}
172
173/// Apply export-time redaction to the event payload — same logic as
174/// `projector::redact_memory_event` but returns an owned `Value`.
175fn redact_event_payload(event: &Event) -> serde_json::Value {
176    if !matches!(
177        event.kind.as_str(),
178        "memory.accepted" | "memory.proposed" | "memory.cited" | "memory.corrected"
179    ) {
180        return event.payload.clone();
181    }
182    redact_json_strings_owned(&event.payload)
183}
184
185fn redact_json_strings_owned(value: &serde_json::Value) -> serde_json::Value {
186    match value {
187        serde_json::Value::String(text) => {
188            serde_json::Value::String(redact::redact_secrets(text).text)
189        }
190        serde_json::Value::Array(arr) => {
191            serde_json::Value::Array(arr.iter().map(redact_json_strings_owned).collect())
192        }
193        serde_json::Value::Object(map) => {
194            let out = map
195                .iter()
196                .map(|(k, v)| (k.clone(), redact_json_strings_owned(v)))
197                .collect();
198            serde_json::Value::Object(out)
199        }
200        other => other.clone(),
201    }
202}
203
204// ---------------------------------------------------------------------------
205// 3.1 — Export
206// ---------------------------------------------------------------------------
207
208/// Summary returned by [`export_events`].
209#[derive(Debug, Clone, Default)]
210pub struct ExportSummary {
211    /// Number of events written to the batch.
212    pub exported: usize,
213    /// The highest rowid included in this batch (= next cursor).
214    pub next_cursor: i64,
215}
216
217/// Export durable events from `conn` after `since_rowid` (exclusive).
218///
219/// Only events whose `kind` is on the sync allowlist are included.
220/// Redaction is applied inline (no workspace paths / secrets leak).
221///
222/// When `out_path` is `None`, returns the JSONL as a `String`.
223/// When `out_path` is `Some(path)`, writes atomically via temp+rename.
224pub fn export_events(
225    conn: &Connection,
226    since_rowid: i64,
227    out_path: Option<&Path>,
228    dry_run: bool,
229) -> KimetsuResult<(ExportSummary, Option<String>)> {
230    let rows = read_durable_events_after(conn, since_rowid)?;
231
232    let mut lines = Vec::new();
233    let mut next_cursor = since_rowid;
234    for (rowid, event) in &rows {
235        if !is_sync_allowed(&event.kind) {
236            continue;
237        }
238        let se = SyncEvent::from(event);
239        let line = serde_json::to_string(&se)
240            .map_err(|e| format!("sync export: serialize event {}: {e}", event.event_id))?;
241        lines.push(line);
242        if *rowid > next_cursor {
243            next_cursor = *rowid;
244        }
245    }
246
247    let summary = ExportSummary {
248        exported: lines.len(),
249        next_cursor,
250    };
251
252    if dry_run {
253        return Ok((summary, None));
254    }
255
256    let jsonl = lines.join("\n");
257    if let Some(path) = out_path {
258        atomic_write(path, jsonl.as_bytes())?;
259        Ok((summary, None))
260    } else {
261        Ok((summary, Some(jsonl)))
262    }
263}
264
265/// Read all (rowid, Event) pairs from the `events` table with rowid > `after`.
266fn read_durable_events_after(conn: &Connection, after: i64) -> KimetsuResult<Vec<(i64, Event)>> {
267    let mut stmt = conn.prepare(
268        "SELECT rowid, event_id, run_id, ts, kind, schema_version, payload_json, origin, hlc
269         FROM events
270         WHERE rowid > ?1
271         ORDER BY rowid",
272    )?;
273    let rows = stmt.query_map(rusqlite::params![after], |row| {
274        let rowid: i64 = row.get(0)?;
275        let event_id_str: String = row.get(1)?;
276        let run_id_str: String = row.get(2)?;
277        let ts_str: String = row.get(3)?;
278        let kind: String = row.get(4)?;
279        let schema_version: u32 = row.get(5)?;
280        let payload_json: String = row.get(6)?;
281        let origin: Option<String> = row.get(7)?;
282        let hlc: Option<String> = row.get(8)?;
283        Ok((
284            rowid,
285            event_id_str,
286            run_id_str,
287            ts_str,
288            kind,
289            schema_version,
290            payload_json,
291            origin,
292            hlc,
293        ))
294    })?;
295
296    let mut out = Vec::new();
297    for row in rows {
298        let (
299            rowid,
300            event_id_str,
301            run_id_str,
302            ts_str,
303            kind,
304            schema_version,
305            payload_json,
306            origin,
307            hlc,
308        ) = row?;
309        let event_id = EventId(
310            Ulid::from_string(&event_id_str)
311                .map_err(|e| format!("invalid event_id {event_id_str:?}: {e}"))?,
312        );
313        let run_id = RunId(
314            Ulid::from_string(&run_id_str)
315                .map_err(|e| format!("invalid run_id {run_id_str:?}: {e}"))?,
316        );
317        let ts = OffsetDateTime::parse(&ts_str, &Rfc3339)
318            .map_err(|e| format!("invalid ts {ts_str:?}: {e}"))?;
319        let payload: serde_json::Value = serde_json::from_str(&payload_json)?;
320        out.push((
321            rowid,
322            Event {
323                event_id,
324                run_id,
325                ts,
326                parent_event_id: None,
327                kind,
328                schema_version,
329                payload,
330                origin,
331                hlc,
332            },
333        ));
334    }
335    Ok(out)
336}
337
338// ---------------------------------------------------------------------------
339// 3.1 — Import
340// ---------------------------------------------------------------------------
341
342/// Summary returned by [`import_events`].
343#[derive(Debug, Clone, Default)]
344pub struct ImportSummary {
345    /// Events actually applied (projected into derived tables).
346    pub applied: usize,
347    /// Events skipped because their `event_id` already existed locally.
348    pub skipped: usize,
349}
350
351/// Import a JSONL batch (one `SyncEvent` per line) into `conn`.
352///
353/// Per-event idempotency: if the `event_id` already exists in the local
354/// `events` table, the event is skipped (no double-apply).
355/// `INSERT OR IGNORE` in `projector::insert_event` provides the underlying
356/// dedup; we additionally count skips for reporting.
357///
358/// When `dry_run` is true, parse and count but do NOT write anything.
359/// Otherwise, stage the entire batch and replay the merged durable log under
360/// one writer lock. Historical edits must not project against today's state.
361pub fn import_events(
362    conn: &Connection,
363    jsonl: &str,
364    dry_run: bool,
365) -> KimetsuResult<ImportSummary> {
366    let mut excluded = 0;
367    let mut events = Vec::new();
368    for (line_no, line) in jsonl.lines().enumerate() {
369        let line = line.trim();
370        if line.is_empty() {
371            continue;
372        }
373        let se: SyncEvent = serde_json::from_str(line)
374            .map_err(|e| format!("sync import: malformed JSON on line {}: {e}", line_no + 1))?;
375
376        // Validate it's an allowed kind — defence-in-depth (the exporter
377        // already filters, but a hand-crafted batch might not).
378        if !is_sync_allowed(&se.kind) {
379            // Skip silently — telemetry/local kinds should never appear.
380            excluded += 1;
381            continue;
382        }
383
384        let event: Event = Event::try_from(se)
385            .map_err(|e| format!("sync import: invalid event on line {}: {e}", line_no + 1))?;
386
387        events.push(event);
388    }
389
390    let mut summary = ImportSummary::default();
391    let mut import = |c: &Connection| -> KimetsuResult<()> {
392        summary = ImportSummary {
393            applied: 0,
394            skipped: excluded,
395        };
396        let mut seen = BTreeSet::new();
397        for event in &events {
398            // Both the count and insertion run under the writer lock. Include
399            // in-batch duplicates in dry-run counts without writing them.
400            let exists: bool = !seen.insert(event.event_id.to_string())
401                || c.query_row(
402                    "SELECT 1 FROM events WHERE event_id = ?1",
403                    rusqlite::params![event.event_id.to_string()],
404                    |_| Ok(true),
405                )
406                .optional()?
407                .unwrap_or(false);
408
409            if exists {
410                summary.skipped += 1;
411                continue;
412            }
413            if !dry_run {
414                let redacted = Event {
415                    payload: redact_event_payload(event),
416                    ..event.clone()
417                };
418                projector::insert_event(c, &redacted)?;
419            }
420            summary.applied += 1;
421        }
422        if !dry_run && summary.applied > 0 {
423            projector::replay_locked(c)?;
424        }
425        Ok(())
426    };
427    if dry_run {
428        import(conn)?;
429    } else {
430        projector::with_write_txn(conn, import)?;
431    }
432    Ok(summary)
433}
434
435/// Slice B: count unresolved concurrent-supersede conflicts surfaced by team
436/// sync (a member superseded to two different survivors). Deterministic across
437/// brains; shown by `kimetsu brain sync --status`.
438pub fn sync_conflict_count(conn: &Connection) -> KimetsuResult<i64> {
439    let n: i64 = conn.query_row("SELECT COUNT(*) FROM sync_conflicts", [], |r| r.get(0))?;
440    Ok(n)
441}
442
443/// Read a JSONL batch file and import it.
444pub fn import_events_from_file(
445    conn: &Connection,
446    path: &Path,
447    dry_run: bool,
448) -> KimetsuResult<ImportSummary> {
449    import_events(conn, &read_batch_file(path)?, dry_run)
450}
451
452fn read_batch_file(path: &Path) -> KimetsuResult<String> {
453    let file = fs::File::open(path)
454        .map_err(|e| format!("sync import: cannot open {:?}: {e}", path.display()))?;
455    let reader = BufReader::new(file);
456    let mut buf = String::new();
457    for line in reader.lines() {
458        let l = line.map_err(|e| format!("sync import: read error {:?}: {e}", path.display()))?;
459        buf.push_str(&l);
460        buf.push('\n');
461    }
462    Ok(buf)
463}
464
465// ---------------------------------------------------------------------------
466// 3.2 — Sync cursor registry
467// ---------------------------------------------------------------------------
468
469/// Per-source cursor state persisted at `.kimetsu/sync-cursors.json`.
470///
471/// Keys are machine_id strings; values are the last rowid imported from that
472/// machine.  Our OWN machine_id is in here too (last exported rowid).
473#[derive(Debug, Clone, Default, Serialize, Deserialize)]
474pub struct SyncCursors {
475    /// map machine_id → last imported rowid (0 = never imported).
476    #[serde(default)]
477    pub sources: BTreeMap<String, i64>,
478}
479
480impl SyncCursors {
481    pub fn load(path: &Path) -> KimetsuResult<Self> {
482        if !path.exists() {
483            return Ok(Self::default());
484        }
485        let text = fs::read_to_string(path)
486            .map_err(|e| format!("sync-cursors: cannot read {:?}: {e}", path.display()))?;
487        serde_json::from_str(&text)
488            .map_err(|e| format!("sync-cursors: malformed JSON at {:?}: {e}", path.display()))
489            .map_err(Into::into)
490    }
491
492    pub fn save(&self, path: &Path) -> KimetsuResult<()> {
493        let text = serde_json::to_string_pretty(self)
494            .map_err(|e| format!("sync-cursors: serialize error: {e}"))?;
495        atomic_write(path, text.as_bytes())
496    }
497
498    pub fn cursor_for(&self, machine_id: &str) -> i64 {
499        *self.sources.get(machine_id).unwrap_or(&0)
500    }
501
502    pub fn set_cursor(&mut self, machine_id: &str, rowid: i64) {
503        self.sources.insert(machine_id.to_string(), rowid);
504    }
505}
506
507// ---------------------------------------------------------------------------
508// 3.2 — Directory protocol
509// ---------------------------------------------------------------------------
510
511/// The max rowid among all rows in the local `events` table with an allowed
512/// kind.  This is what we compare against the stored export cursor to decide
513/// whether there's anything new to push.
514pub fn max_local_sync_rowid(conn: &Connection) -> KimetsuResult<i64> {
515    let placeholders: String = SYNC_ALLOWED_KINDS
516        .iter()
517        .enumerate()
518        .map(|(i, _)| format!("?{}", i + 1))
519        .collect::<Vec<_>>()
520        .join(", ");
521    let sql = format!("SELECT COALESCE(MAX(rowid), 0) FROM events WHERE kind IN ({placeholders})");
522    let mut stmt = conn.prepare(&sql)?;
523    let params: Vec<Box<dyn rusqlite::ToSql>> = SYNC_ALLOWED_KINDS
524        .iter()
525        .map(|k| -> Box<dyn rusqlite::ToSql> { Box::new(k.to_string()) })
526        .collect();
527    let refs: Vec<&dyn rusqlite::ToSql> = params.iter().map(|p| p.as_ref()).collect();
528    let max: i64 = stmt.query_row(refs.as_slice(), |r| r.get(0))?;
529    Ok(max)
530}
531
532/// Write this machine's new events as a JSONL batch under
533/// `<sync_dir>/<machine_id>/<cursor>.jsonl` (atomic write).
534///
535/// Returns the number of events written and the new export cursor.
536pub fn push_machine_batch(
537    conn: &Connection,
538    sync_dir: &Path,
539    machine_id: &str,
540    since_rowid: i64,
541    dry_run: bool,
542) -> KimetsuResult<ExportSummary> {
543    let (dry_summary, _content) = export_events(conn, since_rowid, None, true)?;
544    if dry_summary.exported == 0 || dry_run {
545        return Ok(dry_summary);
546    }
547
548    // Re-run for real (not dry_run) to get the content.
549    let (summary, content) = export_events(conn, since_rowid, None, false)?;
550    let jsonl = content.unwrap_or_default();
551
552    let machine_dir = sync_dir.join(machine_id);
553    fs::create_dir_all(&machine_dir).map_err(|e| {
554        format!(
555            "sync push: cannot create dir {:?}: {e}",
556            machine_dir.display()
557        )
558    })?;
559
560    let batch_name = format!("{}.jsonl", summary.next_cursor);
561    let batch_path = machine_dir.join(&batch_name);
562    atomic_write(&batch_path, jsonl.as_bytes())?;
563
564    Ok(summary)
565}
566
567/// Pull and import all batches from `<sync_dir>/<source_machine_id>/` that
568/// come AFTER `since_cursor`.  Updates the cursor in the registry.
569///
570/// Batches are files named `<rowid>.jsonl`; we sort numerically and process
571/// only those whose stem > since_cursor.
572pub fn pull_machine_batches(
573    conn: &Connection,
574    sync_dir: &Path,
575    source_machine_id: &str,
576    since_cursor: i64,
577    dry_run: bool,
578) -> KimetsuResult<(ImportSummary, i64)> {
579    let (jsonl, cursor) = read_machine_batches(sync_dir, source_machine_id, since_cursor)?;
580    Ok((import_events(conn, &jsonl, dry_run)?, cursor))
581}
582
583/// Read every pending batch before importing: prerequisites can be in a later
584/// batch or on another peer, so directory sync merges these before replay.
585fn read_machine_batches(
586    sync_dir: &Path,
587    source_machine_id: &str,
588    since_cursor: i64,
589) -> KimetsuResult<(String, i64)> {
590    let machine_dir = sync_dir.join(source_machine_id);
591    if !machine_dir.exists() {
592        return Ok((String::new(), since_cursor));
593    }
594
595    // Collect batch files, parse their numeric stem (= the export cursor at
596    // the time they were written, i.e. the highest rowid in that batch on
597    // the source machine).
598    let mut batches: Vec<(i64, PathBuf)> = Vec::new();
599    let entries = fs::read_dir(&machine_dir).map_err(|e| {
600        format!(
601            "sync pull: cannot read dir {:?}: {e}",
602            machine_dir.display()
603        )
604    })?;
605    for entry in entries {
606        let entry = entry.map_err(|e| format!("sync pull: dir entry error: {e}"))?;
607        let path = entry.path();
608        if path.extension().and_then(|e| e.to_str()) != Some("jsonl") {
609            continue;
610        }
611        if let Some(stem) = path.file_stem().and_then(|s| s.to_str()) {
612            if let Ok(cursor_val) = stem.parse::<i64>() {
613                if cursor_val > since_cursor {
614                    batches.push((cursor_val, path));
615                }
616            }
617        }
618    }
619    batches.sort_by_key(|(c, _)| *c);
620
621    let mut jsonl = String::new();
622    let mut new_cursor = since_cursor;
623    for (cursor_val, batch_path) in &batches {
624        jsonl.push_str(&read_batch_file(batch_path)?);
625        if *cursor_val > new_cursor {
626            new_cursor = *cursor_val;
627        }
628    }
629    Ok((jsonl, new_cursor))
630}
631
632/// Full sync cycle:
633/// 1. Push this machine's new events.
634/// 2. Pull every other machine's new batches.
635/// 3. Persist updated cursors.
636///
637/// Returns a summary of what happened.
638pub fn sync_dir(
639    conn: &Connection,
640    sync_dir: &Path,
641    machine_id: &str,
642    cursors_path: &Path,
643    dry_run: bool,
644) -> KimetsuResult<SyncReport> {
645    let mut cursors = SyncCursors::load(cursors_path)?;
646    let export_since = cursors.cursor_for(machine_id);
647
648    // --- push ---
649    let push_summary = push_machine_batch(conn, sync_dir, machine_id, export_since, dry_run)?;
650    if !dry_run && push_summary.exported > 0 {
651        cursors.set_cursor(machine_id, push_summary.next_cursor);
652        cursors.save(cursors_path)?;
653    }
654
655    // --- pull ---
656    let mut total_applied = 0usize;
657    let mut total_skipped = 0usize;
658    let mut machines_pulled: Vec<String> = Vec::new();
659
660    // List subdirs (each = one machine's batch directory).
661    if sync_dir.exists() {
662        let entries = fs::read_dir(sync_dir)
663            .map_err(|e| format!("sync: cannot read sync_dir {:?}: {e}", sync_dir.display()))?;
664        let mut other_machines: Vec<String> = Vec::new();
665        for entry in entries {
666            let entry = entry.map_err(|e| format!("sync: dir entry error: {e}"))?;
667            if entry.path().is_dir() {
668                if let Some(name) = entry.file_name().to_str() {
669                    if name != machine_id {
670                        other_machines.push(name.to_string());
671                    }
672                }
673            }
674        }
675        other_machines.sort(); // deterministic order
676
677        let mut incoming = String::new();
678        for other_id in &other_machines {
679            let since = cursors.cursor_for(other_id);
680            let (jsonl, new_cursor) = read_machine_batches(sync_dir, other_id, since)?;
681            if !dry_run && new_cursor > since {
682                cursors.set_cursor(other_id, new_cursor);
683                machines_pulled.push(other_id.clone());
684            } else if dry_run && !jsonl.trim().is_empty() {
685                machines_pulled.push(other_id.clone());
686            }
687            incoming.push_str(&jsonl);
688        }
689
690        // Commit all peer events and their causal projection together before
691        // advancing pull cursors. A failed import leaves both unchanged.
692        let pull_summary = import_events(conn, &incoming, dry_run)?;
693        total_applied = pull_summary.applied;
694        total_skipped = pull_summary.skipped;
695
696        if !dry_run && !machines_pulled.is_empty() {
697            cursors.save(cursors_path)?;
698        }
699    }
700
701    Ok(SyncReport {
702        pushed: push_summary.exported,
703        pulled_applied: total_applied,
704        pulled_skipped: total_skipped,
705        machines_pulled,
706        dry_run,
707    })
708}
709
710/// Summary of a full sync cycle.
711#[derive(Debug, Clone, Default)]
712pub struct SyncReport {
713    pub pushed: usize,
714    pub pulled_applied: usize,
715    pub pulled_skipped: usize,
716    pub machines_pulled: Vec<String>,
717    pub dry_run: bool,
718}
719
720// ---------------------------------------------------------------------------
721// 3.3 — Doctor / status
722// ---------------------------------------------------------------------------
723
724/// Status of the sync configuration and state.
725#[derive(Debug, Clone)]
726pub struct SyncStatus {
727    pub sync_dir: Option<PathBuf>,
728    pub machine_id: String,
729    /// Per-source: (machine_id, cursor, pending_count)
730    pub sources: Vec<(String, i64, usize)>,
731    pub local_pending: usize,
732}
733
734/// Compute the sync status without performing any writes.
735pub fn sync_status(
736    conn: &Connection,
737    sync_dir_opt: Option<&Path>,
738    machine_id: &str,
739    cursors_path: &Path,
740) -> KimetsuResult<SyncStatus> {
741    let cursors = SyncCursors::load(cursors_path)?;
742    let export_since = cursors.cursor_for(machine_id);
743
744    // Count this machine's unpushed events.
745    let (push_dry, _) = export_events(conn, export_since, None, true)?;
746    let local_pending = push_dry.exported;
747
748    let mut sources: Vec<(String, i64, usize)> = Vec::new();
749    if let Some(sd) = sync_dir_opt {
750        if sd.exists() {
751            let entries = fs::read_dir(sd)
752                .map_err(|e| format!("sync status: cannot read {:?}: {e}", sd.display()))?;
753            let mut other_machines: Vec<String> = Vec::new();
754            for entry in entries {
755                let entry = entry.map_err(|e| format!("sync status: dir entry error: {e}"))?;
756                if entry.path().is_dir() {
757                    if let Some(name) = entry.file_name().to_str() {
758                        if name != machine_id {
759                            other_machines.push(name.to_string());
760                        }
761                    }
762                }
763            }
764            other_machines.sort();
765            for other_id in &other_machines {
766                let since = cursors.cursor_for(other_id);
767                let (pull_summary, _) = pull_machine_batches(conn, sd, other_id, since, true)?;
768                sources.push((
769                    other_id.clone(),
770                    since,
771                    pull_summary.applied + pull_summary.skipped,
772                ));
773            }
774        }
775    }
776
777    Ok(SyncStatus {
778        sync_dir: sync_dir_opt.map(|p| p.to_path_buf()),
779        machine_id: machine_id.to_string(),
780        sources,
781        local_pending,
782    })
783}
784
785// ---------------------------------------------------------------------------
786// Atomic write helper
787// ---------------------------------------------------------------------------
788
789/// Write `data` to `path` atomically via a sibling temp file + rename.
790pub fn atomic_write(path: &Path, data: &[u8]) -> KimetsuResult<()> {
791    let parent = path.parent().unwrap_or(Path::new("."));
792    fs::create_dir_all(parent).map_err(|e| {
793        format!(
794            "atomic_write: cannot create dir {:?}: {e}",
795            parent.display()
796        )
797    })?;
798    let tmp_path = path.with_extension("tmp");
799    {
800        let mut file = fs::File::create(&tmp_path).map_err(|e| {
801            format!(
802                "atomic_write: cannot create tmp {:?}: {e}",
803                tmp_path.display()
804            )
805        })?;
806        file.write_all(data)
807            .map_err(|e| format!("atomic_write: write error {:?}: {e}", tmp_path.display()))?;
808        file.flush()
809            .map_err(|e| format!("atomic_write: flush error {:?}: {e}", tmp_path.display()))?;
810    }
811    fs::rename(&tmp_path, path).map_err(|e| {
812        format!(
813            "atomic_write: rename {:?} -> {:?}: {e}",
814            tmp_path.display(),
815            path.display()
816        )
817    })?;
818    Ok(())
819}
820
821// ---------------------------------------------------------------------------
822// Extension trait for Option with rusqlite
823// ---------------------------------------------------------------------------
824
825trait OptionalExt<T> {
826    fn optional(self) -> KimetsuResult<Option<T>>;
827}
828
829impl<T> OptionalExt<T> for rusqlite::Result<T> {
830    fn optional(self) -> KimetsuResult<Option<T>> {
831        match self {
832            Ok(v) => Ok(Some(v)),
833            Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
834            Err(e) => Err(e.into()),
835        }
836    }
837}
838
839// ---------------------------------------------------------------------------
840// Tests
841// ---------------------------------------------------------------------------
842
843#[cfg(test)]
844mod tests {
845    use kimetsu_core::ids::RunId;
846    use rusqlite::Connection;
847    use serde_json::json;
848
849    use super::*;
850    use crate::projector::apply_events;
851    use crate::schema;
852
853    fn make_conn() -> Connection {
854        let conn = Connection::open_in_memory().expect("open_in_memory");
855        schema::initialize(&conn).expect("schema init");
856        conn
857    }
858
859    fn wire(events: &[Event]) -> String {
860        events
861            .iter()
862            .map(|event| serde_json::to_string(&SyncEvent::from(event)).unwrap())
863            .collect::<Vec<_>>()
864            .join("\n")
865    }
866
867    #[test]
868    fn sync_replays_historical_correction_before_local_retirement() {
869        for reason in ["retired", "forgotten/archived"] {
870            let a = make_conn();
871            let b = make_conn();
872            let run = RunId::new();
873            let accepted = Event::new(
874                run,
875                "memory.accepted",
876                json!({"memory_id":"m", "text":"old claim", "scope":"project", "kind":"fact"}),
877            );
878            apply_events(&a, std::slice::from_ref(&accepted)).unwrap();
879            apply_events(&b, &[accepted]).unwrap();
880            let exposure = Event::new(run, "context.injected", json!({"memory_ids":["m"]}));
881            apply_events(&b, std::slice::from_ref(&exposure)).unwrap();
882            let correction = Event::new(
883                run,
884                "memory.corrected",
885                json!({"memory_id":"m", "text":"corrected claim"}),
886            );
887            apply_events(&a, std::slice::from_ref(&correction)).unwrap();
888            apply_events(
889                &b,
890                &[Event::new(
891                    run,
892                    "memory.invalidated",
893                    json!({"memory_id":"m", "reason":reason}),
894                )],
895            )
896            .unwrap();
897            let tmp = tempfile::tempdir().unwrap();
898            let sd = tmp.path().join("sync");
899            push_machine_batch(&a, &sd, "a", 0, false).unwrap();
900            let cp = tmp.path().join("b-cursors.json");
901            let report = sync_dir(&b, &sd, "b", &cp, false)
902                .expect("historical correction must replay before retirement");
903            assert_eq!((report.pulled_applied, report.pulled_skipped), (1, 1));
904            for _ in 0..2 {
905                let state: (String, String) = b
906                    .query_row(
907                        "SELECT text, invalidated_reason FROM memories WHERE memory_id='m'",
908                        [],
909                        |r| Ok((r.get(0)?, r.get(1)?)),
910                    )
911                    .unwrap();
912                assert_eq!(state, ("corrected claim".into(), reason.into()));
913                let binding: String = b.query_row("SELECT json_extract(payload_json,'$.memory_revisions.m') FROM events WHERE event_id=?1", [exposure.event_id.to_string()], |r| r.get(0)).unwrap();
914                assert_eq!(binding, "baseline:m");
915                assert_eq!(
916                    b.query_row(
917                        "SELECT count(*) FROM memories_fts WHERE memory_id='m'",
918                        [],
919                        |r| r.get::<_, i64>(0)
920                    )
921                    .unwrap(),
922                    0
923                );
924                projector::rebuild_in_place(&b).unwrap();
925            }
926            assert_eq!(SyncCursors::load(&cp).unwrap().cursor_for("a"), 2);
927            assert_eq!(
928                sync_dir(&b, &sd, "b", &cp, false).unwrap().pulled_applied,
929                0
930            );
931        }
932    }
933
934    #[test]
935    fn sync_archive_restore_round_trip() {
936        let a = make_conn();
937        let b = make_conn();
938        let run = RunId::new();
939        apply_events(&a, &[Event::new(run, "memory.accepted", json!({"memory_id":"m", "text":"restorable claim", "scope":"project", "kind":"fact"}))]).unwrap();
940        let tmp = tempfile::tempdir().unwrap();
941        let sd = tmp.path().join("sync");
942        let ca = tmp.path().join("a.json");
943        let cb = tmp.path().join("b.json");
944        sync_dir(&a, &sd, "a", &ca, false).unwrap();
945        sync_dir(&b, &sd, "b", &cb, false).unwrap();
946        for (kind, archived) in [("memory.invalidated", true), ("memory.restored", false)] {
947            apply_events(
948                &a,
949                &[Event::new(
950                    run,
951                    kind,
952                    json!({"memory_id":"m", "reason":"forgotten/archived"}),
953                )],
954            )
955            .unwrap();
956            assert!(sync_dir(&a, &sd, "a", &ca, false).unwrap().pushed > 0);
957            assert_eq!(
958                sync_dir(&b, &sd, "b", &cb, false).unwrap().pulled_applied,
959                1
960            );
961            let actual: bool = b
962                .query_row(
963                    "SELECT invalidated_at IS NOT NULL FROM memories WHERE memory_id='m'",
964                    [],
965                    |r| r.get(0),
966                )
967                .unwrap();
968            assert_eq!(actual, archived);
969            assert_eq!(
970                b.query_row(
971                    "SELECT count(*) FROM memories_fts WHERE memory_id='m'",
972                    [],
973                    |r| r.get::<_, i64>(0)
974                )
975                .unwrap(),
976                if archived { 0 } else { 1 }
977            );
978        }
979        assert_eq!(
980            sync_dir(&b, &sd, "b", &cb, false).unwrap().pulled_applied,
981            0
982        );
983    }
984
985    #[test]
986    fn sync_import_failure_rolls_back_entire_batch() {
987        for malformed_json in [false, true] {
988            let conn = make_conn();
989            let run = RunId::new();
990            let accepted = Event::new(
991                run,
992                "memory.accepted",
993                json!({"memory_id":"m", "text":"kept claim", "scope":"project", "kind":"fact"}),
994            );
995            let bad = Event::new(
996                run,
997                "memory.corrected",
998                json!({"memory_id":"missing", "text":"bad claim"}),
999            );
1000            let input = if malformed_json {
1001                format!("{}\n{{bad", wire(&[accepted]))
1002            } else {
1003                wire(&[accepted, bad])
1004            };
1005            assert!(import_events(&conn, &input, false).is_err());
1006            for table in ["events", "memories", "memory_revisions", "memories_fts"] {
1007                assert_eq!(
1008                    conn.query_row(&format!("SELECT count(*) FROM {table}"), [], |r| r
1009                        .get::<_, i64>(0))
1010                        .unwrap(),
1011                    0,
1012                    "{table} must roll back"
1013                );
1014            }
1015        }
1016    }
1017
1018    #[test]
1019    fn sync_directory_merges_peer_dependencies_before_replay() {
1020        let conn = make_conn();
1021        let run = RunId::new();
1022        let accepted = Event::new(
1023            run,
1024            "memory.accepted",
1025            json!({"memory_id":"m", "text":"old", "scope":"project", "kind":"fact"}),
1026        );
1027        let correction = Event::new(
1028            run,
1029            "memory.corrected",
1030            json!({"memory_id":"m", "text":"new"}),
1031        );
1032        let tmp = tempfile::tempdir().unwrap();
1033        let sd = tmp.path().join("sync");
1034        atomic_write(&sd.join("a/2.jsonl"), wire(&[correction]).as_bytes()).unwrap();
1035        atomic_write(&sd.join("z/1.jsonl"), wire(&[accepted]).as_bytes()).unwrap();
1036        let cp = tmp.path().join("cursors.json");
1037        let report = sync_dir(&conn, &sd, "local", &cp, false)
1038            .expect("replay must include all peers before resolving dependencies");
1039        assert_eq!(report.pulled_applied, 2);
1040        assert_eq!(
1041            conn.query_row("SELECT text FROM memories WHERE memory_id='m'", [], |r| {
1042                r.get::<_, String>(0)
1043            })
1044            .unwrap(),
1045            "new"
1046        );
1047        let cursors = SyncCursors::load(&cp).unwrap();
1048        assert_eq!((cursors.cursor_for("a"), cursors.cursor_for("z")), (2, 1));
1049    }
1050
1051    #[test]
1052    fn sync_directory_failure_preserves_projection_and_pull_cursors() {
1053        let conn = make_conn();
1054        let run = RunId::new();
1055        let accepted = Event::new(
1056            run,
1057            "memory.accepted",
1058            json!({"memory_id":"m", "text":"old", "scope":"project", "kind":"fact"}),
1059        );
1060        apply_events(&conn, &[accepted]).unwrap();
1061        let correction = Event::new(
1062            run,
1063            "memory.corrected",
1064            json!({"memory_id":"m", "text":"new"}),
1065        );
1066        let bad = Event::new(
1067            run,
1068            "memory.corrected",
1069            json!({"memory_id":"missing", "text":"bad"}),
1070        );
1071        let tmp = tempfile::tempdir().unwrap();
1072        let sd = tmp.path().join("sync");
1073        atomic_write(&sd.join("a/2.jsonl"), wire(&[correction]).as_bytes()).unwrap();
1074        atomic_write(&sd.join("z/3.jsonl"), wire(&[bad]).as_bytes()).unwrap();
1075        let cp = tmp.path().join("cursors.json");
1076        assert!(sync_dir(&conn, &sd, "local", &cp, false).is_err());
1077        assert_eq!(
1078            conn.query_row("SELECT text FROM memories WHERE memory_id='m'", [], |r| {
1079                r.get::<_, String>(0)
1080            })
1081            .unwrap(),
1082            "old"
1083        );
1084        assert_eq!(
1085            conn.query_row("SELECT count(*) FROM events", [], |r| r.get::<_, i64>(0))
1086                .unwrap(),
1087            1
1088        );
1089        let cursors = SyncCursors::load(&cp).unwrap();
1090        assert_eq!((cursors.cursor_for("a"), cursors.cursor_for("z")), (0, 0));
1091    }
1092
1093    #[test]
1094    fn sync_import_refuses_to_erase_unlogged_memory() {
1095        let conn = make_conn();
1096        conn.execute("INSERT INTO memories(memory_id,scope,kind,text,normalized_text,confidence,provenance_snapshot_json,created_at) VALUES ('legacy','global_user','fact','original','original',0.7,'{}','2020-01-01T00:00:00Z')", []).unwrap();
1097        let accepted = Event::new(
1098            RunId::new(),
1099            "memory.accepted",
1100            json!({"memory_id":"m", "text":"new", "scope":"project", "kind":"fact"}),
1101        );
1102        let error = import_events(&conn, &wire(&[accepted]), false).unwrap_err();
1103        assert!(error.to_string().contains("absent from replay"));
1104        assert_eq!(
1105            conn.query_row("SELECT count(*) FROM events", [], |r| r.get::<_, i64>(0))
1106                .unwrap(),
1107            0
1108        );
1109        assert_eq!(
1110            conn.query_row(
1111                "SELECT text FROM memories WHERE memory_id='legacy'",
1112                [],
1113                |r| r.get::<_, String>(0)
1114            )
1115            .unwrap(),
1116            "original"
1117        );
1118    }
1119
1120    #[test]
1121    fn sync_import_counts_duplicate_lines_in_dry_run_and_commit() {
1122        let conn = make_conn();
1123        let accepted = Event::new(
1124            RunId::new(),
1125            "memory.accepted",
1126            json!({"memory_id":"m", "text":"new", "scope":"project", "kind":"fact"}),
1127        );
1128        let input = wire(&[accepted.clone(), accepted]);
1129        for dry in [true, false] {
1130            let summary = import_events(&conn, &input, dry).unwrap();
1131            assert_eq!((summary.applied, summary.skipped), (1, 1));
1132        }
1133        let summary = import_events(&conn, &input, false).unwrap();
1134        assert_eq!((summary.applied, summary.skipped), (0, 2));
1135    }
1136
1137    fn seed_events(conn: &Connection) -> (RunId, String, String) {
1138        let run_id = RunId::new();
1139        let mem_id_a = format!("mem-{}", ulid::Ulid::new());
1140        let mem_id_b = format!("mem-{}", ulid::Ulid::new());
1141        apply_events(
1142            conn,
1143            &[
1144                kimetsu_core::event::Event::new(
1145                    run_id,
1146                    "run.started",
1147                    json!({"project_id":"p","task":"t"}),
1148                ),
1149                kimetsu_core::event::Event::new(
1150                    run_id,
1151                    "memory.accepted",
1152                    json!({"memory_id": mem_id_a, "text": "always use cargo --locked", "scope": "project", "kind": "fact"}),
1153                ),
1154                kimetsu_core::event::Event::new(
1155                    run_id,
1156                    "memory.accepted",
1157                    json!({"memory_id": mem_id_b, "text": "prefer ripgrep over grep", "scope": "global_user", "kind": "preference"}),
1158                ),
1159                kimetsu_core::event::Event::new(
1160                    run_id,
1161                    "work.episode",
1162                    json!({"task":"local task","project_id":"p"}),
1163                ),
1164                kimetsu_core::event::Event::new(
1165                    run_id,
1166                    "context.served",
1167                    json!({"query":"test","results":[]}),
1168                ),
1169                kimetsu_core::event::Event::new(
1170                    run_id,
1171                    "run.finished",
1172                    json!({"total_cost_usd":0.01}),
1173                ),
1174            ],
1175        )
1176        .expect("seed events");
1177        (run_id, mem_id_a, mem_id_b)
1178    }
1179
1180    // Slice B headline: two brains that exchange the same events CONVERGE to an
1181    // identical projection regardless of the order edits were made/imported —
1182    // including the one genuinely-divergent op (memory.superseded), which an HLC
1183    // replay resolves last-writer-wins, plus a surfaced conflict.
1184    #[test]
1185    fn two_brains_converge_after_exchange() {
1186        use kimetsu_core::event::Event;
1187        let a = make_conn();
1188        let b = make_conn();
1189        let run = RunId(ulid::Ulid::nil()); // legacy standalone reliance metadata
1190        let (m1, s1, s2) = ("mem-m1", "mem-s1", "mem-s2");
1191
1192        // Shared base: identical accepted events on both brains.
1193        let base = vec![
1194            Event::new(
1195                run,
1196                "memory.accepted",
1197                json!({"memory_id": m1, "text":"alpha rule", "scope":"project","kind":"fact"}),
1198            ),
1199            Event::new(
1200                run,
1201                "memory.accepted",
1202                json!({"memory_id": s1, "text":"survivor one", "scope":"project","kind":"fact"}),
1203            ),
1204            Event::new(
1205                run,
1206                "memory.accepted",
1207                json!({"memory_id": s2, "text":"survivor two", "scope":"project","kind":"fact"}),
1208            ),
1209        ];
1210        apply_events(&a, &base).unwrap();
1211        apply_events(&b, &base).unwrap();
1212
1213        // Divergent edits, created in sequence so B's supersede has a LATER HLC.
1214        let a_mut = vec![
1215            Event::new(run, "memory.cited", json!({"memory_id": m1, "turn": 0})),
1216            Event::new(
1217                run,
1218                "memory.superseded",
1219                json!({"memory_id": m1, "survivor_id": s1}),
1220            ),
1221        ];
1222        apply_events(&a, &a_mut).unwrap();
1223        let b_mut = vec![
1224            Event::new(run, "memory.cited", json!({"memory_id": m1, "turn": 0})),
1225            Event::new(
1226                run,
1227                "memory.superseded",
1228                json!({"memory_id": m1, "survivor_id": s2}),
1229            ),
1230        ];
1231        apply_events(&b, &b_mut).unwrap();
1232
1233        // Cross-exchange the full logs, then converge (rebuild in HLC order).
1234        let ax = export_events(&a, 0, None, false).unwrap().1.unwrap();
1235        let bx = export_events(&b, 0, None, false).unwrap().1.unwrap();
1236        import_events(&b, &ax, false).unwrap();
1237        import_events(&a, &bx, false).unwrap();
1238        crate::projector::rebuild_in_place(&a).unwrap();
1239        crate::projector::rebuild_in_place(&b).unwrap();
1240
1241        // superseded_by converges to the LATER-HLC survivor (s2) on BOTH brains.
1242        let superseded = |c: &Connection| -> Option<String> {
1243            c.query_row(
1244                "SELECT superseded_by FROM memories WHERE memory_id = ?1",
1245                [m1],
1246                |r| r.get::<_, Option<String>>(0),
1247            )
1248            .unwrap()
1249        };
1250        assert_eq!(
1251            superseded(&a),
1252            superseded(&b),
1253            "superseded_by must converge"
1254        );
1255        assert_eq!(
1256            superseded(&a),
1257            Some(s2.to_string()),
1258            "later-HLC supersede wins deterministically"
1259        );
1260
1261        // Reliance metadata converges without manufacturing outcome credit.
1262        let use_count = |c: &Connection| -> i64 {
1263            c.query_row(
1264                "SELECT use_count FROM memories WHERE memory_id = ?1",
1265                [m1],
1266                |r| r.get(0),
1267            )
1268            .unwrap()
1269        };
1270        assert_eq!(use_count(&a), use_count(&b), "use_count must converge");
1271        assert_eq!(use_count(&a), 0, "citations alone are not outcome credit");
1272        for brain in [&a, &b] {
1273            assert_eq!(
1274                brain
1275                    .query_row("SELECT count(*) FROM memory_citations", [], |r| r
1276                        .get::<_, i64>(0))
1277                    .unwrap(),
1278                2
1279            );
1280        }
1281
1282        // Even order-sensitive confidence converges (same HLC replay order).
1283        let confidence = |c: &Connection| -> f64 {
1284            c.query_row(
1285                "SELECT confidence FROM memories WHERE memory_id = ?1",
1286                [m1],
1287                |r| r.get(0),
1288            )
1289            .unwrap()
1290        };
1291        assert!(
1292            (confidence(&a) - confidence(&b)).abs() < 1e-9,
1293            "confidence must converge: {} vs {}",
1294            confidence(&a),
1295            confidence(&b)
1296        );
1297
1298        // The genuine concurrent supersede is surfaced (once) on both brains.
1299        assert_eq!(sync_conflict_count(&a).unwrap(), 1);
1300        assert_eq!(sync_conflict_count(&b).unwrap(), 1);
1301    }
1302
1303    // S3-1: Export excludes telemetry + work.episode; only memory.* kinds appear.
1304    #[test]
1305    fn export_excludes_local_only_kinds() {
1306        let conn = make_conn();
1307        seed_events(&conn);
1308        let (summary, content) = export_events(&conn, 0, None, false).expect("export");
1309        let jsonl = content.expect("content must be Some when out_path is None");
1310        assert!(summary.exported > 0, "must export at least 1 event");
1311        assert!(
1312            summary.exported <= 2,
1313            "only memory.accepted events (2 max); got {}",
1314            summary.exported
1315        );
1316        for line in jsonl.lines() {
1317            if line.trim().is_empty() {
1318                continue;
1319            }
1320            let se: SyncEvent = serde_json::from_str(line).expect("valid json");
1321            assert!(
1322                is_sync_allowed(&se.kind),
1323                "exported kind {:?} is NOT on the allowlist",
1324                se.kind
1325            );
1326            assert_ne!(
1327                se.kind, "work.episode",
1328                "work.episode must never be exported"
1329            );
1330            assert_ne!(
1331                se.kind, "context.served",
1332                "context.served must never be exported"
1333            );
1334            assert_ne!(
1335                se.kind, "run.started",
1336                "run metadata must never be exported"
1337            );
1338            assert_ne!(
1339                se.kind, "run.finished",
1340                "run metadata must never be exported"
1341            );
1342        }
1343    }
1344
1345    // S3-2: Import is idempotent — re-importing the same batch is a NO-OP.
1346    #[test]
1347    fn import_is_idempotent() {
1348        let conn_a = make_conn();
1349        seed_events(&conn_a);
1350        let (_, content) = export_events(&conn_a, 0, None, false).expect("export");
1351        let jsonl = content.expect("content");
1352
1353        let conn_b = make_conn();
1354        let s1 = import_events(&conn_b, &jsonl, false).expect("first import");
1355        assert!(s1.applied > 0, "first import must apply events");
1356        assert_eq!(s1.skipped, 0, "first import must have 0 skipped");
1357
1358        let s2 = import_events(&conn_b, &jsonl, false).expect("second import");
1359        assert_eq!(s2.applied, 0, "re-import must apply 0 (idempotent)");
1360        assert_eq!(
1361            s2.skipped, s1.applied,
1362            "all events must be skipped on re-import"
1363        );
1364    }
1365
1366    // S3-3: Round-trip — memories exported from brain A appear in brain B.
1367    #[test]
1368    fn round_trip_export_import() {
1369        let conn_a = make_conn();
1370        let (_, mem_id_a, mem_id_b) = seed_events(&conn_a);
1371        let (_, content) = export_events(&conn_a, 0, None, false).expect("export");
1372        let jsonl = content.expect("content");
1373
1374        let conn_b = make_conn();
1375        let s = import_events(&conn_b, &jsonl, false).expect("import");
1376        assert!(s.applied > 0, "must have applied events");
1377
1378        // Verify memories are projected in brain B.
1379        let count: i64 = conn_b
1380            .query_row("SELECT COUNT(*) FROM memories", [], |r| r.get(0))
1381            .expect("count");
1382        assert!(
1383            count >= 1,
1384            "at least one memory must appear in B after import"
1385        );
1386
1387        // Both memory ids should exist.
1388        for mid in [&mem_id_a, &mem_id_b] {
1389            let exists: i64 = conn_b
1390                .query_row(
1391                    "SELECT COUNT(*) FROM memories WHERE memory_id = ?1",
1392                    rusqlite::params![mid],
1393                    |r| r.get(0),
1394                )
1395                .expect("exists check");
1396            assert_eq!(exists, 1, "memory {} must exist in B after import", mid);
1397        }
1398    }
1399
1400    // S3-4: Cursor advance — second export only emits the new event.
1401    #[test]
1402    fn cursor_advances_correctly() {
1403        let conn = make_conn();
1404        seed_events(&conn);
1405        let (summary1, _) = export_events(&conn, 0, None, false).expect("export 1");
1406        let cursor_after_first = summary1.next_cursor;
1407
1408        // Add one more memory.accepted event.
1409        let run_id = RunId::new();
1410        let mem_id_c = format!("mem-c-{}", ulid::Ulid::new());
1411        apply_events(
1412            &conn,
1413            &[kimetsu_core::event::Event::new(
1414                run_id,
1415                "memory.accepted",
1416                json!({"memory_id": mem_id_c, "text": "new after cursor", "scope": "project", "kind": "fact"}),
1417            )],
1418        )
1419        .expect("add new event");
1420
1421        let (summary2, content2) =
1422            export_events(&conn, cursor_after_first, None, false).expect("export 2");
1423        let jsonl2 = content2.expect("content");
1424        assert_eq!(
1425            summary2.exported, 1,
1426            "second export must emit exactly 1 new event"
1427        );
1428        let se: SyncEvent = serde_json::from_str(jsonl2.trim()).expect("parse");
1429        let payload_mid = se
1430            .payload
1431            .get("memory_id")
1432            .and_then(|v| v.as_str())
1433            .unwrap_or("");
1434        assert_eq!(
1435            payload_mid, mem_id_c,
1436            "cursor must only export the new event"
1437        );
1438    }
1439
1440    // S3-5: Redaction — secrets in memory.accepted payloads are redacted in export.
1441    #[test]
1442    fn export_redacts_secrets() {
1443        let conn = make_conn();
1444        let run_id = RunId::new();
1445        let secret = "sk-ant-api03-AbCdEfGhIjKlMnOpQrStUv0123456789AbCdEf";
1446        apply_events(
1447            &conn,
1448            &[kimetsu_core::event::Event::new(
1449                run_id,
1450                "memory.accepted",
1451                json!({
1452                    "memory_id": "mem-secret",
1453                    "text": format!("do not use {secret}"),
1454                    "scope": "project",
1455                    "kind": "fact"
1456                }),
1457            )],
1458        )
1459        .expect("seed");
1460        let (_, content) = export_events(&conn, 0, None, false).expect("export");
1461        let jsonl = content.expect("content");
1462        assert!(
1463            !jsonl.contains(secret),
1464            "exported batch must NOT contain the secret"
1465        );
1466        assert!(
1467            jsonl.contains("[REDACTED:anthropic_oauth]"),
1468            "exported batch must contain the REDACTED placeholder"
1469        );
1470    }
1471
1472    // S3-6: Dry-run on import reports what WOULD apply without writing.
1473    #[test]
1474    fn dry_run_import_does_not_write() {
1475        let conn_a = make_conn();
1476        seed_events(&conn_a);
1477        let (_, content) = export_events(&conn_a, 0, None, false).expect("export");
1478        let jsonl = content.expect("content");
1479
1480        let conn_b = make_conn();
1481        let s = import_events(&conn_b, &jsonl, true).expect("dry-run import");
1482        assert!(s.applied > 0, "dry-run must report events it WOULD apply");
1483
1484        let count: i64 = conn_b
1485            .query_row("SELECT COUNT(*) FROM events", [], |r| r.get(0))
1486            .expect("count");
1487        assert_eq!(count, 0, "dry-run must NOT write any events");
1488    }
1489
1490    // S3-7: Directory protocol — push writes a file; pull reads and imports it.
1491    #[test]
1492    fn directory_protocol_push_pull() {
1493        let tmp = tempfile::tempdir().expect("tempdir");
1494        let sync_dir = tmp.path().join("sync");
1495        let cursors_path = tmp.path().join("sync-cursors.json");
1496
1497        // Brain A.
1498        let conn_a = make_conn();
1499        seed_events(&conn_a);
1500        let machine_a = "machine-a";
1501        let report =
1502            sync_dir_fn(&conn_a, &sync_dir, machine_a, &cursors_path, false).expect("sync A");
1503        assert!(report.pushed > 0, "A must push events");
1504
1505        // Brain B — pull from A.
1506        let conn_b = make_conn();
1507        let cursors_b_path = tmp.path().join("cursors-b.json");
1508        let machine_b = "machine-b";
1509        let report_b =
1510            sync_dir_fn(&conn_b, &sync_dir, machine_b, &cursors_b_path, false).expect("sync B");
1511        assert!(report_b.pulled_applied > 0, "B must import events from A");
1512
1513        // Idempotent: sync B again — nothing new to apply.
1514        let report_b2 = sync_dir_fn(&conn_b, &sync_dir, machine_b, &cursors_b_path, false)
1515            .expect("sync B again");
1516        assert_eq!(
1517            report_b2.pulled_applied, 0,
1518            "second sync B must be idempotent (0 applied)"
1519        );
1520    }
1521
1522    /// Thin wrapper so the test can call the module function by its short name.
1523    fn sync_dir_fn(
1524        conn: &Connection,
1525        sd: &Path,
1526        mid: &str,
1527        cp: &Path,
1528        dry: bool,
1529    ) -> KimetsuResult<SyncReport> {
1530        sync_dir(conn, sd, mid, cp, dry)
1531    }
1532}