1use std::collections::{BTreeMap, BTreeSet};
51use std::fs;
52use std::io::{BufRead, BufReader, Write as IoWrite};
53use std::path::{Path, PathBuf};
54
55use kimetsu_core::KimetsuResult;
56use kimetsu_core::event::Event;
57use kimetsu_core::ids::{EventId, RunId};
58use rusqlite::Connection;
59use serde::{Deserialize, Serialize};
60use time::OffsetDateTime;
61use time::format_description::well_known::Rfc3339;
62use ulid::Ulid;
63
64use crate::projector;
65use crate::redact;
66
67const SYNC_ALLOWED_KINDS: &[&str] = &[
73 "memory.accepted",
74 "memory.proposed",
75 "memory.rejected",
76 "memory.invalidated",
77 "memory.restored",
78 "memory.corrected",
79 "memory.cited",
80 "memory.superseded",
81];
82
83pub fn is_sync_allowed(kind: &str) -> bool {
85 SYNC_ALLOWED_KINDS.contains(&kind)
86}
87
88#[derive(Debug, Clone, Serialize, Deserialize)]
96pub struct SyncEvent {
97 pub event_id: String,
98 pub run_id: String,
99 #[serde(with = "time::serde::rfc3339")]
100 pub ts: OffsetDateTime,
101 pub kind: String,
102 pub schema_version: u32,
103 pub payload: serde_json::Value,
104 #[serde(default)]
108 pub origin: Option<String>,
109 #[serde(default)]
113 pub hlc: Option<String>,
114}
115
116impl From<&Event> for SyncEvent {
117 fn from(e: &Event) -> Self {
118 let payload = redact_event_payload(e);
121 Self {
122 event_id: e.event_id.to_string(),
123 run_id: e.run_id.to_string(),
124 ts: e.ts,
125 kind: e.kind.clone(),
126 schema_version: e.schema_version,
127 payload,
128 origin: e.origin.clone(),
129 hlc: e.hlc.clone(),
130 }
131 }
132}
133
134impl TryFrom<SyncEvent> for Event {
135 type Error = Box<dyn std::error::Error + Send + Sync>;
136
137 fn try_from(s: SyncEvent) -> Result<Self, Self::Error> {
138 let event_id = EventId(
139 Ulid::from_string(&s.event_id)
140 .map_err(|e| format!("invalid event_id {:?}: {e}", s.event_id))?,
141 );
142 let run_id = RunId(
143 Ulid::from_string(&s.run_id)
144 .map_err(|e| format!("invalid run_id {:?}: {e}", s.run_id))?,
145 );
146 let hlc = match s.hlc {
150 Some(h) => {
151 if let Some(parsed) = kimetsu_core::clock::Hlc::parse(&h) {
152 kimetsu_core::clock::observe(&parsed);
153 }
154 Some(h)
155 }
156 None => Some(kimetsu_core::clock::now().to_canonical()),
157 };
158 Ok(Event {
159 event_id,
160 run_id,
161 ts: s.ts,
162 parent_event_id: None,
163 kind: s.kind,
164 schema_version: s.schema_version,
165 payload: s.payload,
166 origin: s.origin,
168 hlc,
169 })
170 }
171}
172
173fn redact_event_payload(event: &Event) -> serde_json::Value {
176 if !matches!(
177 event.kind.as_str(),
178 "memory.accepted" | "memory.proposed" | "memory.cited" | "memory.corrected"
179 ) {
180 return event.payload.clone();
181 }
182 redact_json_strings_owned(&event.payload)
183}
184
185fn redact_json_strings_owned(value: &serde_json::Value) -> serde_json::Value {
186 match value {
187 serde_json::Value::String(text) => {
188 serde_json::Value::String(redact::redact_secrets(text).text)
189 }
190 serde_json::Value::Array(arr) => {
191 serde_json::Value::Array(arr.iter().map(redact_json_strings_owned).collect())
192 }
193 serde_json::Value::Object(map) => {
194 let out = map
195 .iter()
196 .map(|(k, v)| (k.clone(), redact_json_strings_owned(v)))
197 .collect();
198 serde_json::Value::Object(out)
199 }
200 other => other.clone(),
201 }
202}
203
204#[derive(Debug, Clone, Default)]
210pub struct ExportSummary {
211 pub exported: usize,
213 pub next_cursor: i64,
215}
216
217pub fn export_events(
225 conn: &Connection,
226 since_rowid: i64,
227 out_path: Option<&Path>,
228 dry_run: bool,
229) -> KimetsuResult<(ExportSummary, Option<String>)> {
230 let rows = read_durable_events_after(conn, since_rowid)?;
231
232 let mut lines = Vec::new();
233 let mut next_cursor = since_rowid;
234 for (rowid, event) in &rows {
235 if !is_sync_allowed(&event.kind) {
236 continue;
237 }
238 let se = SyncEvent::from(event);
239 let line = serde_json::to_string(&se)
240 .map_err(|e| format!("sync export: serialize event {}: {e}", event.event_id))?;
241 lines.push(line);
242 if *rowid > next_cursor {
243 next_cursor = *rowid;
244 }
245 }
246
247 let summary = ExportSummary {
248 exported: lines.len(),
249 next_cursor,
250 };
251
252 if dry_run {
253 return Ok((summary, None));
254 }
255
256 let jsonl = lines.join("\n");
257 if let Some(path) = out_path {
258 atomic_write(path, jsonl.as_bytes())?;
259 Ok((summary, None))
260 } else {
261 Ok((summary, Some(jsonl)))
262 }
263}
264
265fn read_durable_events_after(conn: &Connection, after: i64) -> KimetsuResult<Vec<(i64, Event)>> {
267 let mut stmt = conn.prepare(
268 "SELECT rowid, event_id, run_id, ts, kind, schema_version, payload_json, origin, hlc
269 FROM events
270 WHERE rowid > ?1
271 ORDER BY rowid",
272 )?;
273 let rows = stmt.query_map(rusqlite::params![after], |row| {
274 let rowid: i64 = row.get(0)?;
275 let event_id_str: String = row.get(1)?;
276 let run_id_str: String = row.get(2)?;
277 let ts_str: String = row.get(3)?;
278 let kind: String = row.get(4)?;
279 let schema_version: u32 = row.get(5)?;
280 let payload_json: String = row.get(6)?;
281 let origin: Option<String> = row.get(7)?;
282 let hlc: Option<String> = row.get(8)?;
283 Ok((
284 rowid,
285 event_id_str,
286 run_id_str,
287 ts_str,
288 kind,
289 schema_version,
290 payload_json,
291 origin,
292 hlc,
293 ))
294 })?;
295
296 let mut out = Vec::new();
297 for row in rows {
298 let (
299 rowid,
300 event_id_str,
301 run_id_str,
302 ts_str,
303 kind,
304 schema_version,
305 payload_json,
306 origin,
307 hlc,
308 ) = row?;
309 let event_id = EventId(
310 Ulid::from_string(&event_id_str)
311 .map_err(|e| format!("invalid event_id {event_id_str:?}: {e}"))?,
312 );
313 let run_id = RunId(
314 Ulid::from_string(&run_id_str)
315 .map_err(|e| format!("invalid run_id {run_id_str:?}: {e}"))?,
316 );
317 let ts = OffsetDateTime::parse(&ts_str, &Rfc3339)
318 .map_err(|e| format!("invalid ts {ts_str:?}: {e}"))?;
319 let payload: serde_json::Value = serde_json::from_str(&payload_json)?;
320 out.push((
321 rowid,
322 Event {
323 event_id,
324 run_id,
325 ts,
326 parent_event_id: None,
327 kind,
328 schema_version,
329 payload,
330 origin,
331 hlc,
332 },
333 ));
334 }
335 Ok(out)
336}
337
338#[derive(Debug, Clone, Default)]
344pub struct ImportSummary {
345 pub applied: usize,
347 pub skipped: usize,
349}
350
351pub fn import_events(
362 conn: &Connection,
363 jsonl: &str,
364 dry_run: bool,
365) -> KimetsuResult<ImportSummary> {
366 let mut excluded = 0;
367 let mut events = Vec::new();
368 for (line_no, line) in jsonl.lines().enumerate() {
369 let line = line.trim();
370 if line.is_empty() {
371 continue;
372 }
373 let se: SyncEvent = serde_json::from_str(line)
374 .map_err(|e| format!("sync import: malformed JSON on line {}: {e}", line_no + 1))?;
375
376 if !is_sync_allowed(&se.kind) {
379 excluded += 1;
381 continue;
382 }
383
384 let event: Event = Event::try_from(se)
385 .map_err(|e| format!("sync import: invalid event on line {}: {e}", line_no + 1))?;
386
387 events.push(event);
388 }
389
390 let mut summary = ImportSummary::default();
391 let mut import = |c: &Connection| -> KimetsuResult<()> {
392 summary = ImportSummary {
393 applied: 0,
394 skipped: excluded,
395 };
396 let mut seen = BTreeSet::new();
397 for event in &events {
398 let exists: bool = !seen.insert(event.event_id.to_string())
401 || c.query_row(
402 "SELECT 1 FROM events WHERE event_id = ?1",
403 rusqlite::params![event.event_id.to_string()],
404 |_| Ok(true),
405 )
406 .optional()?
407 .unwrap_or(false);
408
409 if exists {
410 summary.skipped += 1;
411 continue;
412 }
413 if !dry_run {
414 let redacted = Event {
415 payload: redact_event_payload(event),
416 ..event.clone()
417 };
418 projector::insert_event(c, &redacted)?;
419 }
420 summary.applied += 1;
421 }
422 if !dry_run && summary.applied > 0 {
423 projector::replay_locked(c)?;
424 }
425 Ok(())
426 };
427 if dry_run {
428 import(conn)?;
429 } else {
430 projector::with_write_txn(conn, import)?;
431 }
432 Ok(summary)
433}
434
435pub fn sync_conflict_count(conn: &Connection) -> KimetsuResult<i64> {
439 let n: i64 = conn.query_row("SELECT COUNT(*) FROM sync_conflicts", [], |r| r.get(0))?;
440 Ok(n)
441}
442
443pub fn import_events_from_file(
445 conn: &Connection,
446 path: &Path,
447 dry_run: bool,
448) -> KimetsuResult<ImportSummary> {
449 import_events(conn, &read_batch_file(path)?, dry_run)
450}
451
452fn read_batch_file(path: &Path) -> KimetsuResult<String> {
453 let file = fs::File::open(path)
454 .map_err(|e| format!("sync import: cannot open {:?}: {e}", path.display()))?;
455 let reader = BufReader::new(file);
456 let mut buf = String::new();
457 for line in reader.lines() {
458 let l = line.map_err(|e| format!("sync import: read error {:?}: {e}", path.display()))?;
459 buf.push_str(&l);
460 buf.push('\n');
461 }
462 Ok(buf)
463}
464
465#[derive(Debug, Clone, Default, Serialize, Deserialize)]
474pub struct SyncCursors {
475 #[serde(default)]
477 pub sources: BTreeMap<String, i64>,
478}
479
480impl SyncCursors {
481 pub fn load(path: &Path) -> KimetsuResult<Self> {
482 if !path.exists() {
483 return Ok(Self::default());
484 }
485 let text = fs::read_to_string(path)
486 .map_err(|e| format!("sync-cursors: cannot read {:?}: {e}", path.display()))?;
487 serde_json::from_str(&text)
488 .map_err(|e| format!("sync-cursors: malformed JSON at {:?}: {e}", path.display()))
489 .map_err(Into::into)
490 }
491
492 pub fn save(&self, path: &Path) -> KimetsuResult<()> {
493 let text = serde_json::to_string_pretty(self)
494 .map_err(|e| format!("sync-cursors: serialize error: {e}"))?;
495 atomic_write(path, text.as_bytes())
496 }
497
498 pub fn cursor_for(&self, machine_id: &str) -> i64 {
499 *self.sources.get(machine_id).unwrap_or(&0)
500 }
501
502 pub fn set_cursor(&mut self, machine_id: &str, rowid: i64) {
503 self.sources.insert(machine_id.to_string(), rowid);
504 }
505}
506
507pub fn max_local_sync_rowid(conn: &Connection) -> KimetsuResult<i64> {
515 let placeholders: String = SYNC_ALLOWED_KINDS
516 .iter()
517 .enumerate()
518 .map(|(i, _)| format!("?{}", i + 1))
519 .collect::<Vec<_>>()
520 .join(", ");
521 let sql = format!("SELECT COALESCE(MAX(rowid), 0) FROM events WHERE kind IN ({placeholders})");
522 let mut stmt = conn.prepare(&sql)?;
523 let params: Vec<Box<dyn rusqlite::ToSql>> = SYNC_ALLOWED_KINDS
524 .iter()
525 .map(|k| -> Box<dyn rusqlite::ToSql> { Box::new(k.to_string()) })
526 .collect();
527 let refs: Vec<&dyn rusqlite::ToSql> = params.iter().map(|p| p.as_ref()).collect();
528 let max: i64 = stmt.query_row(refs.as_slice(), |r| r.get(0))?;
529 Ok(max)
530}
531
532pub fn push_machine_batch(
537 conn: &Connection,
538 sync_dir: &Path,
539 machine_id: &str,
540 since_rowid: i64,
541 dry_run: bool,
542) -> KimetsuResult<ExportSummary> {
543 let (dry_summary, _content) = export_events(conn, since_rowid, None, true)?;
544 if dry_summary.exported == 0 || dry_run {
545 return Ok(dry_summary);
546 }
547
548 let (summary, content) = export_events(conn, since_rowid, None, false)?;
550 let jsonl = content.unwrap_or_default();
551
552 let machine_dir = sync_dir.join(machine_id);
553 fs::create_dir_all(&machine_dir).map_err(|e| {
554 format!(
555 "sync push: cannot create dir {:?}: {e}",
556 machine_dir.display()
557 )
558 })?;
559
560 let batch_name = format!("{}.jsonl", summary.next_cursor);
561 let batch_path = machine_dir.join(&batch_name);
562 atomic_write(&batch_path, jsonl.as_bytes())?;
563
564 Ok(summary)
565}
566
567pub fn pull_machine_batches(
573 conn: &Connection,
574 sync_dir: &Path,
575 source_machine_id: &str,
576 since_cursor: i64,
577 dry_run: bool,
578) -> KimetsuResult<(ImportSummary, i64)> {
579 let (jsonl, cursor) = read_machine_batches(sync_dir, source_machine_id, since_cursor)?;
580 Ok((import_events(conn, &jsonl, dry_run)?, cursor))
581}
582
583fn read_machine_batches(
586 sync_dir: &Path,
587 source_machine_id: &str,
588 since_cursor: i64,
589) -> KimetsuResult<(String, i64)> {
590 let machine_dir = sync_dir.join(source_machine_id);
591 if !machine_dir.exists() {
592 return Ok((String::new(), since_cursor));
593 }
594
595 let mut batches: Vec<(i64, PathBuf)> = Vec::new();
599 let entries = fs::read_dir(&machine_dir).map_err(|e| {
600 format!(
601 "sync pull: cannot read dir {:?}: {e}",
602 machine_dir.display()
603 )
604 })?;
605 for entry in entries {
606 let entry = entry.map_err(|e| format!("sync pull: dir entry error: {e}"))?;
607 let path = entry.path();
608 if path.extension().and_then(|e| e.to_str()) != Some("jsonl") {
609 continue;
610 }
611 if let Some(stem) = path.file_stem().and_then(|s| s.to_str()) {
612 if let Ok(cursor_val) = stem.parse::<i64>() {
613 if cursor_val > since_cursor {
614 batches.push((cursor_val, path));
615 }
616 }
617 }
618 }
619 batches.sort_by_key(|(c, _)| *c);
620
621 let mut jsonl = String::new();
622 let mut new_cursor = since_cursor;
623 for (cursor_val, batch_path) in &batches {
624 jsonl.push_str(&read_batch_file(batch_path)?);
625 if *cursor_val > new_cursor {
626 new_cursor = *cursor_val;
627 }
628 }
629 Ok((jsonl, new_cursor))
630}
631
632pub fn sync_dir(
639 conn: &Connection,
640 sync_dir: &Path,
641 machine_id: &str,
642 cursors_path: &Path,
643 dry_run: bool,
644) -> KimetsuResult<SyncReport> {
645 let mut cursors = SyncCursors::load(cursors_path)?;
646 let export_since = cursors.cursor_for(machine_id);
647
648 let push_summary = push_machine_batch(conn, sync_dir, machine_id, export_since, dry_run)?;
650 if !dry_run && push_summary.exported > 0 {
651 cursors.set_cursor(machine_id, push_summary.next_cursor);
652 cursors.save(cursors_path)?;
653 }
654
655 let mut total_applied = 0usize;
657 let mut total_skipped = 0usize;
658 let mut machines_pulled: Vec<String> = Vec::new();
659
660 if sync_dir.exists() {
662 let entries = fs::read_dir(sync_dir)
663 .map_err(|e| format!("sync: cannot read sync_dir {:?}: {e}", sync_dir.display()))?;
664 let mut other_machines: Vec<String> = Vec::new();
665 for entry in entries {
666 let entry = entry.map_err(|e| format!("sync: dir entry error: {e}"))?;
667 if entry.path().is_dir() {
668 if let Some(name) = entry.file_name().to_str() {
669 if name != machine_id {
670 other_machines.push(name.to_string());
671 }
672 }
673 }
674 }
675 other_machines.sort(); let mut incoming = String::new();
678 for other_id in &other_machines {
679 let since = cursors.cursor_for(other_id);
680 let (jsonl, new_cursor) = read_machine_batches(sync_dir, other_id, since)?;
681 if !dry_run && new_cursor > since {
682 cursors.set_cursor(other_id, new_cursor);
683 machines_pulled.push(other_id.clone());
684 } else if dry_run && !jsonl.trim().is_empty() {
685 machines_pulled.push(other_id.clone());
686 }
687 incoming.push_str(&jsonl);
688 }
689
690 let pull_summary = import_events(conn, &incoming, dry_run)?;
693 total_applied = pull_summary.applied;
694 total_skipped = pull_summary.skipped;
695
696 if !dry_run && !machines_pulled.is_empty() {
697 cursors.save(cursors_path)?;
698 }
699 }
700
701 Ok(SyncReport {
702 pushed: push_summary.exported,
703 pulled_applied: total_applied,
704 pulled_skipped: total_skipped,
705 machines_pulled,
706 dry_run,
707 })
708}
709
710#[derive(Debug, Clone, Default)]
712pub struct SyncReport {
713 pub pushed: usize,
714 pub pulled_applied: usize,
715 pub pulled_skipped: usize,
716 pub machines_pulled: Vec<String>,
717 pub dry_run: bool,
718}
719
720#[derive(Debug, Clone)]
726pub struct SyncStatus {
727 pub sync_dir: Option<PathBuf>,
728 pub machine_id: String,
729 pub sources: Vec<(String, i64, usize)>,
731 pub local_pending: usize,
732}
733
734pub fn sync_status(
736 conn: &Connection,
737 sync_dir_opt: Option<&Path>,
738 machine_id: &str,
739 cursors_path: &Path,
740) -> KimetsuResult<SyncStatus> {
741 let cursors = SyncCursors::load(cursors_path)?;
742 let export_since = cursors.cursor_for(machine_id);
743
744 let (push_dry, _) = export_events(conn, export_since, None, true)?;
746 let local_pending = push_dry.exported;
747
748 let mut sources: Vec<(String, i64, usize)> = Vec::new();
749 if let Some(sd) = sync_dir_opt {
750 if sd.exists() {
751 let entries = fs::read_dir(sd)
752 .map_err(|e| format!("sync status: cannot read {:?}: {e}", sd.display()))?;
753 let mut other_machines: Vec<String> = Vec::new();
754 for entry in entries {
755 let entry = entry.map_err(|e| format!("sync status: dir entry error: {e}"))?;
756 if entry.path().is_dir() {
757 if let Some(name) = entry.file_name().to_str() {
758 if name != machine_id {
759 other_machines.push(name.to_string());
760 }
761 }
762 }
763 }
764 other_machines.sort();
765 for other_id in &other_machines {
766 let since = cursors.cursor_for(other_id);
767 let (pull_summary, _) = pull_machine_batches(conn, sd, other_id, since, true)?;
768 sources.push((
769 other_id.clone(),
770 since,
771 pull_summary.applied + pull_summary.skipped,
772 ));
773 }
774 }
775 }
776
777 Ok(SyncStatus {
778 sync_dir: sync_dir_opt.map(|p| p.to_path_buf()),
779 machine_id: machine_id.to_string(),
780 sources,
781 local_pending,
782 })
783}
784
785pub fn atomic_write(path: &Path, data: &[u8]) -> KimetsuResult<()> {
791 let parent = path.parent().unwrap_or(Path::new("."));
792 fs::create_dir_all(parent).map_err(|e| {
793 format!(
794 "atomic_write: cannot create dir {:?}: {e}",
795 parent.display()
796 )
797 })?;
798 let tmp_path = path.with_extension("tmp");
799 {
800 let mut file = fs::File::create(&tmp_path).map_err(|e| {
801 format!(
802 "atomic_write: cannot create tmp {:?}: {e}",
803 tmp_path.display()
804 )
805 })?;
806 file.write_all(data)
807 .map_err(|e| format!("atomic_write: write error {:?}: {e}", tmp_path.display()))?;
808 file.flush()
809 .map_err(|e| format!("atomic_write: flush error {:?}: {e}", tmp_path.display()))?;
810 }
811 fs::rename(&tmp_path, path).map_err(|e| {
812 format!(
813 "atomic_write: rename {:?} -> {:?}: {e}",
814 tmp_path.display(),
815 path.display()
816 )
817 })?;
818 Ok(())
819}
820
821trait OptionalExt<T> {
826 fn optional(self) -> KimetsuResult<Option<T>>;
827}
828
829impl<T> OptionalExt<T> for rusqlite::Result<T> {
830 fn optional(self) -> KimetsuResult<Option<T>> {
831 match self {
832 Ok(v) => Ok(Some(v)),
833 Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
834 Err(e) => Err(e.into()),
835 }
836 }
837}
838
839#[cfg(test)]
844mod tests {
845 use kimetsu_core::ids::RunId;
846 use rusqlite::Connection;
847 use serde_json::json;
848
849 use super::*;
850 use crate::projector::apply_events;
851 use crate::schema;
852
853 fn make_conn() -> Connection {
854 let conn = Connection::open_in_memory().expect("open_in_memory");
855 schema::initialize(&conn).expect("schema init");
856 conn
857 }
858
859 fn wire(events: &[Event]) -> String {
860 events
861 .iter()
862 .map(|event| serde_json::to_string(&SyncEvent::from(event)).unwrap())
863 .collect::<Vec<_>>()
864 .join("\n")
865 }
866
867 #[test]
868 fn sync_replays_historical_correction_before_local_retirement() {
869 for reason in ["retired", "forgotten/archived"] {
870 let a = make_conn();
871 let b = make_conn();
872 let run = RunId::new();
873 let accepted = Event::new(
874 run,
875 "memory.accepted",
876 json!({"memory_id":"m", "text":"old claim", "scope":"project", "kind":"fact"}),
877 );
878 apply_events(&a, std::slice::from_ref(&accepted)).unwrap();
879 apply_events(&b, &[accepted]).unwrap();
880 let exposure = Event::new(run, "context.injected", json!({"memory_ids":["m"]}));
881 apply_events(&b, std::slice::from_ref(&exposure)).unwrap();
882 let correction = Event::new(
883 run,
884 "memory.corrected",
885 json!({"memory_id":"m", "text":"corrected claim"}),
886 );
887 apply_events(&a, std::slice::from_ref(&correction)).unwrap();
888 apply_events(
889 &b,
890 &[Event::new(
891 run,
892 "memory.invalidated",
893 json!({"memory_id":"m", "reason":reason}),
894 )],
895 )
896 .unwrap();
897 let tmp = tempfile::tempdir().unwrap();
898 let sd = tmp.path().join("sync");
899 push_machine_batch(&a, &sd, "a", 0, false).unwrap();
900 let cp = tmp.path().join("b-cursors.json");
901 let report = sync_dir(&b, &sd, "b", &cp, false)
902 .expect("historical correction must replay before retirement");
903 assert_eq!((report.pulled_applied, report.pulled_skipped), (1, 1));
904 for _ in 0..2 {
905 let state: (String, String) = b
906 .query_row(
907 "SELECT text, invalidated_reason FROM memories WHERE memory_id='m'",
908 [],
909 |r| Ok((r.get(0)?, r.get(1)?)),
910 )
911 .unwrap();
912 assert_eq!(state, ("corrected claim".into(), reason.into()));
913 let binding: String = b.query_row("SELECT json_extract(payload_json,'$.memory_revisions.m') FROM events WHERE event_id=?1", [exposure.event_id.to_string()], |r| r.get(0)).unwrap();
914 assert_eq!(binding, "baseline:m");
915 assert_eq!(
916 b.query_row(
917 "SELECT count(*) FROM memories_fts WHERE memory_id='m'",
918 [],
919 |r| r.get::<_, i64>(0)
920 )
921 .unwrap(),
922 0
923 );
924 projector::rebuild_in_place(&b).unwrap();
925 }
926 assert_eq!(SyncCursors::load(&cp).unwrap().cursor_for("a"), 2);
927 assert_eq!(
928 sync_dir(&b, &sd, "b", &cp, false).unwrap().pulled_applied,
929 0
930 );
931 }
932 }
933
934 #[test]
935 fn sync_archive_restore_round_trip() {
936 let a = make_conn();
937 let b = make_conn();
938 let run = RunId::new();
939 apply_events(&a, &[Event::new(run, "memory.accepted", json!({"memory_id":"m", "text":"restorable claim", "scope":"project", "kind":"fact"}))]).unwrap();
940 let tmp = tempfile::tempdir().unwrap();
941 let sd = tmp.path().join("sync");
942 let ca = tmp.path().join("a.json");
943 let cb = tmp.path().join("b.json");
944 sync_dir(&a, &sd, "a", &ca, false).unwrap();
945 sync_dir(&b, &sd, "b", &cb, false).unwrap();
946 for (kind, archived) in [("memory.invalidated", true), ("memory.restored", false)] {
947 apply_events(
948 &a,
949 &[Event::new(
950 run,
951 kind,
952 json!({"memory_id":"m", "reason":"forgotten/archived"}),
953 )],
954 )
955 .unwrap();
956 assert!(sync_dir(&a, &sd, "a", &ca, false).unwrap().pushed > 0);
957 assert_eq!(
958 sync_dir(&b, &sd, "b", &cb, false).unwrap().pulled_applied,
959 1
960 );
961 let actual: bool = b
962 .query_row(
963 "SELECT invalidated_at IS NOT NULL FROM memories WHERE memory_id='m'",
964 [],
965 |r| r.get(0),
966 )
967 .unwrap();
968 assert_eq!(actual, archived);
969 assert_eq!(
970 b.query_row(
971 "SELECT count(*) FROM memories_fts WHERE memory_id='m'",
972 [],
973 |r| r.get::<_, i64>(0)
974 )
975 .unwrap(),
976 if archived { 0 } else { 1 }
977 );
978 }
979 assert_eq!(
980 sync_dir(&b, &sd, "b", &cb, false).unwrap().pulled_applied,
981 0
982 );
983 }
984
985 #[test]
986 fn sync_import_failure_rolls_back_entire_batch() {
987 for malformed_json in [false, true] {
988 let conn = make_conn();
989 let run = RunId::new();
990 let accepted = Event::new(
991 run,
992 "memory.accepted",
993 json!({"memory_id":"m", "text":"kept claim", "scope":"project", "kind":"fact"}),
994 );
995 let bad = Event::new(
996 run,
997 "memory.corrected",
998 json!({"memory_id":"missing", "text":"bad claim"}),
999 );
1000 let input = if malformed_json {
1001 format!("{}\n{{bad", wire(&[accepted]))
1002 } else {
1003 wire(&[accepted, bad])
1004 };
1005 assert!(import_events(&conn, &input, false).is_err());
1006 for table in ["events", "memories", "memory_revisions", "memories_fts"] {
1007 assert_eq!(
1008 conn.query_row(&format!("SELECT count(*) FROM {table}"), [], |r| r
1009 .get::<_, i64>(0))
1010 .unwrap(),
1011 0,
1012 "{table} must roll back"
1013 );
1014 }
1015 }
1016 }
1017
1018 #[test]
1019 fn sync_directory_merges_peer_dependencies_before_replay() {
1020 let conn = make_conn();
1021 let run = RunId::new();
1022 let accepted = Event::new(
1023 run,
1024 "memory.accepted",
1025 json!({"memory_id":"m", "text":"old", "scope":"project", "kind":"fact"}),
1026 );
1027 let correction = Event::new(
1028 run,
1029 "memory.corrected",
1030 json!({"memory_id":"m", "text":"new"}),
1031 );
1032 let tmp = tempfile::tempdir().unwrap();
1033 let sd = tmp.path().join("sync");
1034 atomic_write(&sd.join("a/2.jsonl"), wire(&[correction]).as_bytes()).unwrap();
1035 atomic_write(&sd.join("z/1.jsonl"), wire(&[accepted]).as_bytes()).unwrap();
1036 let cp = tmp.path().join("cursors.json");
1037 let report = sync_dir(&conn, &sd, "local", &cp, false)
1038 .expect("replay must include all peers before resolving dependencies");
1039 assert_eq!(report.pulled_applied, 2);
1040 assert_eq!(
1041 conn.query_row("SELECT text FROM memories WHERE memory_id='m'", [], |r| {
1042 r.get::<_, String>(0)
1043 })
1044 .unwrap(),
1045 "new"
1046 );
1047 let cursors = SyncCursors::load(&cp).unwrap();
1048 assert_eq!((cursors.cursor_for("a"), cursors.cursor_for("z")), (2, 1));
1049 }
1050
1051 #[test]
1052 fn sync_directory_failure_preserves_projection_and_pull_cursors() {
1053 let conn = make_conn();
1054 let run = RunId::new();
1055 let accepted = Event::new(
1056 run,
1057 "memory.accepted",
1058 json!({"memory_id":"m", "text":"old", "scope":"project", "kind":"fact"}),
1059 );
1060 apply_events(&conn, &[accepted]).unwrap();
1061 let correction = Event::new(
1062 run,
1063 "memory.corrected",
1064 json!({"memory_id":"m", "text":"new"}),
1065 );
1066 let bad = Event::new(
1067 run,
1068 "memory.corrected",
1069 json!({"memory_id":"missing", "text":"bad"}),
1070 );
1071 let tmp = tempfile::tempdir().unwrap();
1072 let sd = tmp.path().join("sync");
1073 atomic_write(&sd.join("a/2.jsonl"), wire(&[correction]).as_bytes()).unwrap();
1074 atomic_write(&sd.join("z/3.jsonl"), wire(&[bad]).as_bytes()).unwrap();
1075 let cp = tmp.path().join("cursors.json");
1076 assert!(sync_dir(&conn, &sd, "local", &cp, false).is_err());
1077 assert_eq!(
1078 conn.query_row("SELECT text FROM memories WHERE memory_id='m'", [], |r| {
1079 r.get::<_, String>(0)
1080 })
1081 .unwrap(),
1082 "old"
1083 );
1084 assert_eq!(
1085 conn.query_row("SELECT count(*) FROM events", [], |r| r.get::<_, i64>(0))
1086 .unwrap(),
1087 1
1088 );
1089 let cursors = SyncCursors::load(&cp).unwrap();
1090 assert_eq!((cursors.cursor_for("a"), cursors.cursor_for("z")), (0, 0));
1091 }
1092
1093 #[test]
1094 fn sync_import_refuses_to_erase_unlogged_memory() {
1095 let conn = make_conn();
1096 conn.execute("INSERT INTO memories(memory_id,scope,kind,text,normalized_text,confidence,provenance_snapshot_json,created_at) VALUES ('legacy','global_user','fact','original','original',0.7,'{}','2020-01-01T00:00:00Z')", []).unwrap();
1097 let accepted = Event::new(
1098 RunId::new(),
1099 "memory.accepted",
1100 json!({"memory_id":"m", "text":"new", "scope":"project", "kind":"fact"}),
1101 );
1102 let error = import_events(&conn, &wire(&[accepted]), false).unwrap_err();
1103 assert!(error.to_string().contains("absent from replay"));
1104 assert_eq!(
1105 conn.query_row("SELECT count(*) FROM events", [], |r| r.get::<_, i64>(0))
1106 .unwrap(),
1107 0
1108 );
1109 assert_eq!(
1110 conn.query_row(
1111 "SELECT text FROM memories WHERE memory_id='legacy'",
1112 [],
1113 |r| r.get::<_, String>(0)
1114 )
1115 .unwrap(),
1116 "original"
1117 );
1118 }
1119
1120 #[test]
1121 fn sync_import_counts_duplicate_lines_in_dry_run_and_commit() {
1122 let conn = make_conn();
1123 let accepted = Event::new(
1124 RunId::new(),
1125 "memory.accepted",
1126 json!({"memory_id":"m", "text":"new", "scope":"project", "kind":"fact"}),
1127 );
1128 let input = wire(&[accepted.clone(), accepted]);
1129 for dry in [true, false] {
1130 let summary = import_events(&conn, &input, dry).unwrap();
1131 assert_eq!((summary.applied, summary.skipped), (1, 1));
1132 }
1133 let summary = import_events(&conn, &input, false).unwrap();
1134 assert_eq!((summary.applied, summary.skipped), (0, 2));
1135 }
1136
1137 fn seed_events(conn: &Connection) -> (RunId, String, String) {
1138 let run_id = RunId::new();
1139 let mem_id_a = format!("mem-{}", ulid::Ulid::new());
1140 let mem_id_b = format!("mem-{}", ulid::Ulid::new());
1141 apply_events(
1142 conn,
1143 &[
1144 kimetsu_core::event::Event::new(
1145 run_id,
1146 "run.started",
1147 json!({"project_id":"p","task":"t"}),
1148 ),
1149 kimetsu_core::event::Event::new(
1150 run_id,
1151 "memory.accepted",
1152 json!({"memory_id": mem_id_a, "text": "always use cargo --locked", "scope": "project", "kind": "fact"}),
1153 ),
1154 kimetsu_core::event::Event::new(
1155 run_id,
1156 "memory.accepted",
1157 json!({"memory_id": mem_id_b, "text": "prefer ripgrep over grep", "scope": "global_user", "kind": "preference"}),
1158 ),
1159 kimetsu_core::event::Event::new(
1160 run_id,
1161 "work.episode",
1162 json!({"task":"local task","project_id":"p"}),
1163 ),
1164 kimetsu_core::event::Event::new(
1165 run_id,
1166 "context.served",
1167 json!({"query":"test","results":[]}),
1168 ),
1169 kimetsu_core::event::Event::new(
1170 run_id,
1171 "run.finished",
1172 json!({"total_cost_usd":0.01}),
1173 ),
1174 ],
1175 )
1176 .expect("seed events");
1177 (run_id, mem_id_a, mem_id_b)
1178 }
1179
1180 #[test]
1185 fn two_brains_converge_after_exchange() {
1186 use kimetsu_core::event::Event;
1187 let a = make_conn();
1188 let b = make_conn();
1189 let run = RunId(ulid::Ulid::nil()); let (m1, s1, s2) = ("mem-m1", "mem-s1", "mem-s2");
1191
1192 let base = vec![
1194 Event::new(
1195 run,
1196 "memory.accepted",
1197 json!({"memory_id": m1, "text":"alpha rule", "scope":"project","kind":"fact"}),
1198 ),
1199 Event::new(
1200 run,
1201 "memory.accepted",
1202 json!({"memory_id": s1, "text":"survivor one", "scope":"project","kind":"fact"}),
1203 ),
1204 Event::new(
1205 run,
1206 "memory.accepted",
1207 json!({"memory_id": s2, "text":"survivor two", "scope":"project","kind":"fact"}),
1208 ),
1209 ];
1210 apply_events(&a, &base).unwrap();
1211 apply_events(&b, &base).unwrap();
1212
1213 let a_mut = vec![
1215 Event::new(run, "memory.cited", json!({"memory_id": m1, "turn": 0})),
1216 Event::new(
1217 run,
1218 "memory.superseded",
1219 json!({"memory_id": m1, "survivor_id": s1}),
1220 ),
1221 ];
1222 apply_events(&a, &a_mut).unwrap();
1223 let b_mut = vec![
1224 Event::new(run, "memory.cited", json!({"memory_id": m1, "turn": 0})),
1225 Event::new(
1226 run,
1227 "memory.superseded",
1228 json!({"memory_id": m1, "survivor_id": s2}),
1229 ),
1230 ];
1231 apply_events(&b, &b_mut).unwrap();
1232
1233 let ax = export_events(&a, 0, None, false).unwrap().1.unwrap();
1235 let bx = export_events(&b, 0, None, false).unwrap().1.unwrap();
1236 import_events(&b, &ax, false).unwrap();
1237 import_events(&a, &bx, false).unwrap();
1238 crate::projector::rebuild_in_place(&a).unwrap();
1239 crate::projector::rebuild_in_place(&b).unwrap();
1240
1241 let superseded = |c: &Connection| -> Option<String> {
1243 c.query_row(
1244 "SELECT superseded_by FROM memories WHERE memory_id = ?1",
1245 [m1],
1246 |r| r.get::<_, Option<String>>(0),
1247 )
1248 .unwrap()
1249 };
1250 assert_eq!(
1251 superseded(&a),
1252 superseded(&b),
1253 "superseded_by must converge"
1254 );
1255 assert_eq!(
1256 superseded(&a),
1257 Some(s2.to_string()),
1258 "later-HLC supersede wins deterministically"
1259 );
1260
1261 let use_count = |c: &Connection| -> i64 {
1263 c.query_row(
1264 "SELECT use_count FROM memories WHERE memory_id = ?1",
1265 [m1],
1266 |r| r.get(0),
1267 )
1268 .unwrap()
1269 };
1270 assert_eq!(use_count(&a), use_count(&b), "use_count must converge");
1271 assert_eq!(use_count(&a), 0, "citations alone are not outcome credit");
1272 for brain in [&a, &b] {
1273 assert_eq!(
1274 brain
1275 .query_row("SELECT count(*) FROM memory_citations", [], |r| r
1276 .get::<_, i64>(0))
1277 .unwrap(),
1278 2
1279 );
1280 }
1281
1282 let confidence = |c: &Connection| -> f64 {
1284 c.query_row(
1285 "SELECT confidence FROM memories WHERE memory_id = ?1",
1286 [m1],
1287 |r| r.get(0),
1288 )
1289 .unwrap()
1290 };
1291 assert!(
1292 (confidence(&a) - confidence(&b)).abs() < 1e-9,
1293 "confidence must converge: {} vs {}",
1294 confidence(&a),
1295 confidence(&b)
1296 );
1297
1298 assert_eq!(sync_conflict_count(&a).unwrap(), 1);
1300 assert_eq!(sync_conflict_count(&b).unwrap(), 1);
1301 }
1302
1303 #[test]
1305 fn export_excludes_local_only_kinds() {
1306 let conn = make_conn();
1307 seed_events(&conn);
1308 let (summary, content) = export_events(&conn, 0, None, false).expect("export");
1309 let jsonl = content.expect("content must be Some when out_path is None");
1310 assert!(summary.exported > 0, "must export at least 1 event");
1311 assert!(
1312 summary.exported <= 2,
1313 "only memory.accepted events (2 max); got {}",
1314 summary.exported
1315 );
1316 for line in jsonl.lines() {
1317 if line.trim().is_empty() {
1318 continue;
1319 }
1320 let se: SyncEvent = serde_json::from_str(line).expect("valid json");
1321 assert!(
1322 is_sync_allowed(&se.kind),
1323 "exported kind {:?} is NOT on the allowlist",
1324 se.kind
1325 );
1326 assert_ne!(
1327 se.kind, "work.episode",
1328 "work.episode must never be exported"
1329 );
1330 assert_ne!(
1331 se.kind, "context.served",
1332 "context.served must never be exported"
1333 );
1334 assert_ne!(
1335 se.kind, "run.started",
1336 "run metadata must never be exported"
1337 );
1338 assert_ne!(
1339 se.kind, "run.finished",
1340 "run metadata must never be exported"
1341 );
1342 }
1343 }
1344
1345 #[test]
1347 fn import_is_idempotent() {
1348 let conn_a = make_conn();
1349 seed_events(&conn_a);
1350 let (_, content) = export_events(&conn_a, 0, None, false).expect("export");
1351 let jsonl = content.expect("content");
1352
1353 let conn_b = make_conn();
1354 let s1 = import_events(&conn_b, &jsonl, false).expect("first import");
1355 assert!(s1.applied > 0, "first import must apply events");
1356 assert_eq!(s1.skipped, 0, "first import must have 0 skipped");
1357
1358 let s2 = import_events(&conn_b, &jsonl, false).expect("second import");
1359 assert_eq!(s2.applied, 0, "re-import must apply 0 (idempotent)");
1360 assert_eq!(
1361 s2.skipped, s1.applied,
1362 "all events must be skipped on re-import"
1363 );
1364 }
1365
1366 #[test]
1368 fn round_trip_export_import() {
1369 let conn_a = make_conn();
1370 let (_, mem_id_a, mem_id_b) = seed_events(&conn_a);
1371 let (_, content) = export_events(&conn_a, 0, None, false).expect("export");
1372 let jsonl = content.expect("content");
1373
1374 let conn_b = make_conn();
1375 let s = import_events(&conn_b, &jsonl, false).expect("import");
1376 assert!(s.applied > 0, "must have applied events");
1377
1378 let count: i64 = conn_b
1380 .query_row("SELECT COUNT(*) FROM memories", [], |r| r.get(0))
1381 .expect("count");
1382 assert!(
1383 count >= 1,
1384 "at least one memory must appear in B after import"
1385 );
1386
1387 for mid in [&mem_id_a, &mem_id_b] {
1389 let exists: i64 = conn_b
1390 .query_row(
1391 "SELECT COUNT(*) FROM memories WHERE memory_id = ?1",
1392 rusqlite::params![mid],
1393 |r| r.get(0),
1394 )
1395 .expect("exists check");
1396 assert_eq!(exists, 1, "memory {} must exist in B after import", mid);
1397 }
1398 }
1399
1400 #[test]
1402 fn cursor_advances_correctly() {
1403 let conn = make_conn();
1404 seed_events(&conn);
1405 let (summary1, _) = export_events(&conn, 0, None, false).expect("export 1");
1406 let cursor_after_first = summary1.next_cursor;
1407
1408 let run_id = RunId::new();
1410 let mem_id_c = format!("mem-c-{}", ulid::Ulid::new());
1411 apply_events(
1412 &conn,
1413 &[kimetsu_core::event::Event::new(
1414 run_id,
1415 "memory.accepted",
1416 json!({"memory_id": mem_id_c, "text": "new after cursor", "scope": "project", "kind": "fact"}),
1417 )],
1418 )
1419 .expect("add new event");
1420
1421 let (summary2, content2) =
1422 export_events(&conn, cursor_after_first, None, false).expect("export 2");
1423 let jsonl2 = content2.expect("content");
1424 assert_eq!(
1425 summary2.exported, 1,
1426 "second export must emit exactly 1 new event"
1427 );
1428 let se: SyncEvent = serde_json::from_str(jsonl2.trim()).expect("parse");
1429 let payload_mid = se
1430 .payload
1431 .get("memory_id")
1432 .and_then(|v| v.as_str())
1433 .unwrap_or("");
1434 assert_eq!(
1435 payload_mid, mem_id_c,
1436 "cursor must only export the new event"
1437 );
1438 }
1439
1440 #[test]
1442 fn export_redacts_secrets() {
1443 let conn = make_conn();
1444 let run_id = RunId::new();
1445 let secret = "sk-ant-api03-AbCdEfGhIjKlMnOpQrStUv0123456789AbCdEf";
1446 apply_events(
1447 &conn,
1448 &[kimetsu_core::event::Event::new(
1449 run_id,
1450 "memory.accepted",
1451 json!({
1452 "memory_id": "mem-secret",
1453 "text": format!("do not use {secret}"),
1454 "scope": "project",
1455 "kind": "fact"
1456 }),
1457 )],
1458 )
1459 .expect("seed");
1460 let (_, content) = export_events(&conn, 0, None, false).expect("export");
1461 let jsonl = content.expect("content");
1462 assert!(
1463 !jsonl.contains(secret),
1464 "exported batch must NOT contain the secret"
1465 );
1466 assert!(
1467 jsonl.contains("[REDACTED:anthropic_oauth]"),
1468 "exported batch must contain the REDACTED placeholder"
1469 );
1470 }
1471
1472 #[test]
1474 fn dry_run_import_does_not_write() {
1475 let conn_a = make_conn();
1476 seed_events(&conn_a);
1477 let (_, content) = export_events(&conn_a, 0, None, false).expect("export");
1478 let jsonl = content.expect("content");
1479
1480 let conn_b = make_conn();
1481 let s = import_events(&conn_b, &jsonl, true).expect("dry-run import");
1482 assert!(s.applied > 0, "dry-run must report events it WOULD apply");
1483
1484 let count: i64 = conn_b
1485 .query_row("SELECT COUNT(*) FROM events", [], |r| r.get(0))
1486 .expect("count");
1487 assert_eq!(count, 0, "dry-run must NOT write any events");
1488 }
1489
1490 #[test]
1492 fn directory_protocol_push_pull() {
1493 let tmp = tempfile::tempdir().expect("tempdir");
1494 let sync_dir = tmp.path().join("sync");
1495 let cursors_path = tmp.path().join("sync-cursors.json");
1496
1497 let conn_a = make_conn();
1499 seed_events(&conn_a);
1500 let machine_a = "machine-a";
1501 let report =
1502 sync_dir_fn(&conn_a, &sync_dir, machine_a, &cursors_path, false).expect("sync A");
1503 assert!(report.pushed > 0, "A must push events");
1504
1505 let conn_b = make_conn();
1507 let cursors_b_path = tmp.path().join("cursors-b.json");
1508 let machine_b = "machine-b";
1509 let report_b =
1510 sync_dir_fn(&conn_b, &sync_dir, machine_b, &cursors_b_path, false).expect("sync B");
1511 assert!(report_b.pulled_applied > 0, "B must import events from A");
1512
1513 let report_b2 = sync_dir_fn(&conn_b, &sync_dir, machine_b, &cursors_b_path, false)
1515 .expect("sync B again");
1516 assert_eq!(
1517 report_b2.pulled_applied, 0,
1518 "second sync B must be idempotent (0 applied)"
1519 );
1520 }
1521
1522 fn sync_dir_fn(
1524 conn: &Connection,
1525 sd: &Path,
1526 mid: &str,
1527 cp: &Path,
1528 dry: bool,
1529 ) -> KimetsuResult<SyncReport> {
1530 sync_dir(conn, sd, mid, cp, dry)
1531 }
1532}