1use std::collections::BTreeMap;
2
3use anyhow::{Context, Result, bail};
4use base64::Engine;
5use base64::engine::general_purpose::STANDARD as B64;
6use chrono::{DateTime, Utc};
7use rusqlite::{Connection, OptionalExtension, Transaction, params};
8use serde::{Deserialize, Serialize};
9use uuid::Uuid;
10
11use super::authored_order::{self, UPDATED_BY};
12use super::{
13 MEMORY_COLS, SqliteStore, blob_to_embedding, clock, embedding_to_blob, floor, parse_dt,
14 parse_relation_type, parse_uuid, row_to_entity, row_to_memory, settled, sync_dedup,
15};
16use crate::schema::{
17 Conversation, Distillation, Entity, Memory, Relation, SyncEntry, SyncNodeType, SyncOp,
18 SyncRecord,
19};
20
21pub const MERGE_CHUNK: usize = 2_000;
23
24pub const MERGE_CHUNK_BYTES: usize = 32 * 1024 * 1024;
27
28const MERGE_PROGRESS: &str = "snapshot_merge";
29
30#[derive(Debug, Clone, Serialize, Deserialize)]
34#[serde(tag = "t", rename_all = "snake_case")]
35pub enum SnapshotRow {
36 MemoryTombstone {
37 id: String,
38 node_type: String,
39 deleted_at: String,
40 machine_id: Option<String>,
41 },
42 TypeTombstone {
43 node_type: String,
44 node_id: String,
45 deleted_at: i64,
46 origin: String,
47 },
48 Conversation {
49 row: Conversation,
50 updated_at: i64,
51 updated_by: String,
52 },
53 Entity {
54 row: Entity,
55 updated_at: i64,
56 updated_by: String,
57 },
58 Memory {
60 row: Memory,
61 embedding: Option<String>,
62 },
63 Relation {
64 row: Relation,
65 updated_at: i64,
66 updated_by: String,
67 },
68 Distillation {
69 row: Distillation,
70 updated_at: i64,
71 updated_by: String,
72 },
73 Chunks {
74 memory_id: Uuid,
75 content_hash: String,
76 chunks: Vec<SnapshotChunk>,
77 },
78 IngestKey {
79 file_path: String,
80 file_hash: String,
81 ingested_at: Option<String>,
82 memory_count: Option<i64>,
83 source: Option<String>,
84 session_key: Option<String>,
85 content_hash: Option<String>,
86 legacy_ids: i64,
87 },
88 LogTail {
89 record: SyncRecord,
90 },
91 Dedup {
92 origin: String,
93 lo: i64,
94 hi: i64,
95 },
96}
97
98#[derive(Debug, Clone, Serialize, Deserialize)]
99pub struct SnapshotChunk {
100 pub ord: usize,
101 pub text: String,
102 pub embedding: Option<String>,
103}
104
105impl SnapshotRow {
106 pub fn section(&self) -> &'static str {
107 match self {
108 SnapshotRow::MemoryTombstone { .. } => "tombstones",
109 SnapshotRow::TypeTombstone { .. } => "type_tombstones",
110 SnapshotRow::Conversation { .. } => "conversations",
111 SnapshotRow::Entity { .. } => "entities",
112 SnapshotRow::Memory { .. } => "memories",
113 SnapshotRow::Relation { .. } => "relations",
114 SnapshotRow::Distillation { .. } => "distillation_log",
115 SnapshotRow::Chunks { .. } => "memory_chunks",
116 SnapshotRow::IngestKey { .. } => "ingest_log",
117 SnapshotRow::LogTail { .. } => "log_tail",
118 SnapshotRow::Dedup { .. } => "sync_dedup",
119 }
120 }
121}
122
123#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
124pub struct SnapshotOptions {
125 pub with_chunks: bool,
126}
127
128#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
131pub struct SnapshotState {
132 pub seq: u64,
134 pub cursors: BTreeMap<String, u64>,
136 pub counts: BTreeMap<String, u64>,
137}
138
139#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
140pub struct MergeProgress {
141 pub snapshot_id: String,
142 pub exporter: String,
143 pub seq: u64,
144 pub rows_done: u64,
145 pub next_n: u64,
148 pub changed: u64,
149 pub settled: u64,
150}
151
152#[derive(Debug, Clone, Default, PartialEq, Eq)]
153pub struct MergeOutcome {
154 pub rows: u64,
155 pub changed: u64,
156 pub settled: u64,
157}
158
159impl SqliteStore {
160 pub fn export_snapshot(
165 &self,
166 options: SnapshotOptions,
167 sink: &mut dyn FnMut(SnapshotRow) -> Result<()>,
168 ) -> Result<SnapshotState> {
169 let conn = self.reader();
170 let tx = conn.unchecked_transaction()?;
171 let seq = u64::try_from(clock::state(&tx)?.last.max(0))?;
172 let mut state = SnapshotState {
173 seq,
174 cursors: export_cursors(&tx, self.machine_id())?,
175 counts: BTreeMap::new(),
176 };
177 let mut emit = |row: SnapshotRow| -> Result<()> {
178 *state.counts.entry(row.section().to_string()).or_insert(0) += 1;
179 sink(row)
180 };
181 export_tombstones(&tx, &mut emit)?;
182 export_conversations(&tx, &mut emit)?;
183 export_entities(&tx, &mut emit)?;
184 export_memories(self, &tx, &mut emit)?;
185 export_relations(&tx, &mut emit)?;
186 export_distillations(&tx, &mut emit)?;
187 if options.with_chunks {
188 export_chunks(&tx, &mut emit)?;
189 }
190 export_ingest_keys(&tx, &mut emit)?;
191 let tail_after = floor::max_live_ack(&tx, self.machine_id())?.unwrap_or(0);
192 export_tail(&tx, tail_after, seq, &mut emit)?;
193 export_dedup(&tx, &mut emit)?;
194 tx.commit()?;
195 Ok(state)
196 }
197
198 pub fn snapshot_merge_progress(&self) -> Result<Option<MergeProgress>> {
199 read_progress(&self.reader())
200 }
201
202 pub fn begin_snapshot_merge(
206 &self,
207 snapshot_id: &str,
208 exporter: &str,
209 seq: u64,
210 ) -> Result<SnapshotMerge<'_>> {
211 let stored = read_progress(&self.reader())?;
212 let progress = match stored {
213 Some(progress) if progress.snapshot_id == snapshot_id => progress,
214 _ => {
215 let progress = MergeProgress {
216 snapshot_id: snapshot_id.to_string(),
217 exporter: exporter.to_string(),
218 seq,
219 ..MergeProgress::default()
220 };
221 write_progress(&self.conn(), &progress)?;
222 progress
223 }
224 };
225 Ok(SnapshotMerge {
226 store: self,
227 progress,
228 index: 0,
229 batch: Vec::with_capacity(MERGE_CHUNK),
230 batch_bytes: 0,
231 })
232 }
233
234 pub fn abandon_snapshot_merge(&self) -> Result<()> {
235 self.conn().execute(
236 "DELETE FROM index_meta WHERE key = ?1",
237 params![MERGE_PROGRESS],
238 )?;
239 Ok(())
240 }
241}
242
243pub struct SnapshotMerge<'a> {
246 store: &'a SqliteStore,
247 progress: MergeProgress,
248 index: u64,
249 batch: Vec<SnapshotRow>,
250 batch_bytes: usize,
251}
252
253impl SnapshotMerge<'_> {
254 pub fn progress(&self) -> &MergeProgress {
255 &self.progress
256 }
257
258 pub fn push(&mut self, row: SnapshotRow) -> Result<()> {
259 let index = self.index;
260 self.index += 1;
261 if index < self.progress.rows_done {
262 return Ok(());
263 }
264 self.batch_bytes += approx_size(&row);
265 self.batch.push(row);
266 if self.batch.len() >= MERGE_CHUNK || self.batch_bytes >= MERGE_CHUNK_BYTES {
267 self.flush()?;
268 }
269 Ok(())
270 }
271
272 fn flush(&mut self) -> Result<()> {
273 if self.batch.is_empty() {
274 return Ok(());
275 }
276 let rows = std::mem::take(&mut self.batch);
277 self.batch_bytes = 0;
278 let store = self.store;
279 let mut progress = self.progress.clone();
280 let now = Utc::now().timestamp_micros();
281 store.with_txn(|tx| {
282 for row in rows {
283 let index = progress.rows_done;
284 merge_row(store, tx, &mut progress, index, row, now)?;
285 progress.rows_done += 1;
286 }
287 write_progress(tx, &progress)
288 })?;
289 self.progress = progress;
290 Ok(())
291 }
292
293 pub fn finish(mut self, cursors: &BTreeMap<String, u64>) -> Result<MergeOutcome> {
298 self.flush()?;
299 let progress = self.progress.clone();
300 let own = self.store.machine_id().to_string();
301 self.store.with_txn(|tx| {
302 raise_cursor(tx, &progress.exporter, progress.seq)?;
303 for (peer, seq) in cursors {
304 if *peer != own && *peer != progress.exporter {
305 raise_cursor(tx, peer, *seq)?;
306 }
307 }
308 tx.execute(
309 "INSERT INTO peer_in (peer_id, seq) VALUES (?1, ?2) \
310 ON CONFLICT(peer_id) DO UPDATE SET seq = MAX(peer_in.seq, excluded.seq)",
311 params![progress.exporter, i64::try_from(progress.seq)?],
312 )?;
313 tx.execute(
314 "DELETE FROM index_meta WHERE key = ?1",
315 params![MERGE_PROGRESS],
316 )?;
317 Ok(())
318 })?;
319 Ok(MergeOutcome {
320 rows: progress.rows_done,
321 changed: progress.changed,
322 settled: progress.settled,
323 })
324 }
325}
326
327fn read_progress(conn: &Connection) -> Result<Option<MergeProgress>> {
328 let value: Option<String> = conn
329 .query_row(
330 "SELECT value FROM index_meta WHERE key = ?1",
331 params![MERGE_PROGRESS],
332 |r| r.get(0),
333 )
334 .optional()?;
335 value
336 .map(|v| serde_json::from_str(&v).context("reading the snapshot merge's progress"))
337 .transpose()
338}
339
340fn write_progress(conn: &Connection, progress: &MergeProgress) -> Result<()> {
341 conn.execute(
342 "INSERT OR REPLACE INTO index_meta (key, value) VALUES (?1, ?2)",
343 params![MERGE_PROGRESS, serde_json::to_string(progress)?],
344 )?;
345 Ok(())
346}
347
348fn raise_cursor(tx: &Transaction<'_>, peer: &str, seq: u64) -> Result<()> {
349 tx.execute(
350 "INSERT INTO sync_state (peer_id, last_seq, last_sync_at) VALUES (?1, ?2, ?3) \
351 ON CONFLICT(peer_id) DO UPDATE SET last_seq = MAX(sync_state.last_seq, excluded.last_seq)",
352 params![peer, i64::try_from(seq)?, Utc::now().to_rfc3339()],
353 )?;
354 Ok(())
355}
356
357fn micros_to_time(micros: i64) -> DateTime<Utc> {
358 DateTime::from_timestamp_micros(micros).unwrap_or(DateTime::UNIX_EPOCH)
359}
360
361fn with_updated_by<T: Serialize>(value: &T, by: &str) -> Result<serde_json::Value> {
362 let mut json = serde_json::to_value(value)?;
363 if let Some(object) = json.as_object_mut() {
364 object.insert(UPDATED_BY.to_string(), by.into());
365 }
366 Ok(json)
367}
368
369fn encode_embedding(embedding: &[f32]) -> Option<String> {
370 (!embedding.is_empty()).then(|| B64.encode(embedding_to_blob(embedding)))
371}
372
373fn decode_embedding(encoded: Option<&str>) -> Result<Vec<f32>> {
374 match encoded {
375 Some(encoded) => Ok(blob_to_embedding(
376 &B64.decode(encoded)
377 .context("decoding a snapshot embedding")?,
378 )),
379 None => Ok(Vec::new()),
380 }
381}
382
383fn row_record(row: &SnapshotRow, origin: &str, n: u64) -> Result<Option<SyncRecord>> {
386 let record = |op, node_type, node_id: String, micros: i64, data| SyncRecord {
387 local_seq: 0,
388 origin_machine_id: origin.to_string(),
389 origin_seq: n,
390 op,
391 node_type,
392 node_id,
393 timestamp: micros_to_time(micros),
394 data,
395 };
396 Ok(Some(match row {
397 SnapshotRow::Conversation {
398 row,
399 updated_at,
400 updated_by,
401 } => record(
402 SyncOp::Update,
403 SyncNodeType::Conversation,
404 row.id.to_string(),
405 *updated_at,
406 Some(with_updated_by(row, updated_by)?),
407 ),
408 SnapshotRow::Entity {
409 row,
410 updated_at,
411 updated_by,
412 } => record(
413 SyncOp::Update,
414 SyncNodeType::Entity,
415 row.id.to_string(),
416 *updated_at,
417 Some(with_updated_by(row, updated_by)?),
418 ),
419 SnapshotRow::Memory { row, embedding } => {
420 let memory = Memory {
421 embedding: decode_embedding(embedding.as_deref())?,
422 ..row.clone()
423 };
424 record(
425 SyncOp::Update,
426 SyncNodeType::Memory,
427 memory.id.to_string(),
428 memory.updated_at.timestamp_micros(),
429 Some(serde_json::to_value(&memory)?),
430 )
431 }
432 SnapshotRow::Relation {
433 row,
434 updated_at,
435 updated_by,
436 } => record(
437 SyncOp::Create,
438 SyncNodeType::Relation,
439 authored_order::relation_key(row),
440 *updated_at,
441 Some(with_updated_by(row, updated_by)?),
442 ),
443 SnapshotRow::Distillation {
444 row,
445 updated_at,
446 updated_by,
447 } => record(
448 SyncOp::Create,
449 SyncNodeType::Distillation,
450 row.memory_id.to_string(),
451 *updated_at,
452 Some(with_updated_by(row, updated_by)?),
453 ),
454 SnapshotRow::TypeTombstone {
455 node_type,
456 node_id,
457 deleted_at,
458 origin: by,
459 } => match node_type.as_str() {
460 "conversation" | "entity" => record(
461 SyncOp::Delete,
462 if node_type == "entity" {
463 SyncNodeType::Entity
464 } else {
465 SyncNodeType::Conversation
466 },
467 node_id.clone(),
468 *deleted_at,
469 Some(serde_json::json!({ UPDATED_BY: by })),
470 ),
471 "relation" => {
472 let relation = relation_of_key(node_id)?;
473 record(
474 SyncOp::Delete,
475 SyncNodeType::Relation,
476 node_id.clone(),
477 *deleted_at,
478 Some(with_updated_by(&relation, by)?),
479 )
480 }
481 "distillation" => {
482 let dropped = distillation_of_key(node_id)?;
483 record(
484 SyncOp::Delete,
485 SyncNodeType::Distillation,
486 dropped.memory_id.to_string(),
487 *deleted_at,
488 Some(with_updated_by(&dropped, by)?),
489 )
490 }
491 _ => return Ok(None),
492 },
493 _ => return Ok(None),
494 }))
495}
496
497fn relation_of_key(key: &str) -> Result<Relation> {
498 let mut parts = key.splitn(3, ':');
499 let (Some(from), Some(to), Some(kind)) = (parts.next(), parts.next(), parts.next()) else {
500 bail!("relation tombstone key {key:?} is not from:to:type");
501 };
502 Ok(Relation {
503 from_id: parse_uuid(from)?,
504 to_id: parse_uuid(to)?,
505 relation_type: parse_relation_type(kind)?,
506 strength: 0.0,
507 context: None,
508 })
509}
510
511fn distillation_of_key(key: &str) -> Result<Distillation> {
512 let (memory, distilled) = key
513 .split_once(':')
514 .with_context(|| format!("distillation tombstone key {key:?} is not memory:distilled"))?;
515 Ok(Distillation {
516 memory_id: parse_uuid(memory)?,
517 distilled_id: (!distilled.is_empty())
518 .then(|| parse_uuid(distilled))
519 .transpose()?,
520 distilled_at: None,
521 model: None,
522 status: String::new(),
523 })
524}
525
526fn merge_row(
527 store: &SqliteStore,
528 tx: &Transaction<'_>,
529 progress: &mut MergeProgress,
530 index: u64,
531 row: SnapshotRow,
532 now: i64,
533) -> Result<()> {
534 let origin = format!("snap:{}", progress.snapshot_id);
535 let mut attempt = progress.clone();
536 tx.execute_batch("SAVEPOINT merge_row")?;
537 match apply_row(store, tx, &mut attempt, &origin, &row) {
538 Ok(()) => {
539 tx.execute_batch("RELEASE merge_row")?;
540 *progress = attempt;
541 Ok(())
542 }
543 Err(e) => {
544 tx.execute_batch("ROLLBACK TO merge_row; RELEASE merge_row")?;
545 if settled::is_outer(&e) {
546 return Err(e);
547 }
548 progress.settled += 1;
549 match settle_record(&row, &origin, progress.next_n) {
550 Some(record) => settled::settle_on_tx(tx, &origin, index, &record, &e, now),
551 None => {
552 tracing::warn!(
553 kind = row.section(),
554 error = format!("{e:#}"),
555 "settled a snapshot row that cannot be merged"
556 );
557 Ok(())
558 }
559 }
560 }
561 }
562}
563
564fn apply_row(
565 store: &SqliteStore,
566 tx: &Transaction<'_>,
567 progress: &mut MergeProgress,
568 origin: &str,
569 row: &SnapshotRow,
570) -> Result<()> {
571 match row {
572 SnapshotRow::MemoryTombstone {
573 id,
574 node_type,
575 deleted_at,
576 machine_id,
577 } => {
578 if merge_memory_tombstone(
579 store,
580 tx,
581 origin,
582 progress.next_n,
583 id,
584 node_type,
585 deleted_at,
586 machine_id.as_deref(),
587 )? {
588 progress.next_n += 1;
589 progress.changed += 1;
590 }
591 Ok(())
592 }
593 SnapshotRow::TypeTombstone {
594 node_type,
595 node_id,
596 deleted_at,
597 origin: by,
598 } if node_type == "distillation_slot" => merge_slot_floor(tx, node_id, *deleted_at, by),
599 SnapshotRow::Chunks {
600 memory_id,
601 content_hash,
602 chunks,
603 } => merge_chunks(tx, *memory_id, content_hash, chunks),
604 SnapshotRow::IngestKey {
605 file_path,
606 file_hash,
607 ingested_at,
608 memory_count,
609 source,
610 session_key,
611 content_hash,
612 legacy_ids,
613 } => {
614 tx.execute(
615 "INSERT OR IGNORE INTO ingest_log (file_path, file_hash, ingested_at, memory_count, \
616 source, session_key, content_hash, legacy_ids) VALUES (?1,?2,?3,?4,?5,?6,?7,?8)",
617 params![
618 file_path,
619 file_hash,
620 ingested_at,
621 memory_count,
622 source,
623 session_key,
624 content_hash,
625 legacy_ids
626 ],
627 )?;
628 Ok(())
629 }
630 SnapshotRow::LogTail { record } => merge_tail_record(store, tx, progress, record),
631 SnapshotRow::Dedup { origin, lo, hi } => sync_dedup::merge_range(tx, origin, *lo, *hi),
632 _ => {
633 let Some(record) = row_record(row, origin, progress.next_n)? else {
634 return Ok(());
635 };
636 if store.apply_logged_on_tx(tx, &record)?? {
637 progress.next_n += 1;
638 progress.changed += 1;
639 }
640 Ok(())
641 }
642 }
643}
644
645fn settle_record(row: &SnapshotRow, origin: &str, n: u64) -> Option<SyncRecord> {
646 match row {
647 SnapshotRow::LogTail { record } => Some(record.clone()),
648 SnapshotRow::MemoryTombstone {
649 id,
650 node_type,
651 deleted_at,
652 ..
653 } if node_type == "memory" => Some(memory_delete_record(
654 origin,
655 n,
656 id,
657 parse_dt(deleted_at).unwrap_or(DateTime::UNIX_EPOCH),
658 )),
659 _ => row_record(row, origin, n).ok().flatten(),
660 }
661}
662
663fn memory_delete_record(origin: &str, n: u64, id: &str, authored: DateTime<Utc>) -> SyncRecord {
664 SyncRecord {
665 local_seq: 0,
666 origin_machine_id: origin.to_string(),
667 origin_seq: n,
668 op: SyncOp::Delete,
669 node_type: SyncNodeType::Memory,
670 node_id: id.to_string(),
671 timestamp: authored,
672 data: None,
673 }
674}
675
676#[allow(clippy::too_many_arguments)]
679fn merge_memory_tombstone(
680 store: &SqliteStore,
681 tx: &Transaction<'_>,
682 origin: &str,
683 n: u64,
684 id: &str,
685 node_type: &str,
686 deleted_at: &str,
687 machine_id: Option<&str>,
688) -> Result<bool> {
689 let known: Option<i64> = tx
690 .query_row(
691 "SELECT 1 FROM tombstones WHERE node_id = ?1",
692 params![id],
693 |r| r.get(0),
694 )
695 .optional()?;
696 if known.is_some() {
697 return Ok(false);
698 }
699 tx.execute(
700 "INSERT INTO tombstones (node_id, node_type, deleted_at, machine_id) VALUES (?1,?2,?3,?4)",
701 params![id, node_type, deleted_at, machine_id],
702 )?;
703 if node_type != "memory" {
704 return Ok(false);
705 }
706 let record = memory_delete_record(origin, n, id, parse_dt(deleted_at)?);
707 store.apply_memory_delete(tx, &record)?;
708 Ok(true)
709}
710
711fn merge_slot_floor(
714 tx: &Transaction<'_>,
715 memory_id: &str,
716 deleted_at: i64,
717 by: &str,
718) -> Result<()> {
719 let floor = authored_order::Tombstone {
720 deleted_at,
721 origin: by.to_string(),
722 };
723 if let Ok(memory) = Uuid::parse_str(memory_id)
724 && let Some((_, row)) = authored_order::marker_stamp(tx, memory)?
725 && floor.beats(&row)
726 {
727 tx.execute(
728 "DELETE FROM distillation_log WHERE memory_id = ?1",
729 params![memory_id],
730 )?;
731 }
732 authored_order::record_tombstone(tx, "distillation_slot", memory_id, &floor)?;
733 Ok(())
734}
735
736fn merge_chunks(
739 tx: &Transaction<'_>,
740 memory_id: Uuid,
741 content_hash: &str,
742 chunks: &[SnapshotChunk],
743) -> Result<()> {
744 let id = memory_id.to_string();
745 let content: Option<String> = tx
746 .query_row(
747 "SELECT content FROM memories WHERE id = ?1 AND tier = 'hot' AND superseded_by IS NULL",
748 params![id],
749 |r| r.get(0),
750 )
751 .optional()?;
752 if content.is_none_or(|c| crate::chunk::content_hash(&c) != content_hash) {
753 return Ok(());
754 }
755 let embedded_here: i64 = tx.query_row(
756 "SELECT COUNT(*) FROM memory_chunks WHERE memory_id = ?1 AND content_hash = ?2 \
757 AND embedding IS NOT NULL",
758 params![id, content_hash],
759 |r| r.get(0),
760 )?;
761 if embedded_here as usize >= chunks.len() {
762 return Ok(());
763 }
764 let decoded = chunks
765 .iter()
766 .map(|c| {
767 Ok((
768 c.ord,
769 c.text.as_str(),
770 decode_embedding(c.embedding.as_deref())?,
771 ))
772 })
773 .collect::<Result<Vec<_>>>()?;
774 SqliteStore::replace_chunk_rows(
775 tx,
776 &id,
777 content_hash,
778 decoded
779 .iter()
780 .map(|(ord, text, e)| (*ord, *text, e.as_slice())),
781 )?;
782 SqliteStore::append_outbox(tx, memory_id, "upsert")
783}
784
785fn merge_tail_record(
789 store: &SqliteStore,
790 tx: &Transaction<'_>,
791 progress: &mut MergeProgress,
792 record: &SyncRecord,
793) -> Result<()> {
794 let origin = record.origin_machine_id.as_str();
795 let origin_seq = record.origin_seq as i64;
796 if sync_dedup::seen(tx, origin, origin_seq)? || origin == store.machine_id() {
797 return Ok(());
798 }
799 store.apply_on_tx(tx, record)?;
800 let data = record
801 .data
802 .as_ref()
803 .map(serde_json::to_string)
804 .transpose()?;
805 store.append_sync_log(
806 tx,
807 record.op,
808 record.node_type,
809 &record.node_id,
810 data.as_deref(),
811 Some((origin, origin_seq, record.timestamp)),
812 )?;
813 progress.changed += 1;
814 Ok(())
815}
816
817fn approx_size(row: &SnapshotRow) -> usize {
818 let text = |s: &Option<String>| s.as_ref().map_or(0, String::len);
819 match row {
820 SnapshotRow::Memory { row, embedding } => {
821 row.content.len() + text(&row.context) + text(embedding)
822 }
823 SnapshotRow::Chunks { chunks, .. } => chunks
824 .iter()
825 .map(|c| c.text.len() + text(&c.embedding))
826 .sum(),
827 SnapshotRow::LogTail { record } => record.data.as_ref().map_or(0, |d| d.to_string().len()),
828 _ => 0,
829 }
830}
831
832fn export_cursors(conn: &Connection, own: &str) -> Result<BTreeMap<String, u64>> {
835 let mut cursors = BTreeMap::new();
836 let mut add = |sql: &str| -> Result<()> {
837 let mut stmt = conn.prepare(sql)?;
838 let rows = stmt.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)?)))?;
839 for row in rows {
840 let (peer, seq) = row?;
841 if peer == own
842 || peer.starts_with("push:")
843 || crate::maintenance::is_stale_cursor(&peer)
844 {
845 continue;
846 }
847 let seq = u64::try_from(seq.max(0))?;
848 let entry = cursors.entry(peer).or_insert(0);
849 *entry = (*entry).max(seq);
850 }
851 Ok(())
852 };
853 add("SELECT peer_id, last_seq FROM sync_state")?;
854 add("SELECT peer_id, seq FROM peer_in")?;
855 Ok(cursors)
856}
857
858type Emit<'a> = dyn FnMut(SnapshotRow) -> Result<()> + 'a;
859
860fn export_tombstones(conn: &Connection, emit: &mut Emit<'_>) -> Result<()> {
861 let mut stmt =
862 conn.prepare("SELECT node_id, node_type, deleted_at, machine_id FROM tombstones")?;
863 let mut rows = stmt.query([])?;
864 while let Some(r) = rows.next()? {
865 emit(SnapshotRow::MemoryTombstone {
866 id: r.get(0)?,
867 node_type: r
868 .get::<_, Option<String>>(1)?
869 .unwrap_or_else(|| "memory".to_string()),
870 deleted_at: r.get(2)?,
871 machine_id: r.get(3)?,
872 })?;
873 }
874 let mut stmt =
875 conn.prepare("SELECT node_type, node_id, deleted_at, origin FROM type_tombstones")?;
876 let mut rows = stmt.query([])?;
877 while let Some(r) = rows.next()? {
878 emit(SnapshotRow::TypeTombstone {
879 node_type: r.get(0)?,
880 node_id: r.get(1)?,
881 deleted_at: r.get(2)?,
882 origin: r.get(3)?,
883 })?;
884 }
885 Ok(())
886}
887
888fn export_conversations(conn: &Connection, emit: &mut Emit<'_>) -> Result<()> {
889 let mut stmt = conn.prepare(
890 "SELECT id, source, machine_id, started_at, project_path, updated_at, updated_by \
891 FROM conversations",
892 )?;
893 let mut rows = stmt.query([])?;
894 while let Some(r) = rows.next()? {
895 emit(SnapshotRow::Conversation {
896 row: Conversation {
897 id: parse_uuid(&r.get::<_, String>(0)?)?,
898 source: r.get(1)?,
899 machine_id: r.get(2)?,
900 started_at: parse_dt(&r.get::<_, String>(3)?)?,
901 project_path: r.get(4)?,
902 },
903 updated_at: r.get(5)?,
904 updated_by: r.get(6)?,
905 })?;
906 }
907 Ok(())
908}
909
910fn export_entities(conn: &Connection, emit: &mut Emit<'_>) -> Result<()> {
911 let mut stmt = conn.prepare(
912 "SELECT id, name, entity_type, embedding, aliases, updated_at, updated_by FROM entities",
913 )?;
914 let mut rows = stmt.query([])?;
915 while let Some(r) = rows.next()? {
916 emit(SnapshotRow::Entity {
917 row: row_to_entity(r)?,
918 updated_at: r.get(5)?,
919 updated_by: r.get(6)?,
920 })?;
921 }
922 Ok(())
923}
924
925fn export_memories(store: &SqliteStore, conn: &Connection, emit: &mut Emit<'_>) -> Result<()> {
926 let mut stmt = conn.prepare(&format!("SELECT {MEMORY_COLS}, embedding FROM memories"))?;
927 let mut rows = stmt.query([])?;
928 while let Some(r) = rows.next()? {
929 let stored = row_to_memory(r, true)?;
930 let mut memory = store.sync_form(conn, &stored)?.into_owned();
931 let embedding = encode_embedding(&memory.embedding);
932 memory.embedding = Vec::new();
933 emit(SnapshotRow::Memory {
934 row: memory,
935 embedding,
936 })?;
937 }
938 Ok(())
939}
940
941fn export_relations(conn: &Connection, emit: &mut Emit<'_>) -> Result<()> {
942 let mut stmt = conn.prepare(
943 "SELECT from_id, to_id, relation_type, strength, context, updated_at, updated_by \
944 FROM relations",
945 )?;
946 let mut rows = stmt.query([])?;
947 while let Some(r) = rows.next()? {
948 emit(SnapshotRow::Relation {
949 row: Relation {
950 from_id: parse_uuid(&r.get::<_, String>(0)?)?,
951 to_id: parse_uuid(&r.get::<_, String>(1)?)?,
952 relation_type: parse_relation_type(&r.get::<_, String>(2)?)?,
953 strength: r.get::<_, Option<f64>>(3)?.unwrap_or(1.0) as f32,
954 context: r.get(4)?,
955 },
956 updated_at: r.get(5)?,
957 updated_by: r.get(6)?,
958 })?;
959 }
960 Ok(())
961}
962
963fn export_distillations(conn: &Connection, emit: &mut Emit<'_>) -> Result<()> {
964 let mut stmt = conn.prepare(
965 "SELECT memory_id, distilled_id, distilled_at, model, status, updated_at, updated_by \
966 FROM distillation_log",
967 )?;
968 let mut rows = stmt.query([])?;
969 while let Some(r) = rows.next()? {
970 emit(SnapshotRow::Distillation {
971 row: Distillation {
972 memory_id: parse_uuid(&r.get::<_, String>(0)?)?,
973 distilled_id: r
974 .get::<_, Option<String>>(1)?
975 .map(|s| parse_uuid(&s))
976 .transpose()?,
977 distilled_at: r
978 .get::<_, Option<String>>(2)?
979 .map(|s| parse_dt(&s))
980 .transpose()?,
981 model: r.get(3)?,
982 status: r.get(4)?,
983 },
984 updated_at: r.get(5)?,
985 updated_by: r.get(6)?,
986 })?;
987 }
988 Ok(())
989}
990
991fn export_chunks(conn: &Connection, emit: &mut Emit<'_>) -> Result<()> {
992 let mut stmt = conn.prepare(
993 "SELECT memory_id, ord, text, embedding, content_hash FROM memory_chunks \
994 ORDER BY memory_id, ord",
995 )?;
996 let mut rows = stmt.query([])?;
997 let mut current: Option<(String, String, Vec<SnapshotChunk>)> = None;
998 while let Some(r) = rows.next()? {
999 let memory_id: String = r.get(0)?;
1000 let chunk = SnapshotChunk {
1001 ord: r.get::<_, i64>(1)? as usize,
1002 text: r.get(2)?,
1003 embedding: r.get::<_, Option<Vec<u8>>>(3)?.map(|blob| B64.encode(blob)),
1004 };
1005 let hash: String = r.get(4)?;
1006 match &mut current {
1007 Some((id, _, chunks)) if *id == memory_id => chunks.push(chunk),
1008 _ => {
1009 if let Some(done) = current.take() {
1010 emit_chunks(done, emit)?;
1011 }
1012 current = Some((memory_id, hash, vec![chunk]));
1013 }
1014 }
1015 }
1016 if let Some(done) = current {
1017 emit_chunks(done, emit)?;
1018 }
1019 Ok(())
1020}
1021
1022fn emit_chunks(
1023 (memory_id, content_hash, chunks): (String, String, Vec<SnapshotChunk>),
1024 emit: &mut Emit<'_>,
1025) -> Result<()> {
1026 emit(SnapshotRow::Chunks {
1027 memory_id: parse_uuid(&memory_id)?,
1028 content_hash,
1029 chunks,
1030 })
1031}
1032
1033fn export_ingest_keys(conn: &Connection, emit: &mut Emit<'_>) -> Result<()> {
1034 let mut stmt = conn.prepare(
1035 "SELECT file_path, file_hash, ingested_at, memory_count, source, session_key, \
1036 content_hash, legacy_ids FROM ingest_log",
1037 )?;
1038 let mut rows = stmt.query([])?;
1039 while let Some(r) = rows.next()? {
1040 emit(SnapshotRow::IngestKey {
1041 file_path: r.get(0)?,
1042 file_hash: r.get(1)?,
1043 ingested_at: r.get(2)?,
1044 memory_count: r.get(3)?,
1045 source: r.get(4)?,
1046 session_key: r.get(5)?,
1047 content_hash: r.get(6)?,
1048 legacy_ids: r.get(7)?,
1049 })?;
1050 }
1051 Ok(())
1052}
1053
1054fn export_tail(conn: &Connection, after: u64, seq: u64, emit: &mut Emit<'_>) -> Result<()> {
1057 let mut stmt = conn.prepare(
1058 "SELECT id, local_seq, origin_machine_id, origin_seq, op, node_type, node_id, \
1059 timestamp, data FROM sync_log WHERE local_seq > ?1 AND local_seq <= ?2 \
1060 ORDER BY local_seq",
1061 )?;
1062 let mut rows = stmt.query(params![i64::try_from(after)?, i64::try_from(seq)?])?;
1063 while let Some(r) = rows.next()? {
1064 let entry = super::sync_apply::row_to_sync_entry(r)?;
1065 emit(SnapshotRow::LogTail {
1066 record: record_of_entry(entry)?,
1067 })?;
1068 }
1069 Ok(())
1070}
1071
1072fn record_of_entry(entry: SyncEntry) -> Result<SyncRecord> {
1073 Ok(SyncRecord {
1074 local_seq: u64::try_from(entry.local_seq)?,
1075 origin_machine_id: entry.origin_machine_id,
1076 origin_seq: u64::try_from(entry.origin_seq)?,
1077 op: entry.op,
1078 node_type: entry.node_type,
1079 node_id: entry.node_id,
1080 timestamp: entry.timestamp,
1081 data: entry.data.map(|d| serde_json::from_str(&d)).transpose()?,
1082 })
1083}
1084
1085fn export_dedup(conn: &Connection, emit: &mut Emit<'_>) -> Result<()> {
1089 let mut stmt = conn.prepare("SELECT origin, lo, hi FROM sync_dedup ORDER BY origin, lo")?;
1090 let mut rows = stmt.query([])?;
1091 while let Some(r) = rows.next()? {
1092 emit(SnapshotRow::Dedup {
1093 origin: r.get(0)?,
1094 lo: r.get(1)?,
1095 hi: r.get(2)?,
1096 })?;
1097 }
1098 let mut stmt = conn.prepare(
1099 "SELECT origin_machine_id, origin_seq FROM sync_log \
1100 ORDER BY origin_machine_id, origin_seq",
1101 )?;
1102 let mut rows = stmt.query([])?;
1103 let mut run: Option<(String, i64, i64)> = None;
1104 while let Some(r) = rows.next()? {
1105 let origin: String = r.get(0)?;
1106 let seq: i64 = r.get(1)?;
1107 if !clock::is_new_layout(&origin, seq) {
1108 continue;
1109 }
1110 match &mut run {
1111 Some((o, _, hi)) if *o == origin && seq <= hi.saturating_add(1) => {
1112 *hi = (*hi).max(seq);
1113 }
1114 _ => {
1115 if let Some((origin, lo, hi)) = run.take() {
1116 emit(SnapshotRow::Dedup { origin, lo, hi })?;
1117 }
1118 run = Some((origin, seq, seq));
1119 }
1120 }
1121 }
1122 if let Some((origin, lo, hi)) = run {
1123 emit(SnapshotRow::Dedup { origin, lo, hi })?;
1124 }
1125 Ok(())
1126}
1127
1128#[cfg(test)]
1129mod tests {
1130 use super::*;
1131
1132 #[test]
1133 fn large_rows_flush_before_a_chunk_of_rows_fills() {
1134 let store =
1135 SqliteStore::in_memory("00000000-0000-4000-8000-000000000000".to_string()).unwrap();
1136 let mut merge = store
1137 .begin_snapshot_merge("s1", "00000000-0000-4000-8000-00000000000a", 0)
1138 .unwrap();
1139 let large = || SnapshotRow::Chunks {
1140 memory_id: Uuid::new_v4(),
1141 content_hash: String::new(),
1142 chunks: vec![SnapshotChunk {
1143 ord: 0,
1144 text: "x".repeat(MERGE_CHUNK_BYTES / 2 + 1),
1145 embedding: None,
1146 }],
1147 };
1148
1149 merge.push(large()).unwrap();
1150 assert_eq!(merge.progress().rows_done, 0);
1151 merge.push(large()).unwrap();
1152 assert_eq!(merge.progress().rows_done, 2, "the byte bound flushed");
1153 }
1154}