Skip to main content

second_brain_core/sqlite_store/
snapshot.rs

1use std::collections::BTreeMap;
2
3use anyhow::{Context, Result, bail};
4use base64::Engine;
5use base64::engine::general_purpose::STANDARD as B64;
6use chrono::{DateTime, Utc};
7use rusqlite::{Connection, OptionalExtension, Transaction, params};
8use serde::{Deserialize, Serialize};
9use uuid::Uuid;
10
11use super::authored_order::{self, UPDATED_BY};
12use super::{
13    MEMORY_COLS, SqliteStore, blob_to_embedding, clock, embedding_to_blob, floor, parse_dt,
14    parse_relation_type, parse_uuid, row_to_entity, row_to_memory, settled, sync_dedup,
15};
16use crate::schema::{
17    Conversation, Distillation, Entity, Memory, Relation, SyncEntry, SyncNodeType, SyncOp,
18    SyncRecord,
19};
20
21/// Rows applied per write transaction, so a merge runs between other writes.
22pub const MERGE_CHUNK: usize = 2_000;
23
24/// Approximate bytes of buffered rows that flush a chunk before
25/// `MERGE_CHUNK` rows, so a chunk of large rows stays bounded in memory.
26pub const MERGE_CHUNK_BYTES: usize = 32 * 1024 * 1024;
27
28const MERGE_PROGRESS: &str = "snapshot_merge";
29
30/// One row of a snapshot, in the order the exporter writes them: tombstones
31/// first, then live rows parents before children, then the carried log tail,
32/// and the dedup ranges last, so they are merged only once every row is.
33#[derive(Debug, Clone, Serialize, Deserialize)]
34#[serde(tag = "t", rename_all = "snake_case")]
35pub enum SnapshotRow {
36    MemoryTombstone {
37        id: String,
38        node_type: String,
39        deleted_at: String,
40        machine_id: Option<String>,
41    },
42    TypeTombstone {
43        node_type: String,
44        node_id: String,
45        deleted_at: i64,
46        origin: String,
47    },
48    Conversation {
49        row: Conversation,
50        updated_at: i64,
51        updated_by: String,
52    },
53    Entity {
54        row: Entity,
55        updated_at: i64,
56        updated_by: String,
57    },
58    /// The memory without its embedding, which travels as base64 f32s.
59    Memory {
60        row: Memory,
61        embedding: Option<String>,
62    },
63    Relation {
64        row: Relation,
65        updated_at: i64,
66        updated_by: String,
67    },
68    Distillation {
69        row: Distillation,
70        updated_at: i64,
71        updated_by: String,
72    },
73    Chunks {
74        memory_id: Uuid,
75        content_hash: String,
76        chunks: Vec<SnapshotChunk>,
77    },
78    IngestKey {
79        file_path: String,
80        file_hash: String,
81        ingested_at: Option<String>,
82        memory_count: Option<i64>,
83        source: Option<String>,
84        session_key: Option<String>,
85        content_hash: Option<String>,
86        legacy_ids: i64,
87    },
88    LogTail {
89        record: SyncRecord,
90    },
91    Dedup {
92        origin: String,
93        lo: i64,
94        hi: i64,
95    },
96}
97
98#[derive(Debug, Clone, Serialize, Deserialize)]
99pub struct SnapshotChunk {
100    pub ord: usize,
101    pub text: String,
102    pub embedding: Option<String>,
103}
104
105impl SnapshotRow {
106    pub fn section(&self) -> &'static str {
107        match self {
108            SnapshotRow::MemoryTombstone { .. } => "tombstones",
109            SnapshotRow::TypeTombstone { .. } => "type_tombstones",
110            SnapshotRow::Conversation { .. } => "conversations",
111            SnapshotRow::Entity { .. } => "entities",
112            SnapshotRow::Memory { .. } => "memories",
113            SnapshotRow::Relation { .. } => "relations",
114            SnapshotRow::Distillation { .. } => "distillation_log",
115            SnapshotRow::Chunks { .. } => "memory_chunks",
116            SnapshotRow::IngestKey { .. } => "ingest_log",
117            SnapshotRow::LogTail { .. } => "log_tail",
118            SnapshotRow::Dedup { .. } => "sync_dedup",
119        }
120    }
121}
122
123#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
124pub struct SnapshotOptions {
125    pub with_chunks: bool,
126}
127
128/// What the exporter states about the rows it wrote, all read in the same
129/// transaction as the rows.
130#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
131pub struct SnapshotState {
132    /// `S`: every record the exporter logged at or below it is in the rows.
133    pub seq: u64,
134    /// For each peer, the highest seq of its log the exporter has applied.
135    pub cursors: BTreeMap<String, u64>,
136    pub counts: BTreeMap<String, u64>,
137}
138
139#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
140pub struct MergeProgress {
141    pub snapshot_id: String,
142    pub exporter: String,
143    pub seq: u64,
144    pub rows_done: u64,
145    /// The next `n` of `snap:<id>:<n>`: relays are numbered densely, so a
146    /// retried merge reuses the same keys and they form one dedup range.
147    pub next_n: u64,
148    pub changed: u64,
149    pub settled: u64,
150}
151
152#[derive(Debug, Clone, Default, PartialEq, Eq)]
153pub struct MergeOutcome {
154    pub rows: u64,
155    pub changed: u64,
156    pub settled: u64,
157}
158
159impl SqliteStore {
160    /// Streams every row a peer needs to converge with this store, from one
161    /// read transaction, into `sink`. A caller that pins the reader does so
162    /// at `log_head` before calling, which is at or below the `seq` read
163    /// here, so no row above the pin is truncated meanwhile.
164    pub fn export_snapshot(
165        &self,
166        options: SnapshotOptions,
167        sink: &mut dyn FnMut(SnapshotRow) -> Result<()>,
168    ) -> Result<SnapshotState> {
169        let conn = self.reader();
170        let tx = conn.unchecked_transaction()?;
171        let seq = u64::try_from(clock::state(&tx)?.last.max(0))?;
172        let mut state = SnapshotState {
173            seq,
174            cursors: export_cursors(&tx, self.machine_id())?,
175            counts: BTreeMap::new(),
176        };
177        let mut emit = |row: SnapshotRow| -> Result<()> {
178            *state.counts.entry(row.section().to_string()).or_insert(0) += 1;
179            sink(row)
180        };
181        export_tombstones(&tx, &mut emit)?;
182        export_conversations(&tx, &mut emit)?;
183        export_entities(&tx, &mut emit)?;
184        export_memories(self, &tx, &mut emit)?;
185        export_relations(&tx, &mut emit)?;
186        export_distillations(&tx, &mut emit)?;
187        if options.with_chunks {
188            export_chunks(&tx, &mut emit)?;
189        }
190        export_ingest_keys(&tx, &mut emit)?;
191        let tail_after = floor::max_live_ack(&tx, self.machine_id())?.unwrap_or(0);
192        export_tail(&tx, tail_after, seq, &mut emit)?;
193        export_dedup(&tx, &mut emit)?;
194        tx.commit()?;
195        Ok(state)
196    }
197
198    pub fn snapshot_merge_progress(&self) -> Result<Option<MergeProgress>> {
199        read_progress(&self.reader())
200    }
201
202    /// Starts a merge of snapshot `snapshot_id`, or resumes the one already
203    /// in progress for it. A merge of another snapshot is abandoned: rows
204    /// it applied stay, since merging never removes anything.
205    pub fn begin_snapshot_merge(
206        &self,
207        snapshot_id: &str,
208        exporter: &str,
209        seq: u64,
210    ) -> Result<SnapshotMerge<'_>> {
211        let stored = read_progress(&self.reader())?;
212        let progress = match stored {
213            Some(progress) if progress.snapshot_id == snapshot_id => progress,
214            _ => {
215                let progress = MergeProgress {
216                    snapshot_id: snapshot_id.to_string(),
217                    exporter: exporter.to_string(),
218                    seq,
219                    ..MergeProgress::default()
220                };
221                write_progress(&self.conn(), &progress)?;
222                progress
223            }
224        };
225        Ok(SnapshotMerge {
226            store: self,
227            progress,
228            index: 0,
229            batch: Vec::with_capacity(MERGE_CHUNK),
230            batch_bytes: 0,
231        })
232    }
233
234    pub fn abandon_snapshot_merge(&self) -> Result<()> {
235        self.conn().execute(
236            "DELETE FROM index_meta WHERE key = ?1",
237            params![MERGE_PROGRESS],
238        )?;
239        Ok(())
240    }
241}
242
243/// A merge in progress. Rows go in through `push` in the snapshot's order;
244/// those a previous attempt already applied are passed over.
245pub struct SnapshotMerge<'a> {
246    store: &'a SqliteStore,
247    progress: MergeProgress,
248    index: u64,
249    batch: Vec<SnapshotRow>,
250    batch_bytes: usize,
251}
252
253impl SnapshotMerge<'_> {
254    pub fn progress(&self) -> &MergeProgress {
255        &self.progress
256    }
257
258    pub fn push(&mut self, row: SnapshotRow) -> Result<()> {
259        let index = self.index;
260        self.index += 1;
261        if index < self.progress.rows_done {
262            return Ok(());
263        }
264        self.batch_bytes += approx_size(&row);
265        self.batch.push(row);
266        if self.batch.len() >= MERGE_CHUNK || self.batch_bytes >= MERGE_CHUNK_BYTES {
267            self.flush()?;
268        }
269        Ok(())
270    }
271
272    fn flush(&mut self) -> Result<()> {
273        if self.batch.is_empty() {
274            return Ok(());
275        }
276        let rows = std::mem::take(&mut self.batch);
277        self.batch_bytes = 0;
278        let store = self.store;
279        let mut progress = self.progress.clone();
280        let now = Utc::now().timestamp_micros();
281        store.with_txn(|tx| {
282            for row in rows {
283                let index = progress.rows_done;
284                merge_row(store, tx, &mut progress, index, row, now)?;
285                progress.rows_done += 1;
286            }
287            write_progress(tx, &progress)
288        })?;
289        self.progress = progress;
290        Ok(())
291    }
292
293    /// Applies what is left, then raises cursors: the exporter's to `seq`,
294    /// every other one to the exporter's, and the exporter's `peer_in` to
295    /// `seq`. Cursors only rise, so a merge never makes this store pull or
296    /// accept anything twice and never forgets what it held.
297    pub fn finish(mut self, cursors: &BTreeMap<String, u64>) -> Result<MergeOutcome> {
298        self.flush()?;
299        let progress = self.progress.clone();
300        let own = self.store.machine_id().to_string();
301        self.store.with_txn(|tx| {
302            raise_cursor(tx, &progress.exporter, progress.seq)?;
303            for (peer, seq) in cursors {
304                if *peer != own && *peer != progress.exporter {
305                    raise_cursor(tx, peer, *seq)?;
306                }
307            }
308            tx.execute(
309                "INSERT INTO peer_in (peer_id, seq) VALUES (?1, ?2) \
310                 ON CONFLICT(peer_id) DO UPDATE SET seq = MAX(peer_in.seq, excluded.seq)",
311                params![progress.exporter, i64::try_from(progress.seq)?],
312            )?;
313            tx.execute(
314                "DELETE FROM index_meta WHERE key = ?1",
315                params![MERGE_PROGRESS],
316            )?;
317            Ok(())
318        })?;
319        Ok(MergeOutcome {
320            rows: progress.rows_done,
321            changed: progress.changed,
322            settled: progress.settled,
323        })
324    }
325}
326
327fn read_progress(conn: &Connection) -> Result<Option<MergeProgress>> {
328    let value: Option<String> = conn
329        .query_row(
330            "SELECT value FROM index_meta WHERE key = ?1",
331            params![MERGE_PROGRESS],
332            |r| r.get(0),
333        )
334        .optional()?;
335    value
336        .map(|v| serde_json::from_str(&v).context("reading the snapshot merge's progress"))
337        .transpose()
338}
339
340fn write_progress(conn: &Connection, progress: &MergeProgress) -> Result<()> {
341    conn.execute(
342        "INSERT OR REPLACE INTO index_meta (key, value) VALUES (?1, ?2)",
343        params![MERGE_PROGRESS, serde_json::to_string(progress)?],
344    )?;
345    Ok(())
346}
347
348fn raise_cursor(tx: &Transaction<'_>, peer: &str, seq: u64) -> Result<()> {
349    tx.execute(
350        "INSERT INTO sync_state (peer_id, last_seq, last_sync_at) VALUES (?1, ?2, ?3) \
351         ON CONFLICT(peer_id) DO UPDATE SET last_seq = MAX(sync_state.last_seq, excluded.last_seq)",
352        params![peer, i64::try_from(seq)?, Utc::now().to_rfc3339()],
353    )?;
354    Ok(())
355}
356
357fn micros_to_time(micros: i64) -> DateTime<Utc> {
358    DateTime::from_timestamp_micros(micros).unwrap_or(DateTime::UNIX_EPOCH)
359}
360
361fn with_updated_by<T: Serialize>(value: &T, by: &str) -> Result<serde_json::Value> {
362    let mut json = serde_json::to_value(value)?;
363    if let Some(object) = json.as_object_mut() {
364        object.insert(UPDATED_BY.to_string(), by.into());
365    }
366    Ok(json)
367}
368
369fn encode_embedding(embedding: &[f32]) -> Option<String> {
370    (!embedding.is_empty()).then(|| B64.encode(embedding_to_blob(embedding)))
371}
372
373fn decode_embedding(encoded: Option<&str>) -> Result<Vec<f32>> {
374    match encoded {
375        Some(encoded) => Ok(blob_to_embedding(
376            &B64.decode(encoded)
377                .context("decoding a snapshot embedding")?,
378        )),
379        None => Ok(Vec::new()),
380    }
381}
382
383/// The record a snapshot row stands for, keyed `snap:<id>:<n>`, so it goes
384/// through the same authored-order rules as any synced record.
385fn row_record(row: &SnapshotRow, origin: &str, n: u64) -> Result<Option<SyncRecord>> {
386    let record = |op, node_type, node_id: String, micros: i64, data| SyncRecord {
387        local_seq: 0,
388        origin_machine_id: origin.to_string(),
389        origin_seq: n,
390        op,
391        node_type,
392        node_id,
393        timestamp: micros_to_time(micros),
394        data,
395    };
396    Ok(Some(match row {
397        SnapshotRow::Conversation {
398            row,
399            updated_at,
400            updated_by,
401        } => record(
402            SyncOp::Update,
403            SyncNodeType::Conversation,
404            row.id.to_string(),
405            *updated_at,
406            Some(with_updated_by(row, updated_by)?),
407        ),
408        SnapshotRow::Entity {
409            row,
410            updated_at,
411            updated_by,
412        } => record(
413            SyncOp::Update,
414            SyncNodeType::Entity,
415            row.id.to_string(),
416            *updated_at,
417            Some(with_updated_by(row, updated_by)?),
418        ),
419        SnapshotRow::Memory { row, embedding } => {
420            let memory = Memory {
421                embedding: decode_embedding(embedding.as_deref())?,
422                ..row.clone()
423            };
424            record(
425                SyncOp::Update,
426                SyncNodeType::Memory,
427                memory.id.to_string(),
428                memory.updated_at.timestamp_micros(),
429                Some(serde_json::to_value(&memory)?),
430            )
431        }
432        SnapshotRow::Relation {
433            row,
434            updated_at,
435            updated_by,
436        } => record(
437            SyncOp::Create,
438            SyncNodeType::Relation,
439            authored_order::relation_key(row),
440            *updated_at,
441            Some(with_updated_by(row, updated_by)?),
442        ),
443        SnapshotRow::Distillation {
444            row,
445            updated_at,
446            updated_by,
447        } => record(
448            SyncOp::Create,
449            SyncNodeType::Distillation,
450            row.memory_id.to_string(),
451            *updated_at,
452            Some(with_updated_by(row, updated_by)?),
453        ),
454        SnapshotRow::TypeTombstone {
455            node_type,
456            node_id,
457            deleted_at,
458            origin: by,
459        } => match node_type.as_str() {
460            "conversation" | "entity" => record(
461                SyncOp::Delete,
462                if node_type == "entity" {
463                    SyncNodeType::Entity
464                } else {
465                    SyncNodeType::Conversation
466                },
467                node_id.clone(),
468                *deleted_at,
469                Some(serde_json::json!({ UPDATED_BY: by })),
470            ),
471            "relation" => {
472                let relation = relation_of_key(node_id)?;
473                record(
474                    SyncOp::Delete,
475                    SyncNodeType::Relation,
476                    node_id.clone(),
477                    *deleted_at,
478                    Some(with_updated_by(&relation, by)?),
479                )
480            }
481            "distillation" => {
482                let dropped = distillation_of_key(node_id)?;
483                record(
484                    SyncOp::Delete,
485                    SyncNodeType::Distillation,
486                    dropped.memory_id.to_string(),
487                    *deleted_at,
488                    Some(with_updated_by(&dropped, by)?),
489                )
490            }
491            _ => return Ok(None),
492        },
493        _ => return Ok(None),
494    }))
495}
496
497fn relation_of_key(key: &str) -> Result<Relation> {
498    let mut parts = key.splitn(3, ':');
499    let (Some(from), Some(to), Some(kind)) = (parts.next(), parts.next(), parts.next()) else {
500        bail!("relation tombstone key {key:?} is not from:to:type");
501    };
502    Ok(Relation {
503        from_id: parse_uuid(from)?,
504        to_id: parse_uuid(to)?,
505        relation_type: parse_relation_type(kind)?,
506        strength: 0.0,
507        context: None,
508    })
509}
510
511fn distillation_of_key(key: &str) -> Result<Distillation> {
512    let (memory, distilled) = key
513        .split_once(':')
514        .with_context(|| format!("distillation tombstone key {key:?} is not memory:distilled"))?;
515    Ok(Distillation {
516        memory_id: parse_uuid(memory)?,
517        distilled_id: (!distilled.is_empty())
518            .then(|| parse_uuid(distilled))
519            .transpose()?,
520        distilled_at: None,
521        model: None,
522        status: String::new(),
523    })
524}
525
526fn merge_row(
527    store: &SqliteStore,
528    tx: &Transaction<'_>,
529    progress: &mut MergeProgress,
530    index: u64,
531    row: SnapshotRow,
532    now: i64,
533) -> Result<()> {
534    let origin = format!("snap:{}", progress.snapshot_id);
535    let mut attempt = progress.clone();
536    tx.execute_batch("SAVEPOINT merge_row")?;
537    match apply_row(store, tx, &mut attempt, &origin, &row) {
538        Ok(()) => {
539            tx.execute_batch("RELEASE merge_row")?;
540            *progress = attempt;
541            Ok(())
542        }
543        Err(e) => {
544            tx.execute_batch("ROLLBACK TO merge_row; RELEASE merge_row")?;
545            if settled::is_outer(&e) {
546                return Err(e);
547            }
548            progress.settled += 1;
549            match settle_record(&row, &origin, progress.next_n) {
550                Some(record) => settled::settle_on_tx(tx, &origin, index, &record, &e, now),
551                None => {
552                    tracing::warn!(
553                        kind = row.section(),
554                        error = format!("{e:#}"),
555                        "settled a snapshot row that cannot be merged"
556                    );
557                    Ok(())
558                }
559            }
560        }
561    }
562}
563
564fn apply_row(
565    store: &SqliteStore,
566    tx: &Transaction<'_>,
567    progress: &mut MergeProgress,
568    origin: &str,
569    row: &SnapshotRow,
570) -> Result<()> {
571    match row {
572        SnapshotRow::MemoryTombstone {
573            id,
574            node_type,
575            deleted_at,
576            machine_id,
577        } => {
578            if merge_memory_tombstone(
579                store,
580                tx,
581                origin,
582                progress.next_n,
583                id,
584                node_type,
585                deleted_at,
586                machine_id.as_deref(),
587            )? {
588                progress.next_n += 1;
589                progress.changed += 1;
590            }
591            Ok(())
592        }
593        SnapshotRow::TypeTombstone {
594            node_type,
595            node_id,
596            deleted_at,
597            origin: by,
598        } if node_type == "distillation_slot" => merge_slot_floor(tx, node_id, *deleted_at, by),
599        SnapshotRow::Chunks {
600            memory_id,
601            content_hash,
602            chunks,
603        } => merge_chunks(tx, *memory_id, content_hash, chunks),
604        SnapshotRow::IngestKey {
605            file_path,
606            file_hash,
607            ingested_at,
608            memory_count,
609            source,
610            session_key,
611            content_hash,
612            legacy_ids,
613        } => {
614            tx.execute(
615                "INSERT OR IGNORE INTO ingest_log (file_path, file_hash, ingested_at, memory_count, \
616                 source, session_key, content_hash, legacy_ids) VALUES (?1,?2,?3,?4,?5,?6,?7,?8)",
617                params![
618                    file_path,
619                    file_hash,
620                    ingested_at,
621                    memory_count,
622                    source,
623                    session_key,
624                    content_hash,
625                    legacy_ids
626                ],
627            )?;
628            Ok(())
629        }
630        SnapshotRow::LogTail { record } => merge_tail_record(store, tx, progress, record),
631        SnapshotRow::Dedup { origin, lo, hi } => sync_dedup::merge_range(tx, origin, *lo, *hi),
632        _ => {
633            let Some(record) = row_record(row, origin, progress.next_n)? else {
634                return Ok(());
635            };
636            if store.apply_logged_on_tx(tx, &record)?? {
637                progress.next_n += 1;
638                progress.changed += 1;
639            }
640            Ok(())
641        }
642    }
643}
644
645fn settle_record(row: &SnapshotRow, origin: &str, n: u64) -> Option<SyncRecord> {
646    match row {
647        SnapshotRow::LogTail { record } => Some(record.clone()),
648        SnapshotRow::MemoryTombstone {
649            id,
650            node_type,
651            deleted_at,
652            ..
653        } if node_type == "memory" => Some(memory_delete_record(
654            origin,
655            n,
656            id,
657            parse_dt(deleted_at).unwrap_or(DateTime::UNIX_EPOCH),
658        )),
659        _ => row_record(row, origin, n).ok().flatten(),
660    }
661}
662
663fn memory_delete_record(origin: &str, n: u64, id: &str, authored: DateTime<Utc>) -> SyncRecord {
664    SyncRecord {
665        local_seq: 0,
666        origin_machine_id: origin.to_string(),
667        origin_seq: n,
668        op: SyncOp::Delete,
669        node_type: SyncNodeType::Memory,
670        node_id: id.to_string(),
671        timestamp: authored,
672        data: None,
673    }
674}
675
676/// A memory tombstone keeps the machine that deleted the memory, which a
677/// relayed delete would replace with the relay's origin.
678#[allow(clippy::too_many_arguments)]
679fn merge_memory_tombstone(
680    store: &SqliteStore,
681    tx: &Transaction<'_>,
682    origin: &str,
683    n: u64,
684    id: &str,
685    node_type: &str,
686    deleted_at: &str,
687    machine_id: Option<&str>,
688) -> Result<bool> {
689    let known: Option<i64> = tx
690        .query_row(
691            "SELECT 1 FROM tombstones WHERE node_id = ?1",
692            params![id],
693            |r| r.get(0),
694        )
695        .optional()?;
696    if known.is_some() {
697        return Ok(false);
698    }
699    tx.execute(
700        "INSERT INTO tombstones (node_id, node_type, deleted_at, machine_id) VALUES (?1,?2,?3,?4)",
701        params![id, node_type, deleted_at, machine_id],
702    )?;
703    if node_type != "memory" {
704        return Ok(false);
705    }
706    let record = memory_delete_record(origin, n, id, parse_dt(deleted_at)?);
707    store.apply_memory_delete(tx, &record)?;
708    Ok(true)
709}
710
711/// A slot floor is the stamp of a create that emptied the slot; a marker
712/// older than it would not have survived on the exporter.
713fn merge_slot_floor(
714    tx: &Transaction<'_>,
715    memory_id: &str,
716    deleted_at: i64,
717    by: &str,
718) -> Result<()> {
719    let floor = authored_order::Tombstone {
720        deleted_at,
721        origin: by.to_string(),
722    };
723    if let Ok(memory) = Uuid::parse_str(memory_id)
724        && let Some((_, row)) = authored_order::marker_stamp(tx, memory)?
725        && floor.beats(&row)
726    {
727        tx.execute(
728            "DELETE FROM distillation_log WHERE memory_id = ?1",
729            params![memory_id],
730        )?;
731    }
732    authored_order::record_tombstone(tx, "distillation_slot", memory_id, &floor)?;
733    Ok(())
734}
735
736/// Chunks are derived locally and never logged; a carried set is kept only
737/// beside the live memory it was split from, when no embedded set is here.
738fn merge_chunks(
739    tx: &Transaction<'_>,
740    memory_id: Uuid,
741    content_hash: &str,
742    chunks: &[SnapshotChunk],
743) -> Result<()> {
744    let id = memory_id.to_string();
745    let content: Option<String> = tx
746        .query_row(
747            "SELECT content FROM memories WHERE id = ?1 AND tier = 'hot' AND superseded_by IS NULL",
748            params![id],
749            |r| r.get(0),
750        )
751        .optional()?;
752    if content.is_none_or(|c| crate::chunk::content_hash(&c) != content_hash) {
753        return Ok(());
754    }
755    let embedded_here: i64 = tx.query_row(
756        "SELECT COUNT(*) FROM memory_chunks WHERE memory_id = ?1 AND content_hash = ?2 \
757         AND embedding IS NOT NULL",
758        params![id, content_hash],
759        |r| r.get(0),
760    )?;
761    if embedded_here as usize >= chunks.len() {
762        return Ok(());
763    }
764    let decoded = chunks
765        .iter()
766        .map(|c| {
767            Ok((
768                c.ord,
769                c.text.as_str(),
770                decode_embedding(c.embedding.as_deref())?,
771            ))
772        })
773        .collect::<Result<Vec<_>>>()?;
774    SqliteStore::replace_chunk_rows(
775        tx,
776        &id,
777        content_hash,
778        decoded
779            .iter()
780            .map(|(ord, text, e)| (*ord, *text, e.as_slice())),
781    )?;
782    SqliteStore::append_outbox(tx, memory_id, "upsert")
783}
784
785/// The exporter's unacknowledged tail keeps its original keys and is logged
786/// here whether or not it changed this store, so it reaches this store's
787/// pullers; a key seen before is already on its way through them.
788fn merge_tail_record(
789    store: &SqliteStore,
790    tx: &Transaction<'_>,
791    progress: &mut MergeProgress,
792    record: &SyncRecord,
793) -> Result<()> {
794    let origin = record.origin_machine_id.as_str();
795    let origin_seq = record.origin_seq as i64;
796    if sync_dedup::seen(tx, origin, origin_seq)? || origin == store.machine_id() {
797        return Ok(());
798    }
799    store.apply_on_tx(tx, record)?;
800    let data = record
801        .data
802        .as_ref()
803        .map(serde_json::to_string)
804        .transpose()?;
805    store.append_sync_log(
806        tx,
807        record.op,
808        record.node_type,
809        &record.node_id,
810        data.as_deref(),
811        Some((origin, origin_seq, record.timestamp)),
812    )?;
813    progress.changed += 1;
814    Ok(())
815}
816
817fn approx_size(row: &SnapshotRow) -> usize {
818    let text = |s: &Option<String>| s.as_ref().map_or(0, String::len);
819    match row {
820        SnapshotRow::Memory { row, embedding } => {
821            row.content.len() + text(&row.context) + text(embedding)
822        }
823        SnapshotRow::Chunks { chunks, .. } => chunks
824            .iter()
825            .map(|c| c.text.len() + text(&c.embedding))
826            .sum(),
827        SnapshotRow::LogTail { record } => record.data.as_ref().map_or(0, |d| d.to_string().len()),
828        _ => 0,
829    }
830}
831
832/// For each peer, the most of its log this store holds: the higher of its
833/// pull cursor and that peer's last push into it.
834fn export_cursors(conn: &Connection, own: &str) -> Result<BTreeMap<String, u64>> {
835    let mut cursors = BTreeMap::new();
836    let mut add = |sql: &str| -> Result<()> {
837        let mut stmt = conn.prepare(sql)?;
838        let rows = stmt.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)?)))?;
839        for row in rows {
840            let (peer, seq) = row?;
841            if peer == own
842                || peer.starts_with("push:")
843                || crate::maintenance::is_stale_cursor(&peer)
844            {
845                continue;
846            }
847            let seq = u64::try_from(seq.max(0))?;
848            let entry = cursors.entry(peer).or_insert(0);
849            *entry = (*entry).max(seq);
850        }
851        Ok(())
852    };
853    add("SELECT peer_id, last_seq FROM sync_state")?;
854    add("SELECT peer_id, seq FROM peer_in")?;
855    Ok(cursors)
856}
857
858type Emit<'a> = dyn FnMut(SnapshotRow) -> Result<()> + 'a;
859
860fn export_tombstones(conn: &Connection, emit: &mut Emit<'_>) -> Result<()> {
861    let mut stmt =
862        conn.prepare("SELECT node_id, node_type, deleted_at, machine_id FROM tombstones")?;
863    let mut rows = stmt.query([])?;
864    while let Some(r) = rows.next()? {
865        emit(SnapshotRow::MemoryTombstone {
866            id: r.get(0)?,
867            node_type: r
868                .get::<_, Option<String>>(1)?
869                .unwrap_or_else(|| "memory".to_string()),
870            deleted_at: r.get(2)?,
871            machine_id: r.get(3)?,
872        })?;
873    }
874    let mut stmt =
875        conn.prepare("SELECT node_type, node_id, deleted_at, origin FROM type_tombstones")?;
876    let mut rows = stmt.query([])?;
877    while let Some(r) = rows.next()? {
878        emit(SnapshotRow::TypeTombstone {
879            node_type: r.get(0)?,
880            node_id: r.get(1)?,
881            deleted_at: r.get(2)?,
882            origin: r.get(3)?,
883        })?;
884    }
885    Ok(())
886}
887
888fn export_conversations(conn: &Connection, emit: &mut Emit<'_>) -> Result<()> {
889    let mut stmt = conn.prepare(
890        "SELECT id, source, machine_id, started_at, project_path, updated_at, updated_by \
891         FROM conversations",
892    )?;
893    let mut rows = stmt.query([])?;
894    while let Some(r) = rows.next()? {
895        emit(SnapshotRow::Conversation {
896            row: Conversation {
897                id: parse_uuid(&r.get::<_, String>(0)?)?,
898                source: r.get(1)?,
899                machine_id: r.get(2)?,
900                started_at: parse_dt(&r.get::<_, String>(3)?)?,
901                project_path: r.get(4)?,
902            },
903            updated_at: r.get(5)?,
904            updated_by: r.get(6)?,
905        })?;
906    }
907    Ok(())
908}
909
910fn export_entities(conn: &Connection, emit: &mut Emit<'_>) -> Result<()> {
911    let mut stmt = conn.prepare(
912        "SELECT id, name, entity_type, embedding, aliases, updated_at, updated_by FROM entities",
913    )?;
914    let mut rows = stmt.query([])?;
915    while let Some(r) = rows.next()? {
916        emit(SnapshotRow::Entity {
917            row: row_to_entity(r)?,
918            updated_at: r.get(5)?,
919            updated_by: r.get(6)?,
920        })?;
921    }
922    Ok(())
923}
924
925fn export_memories(store: &SqliteStore, conn: &Connection, emit: &mut Emit<'_>) -> Result<()> {
926    let mut stmt = conn.prepare(&format!("SELECT {MEMORY_COLS}, embedding FROM memories"))?;
927    let mut rows = stmt.query([])?;
928    while let Some(r) = rows.next()? {
929        let stored = row_to_memory(r, true)?;
930        let mut memory = store.sync_form(conn, &stored)?.into_owned();
931        let embedding = encode_embedding(&memory.embedding);
932        memory.embedding = Vec::new();
933        emit(SnapshotRow::Memory {
934            row: memory,
935            embedding,
936        })?;
937    }
938    Ok(())
939}
940
941fn export_relations(conn: &Connection, emit: &mut Emit<'_>) -> Result<()> {
942    let mut stmt = conn.prepare(
943        "SELECT from_id, to_id, relation_type, strength, context, updated_at, updated_by \
944         FROM relations",
945    )?;
946    let mut rows = stmt.query([])?;
947    while let Some(r) = rows.next()? {
948        emit(SnapshotRow::Relation {
949            row: Relation {
950                from_id: parse_uuid(&r.get::<_, String>(0)?)?,
951                to_id: parse_uuid(&r.get::<_, String>(1)?)?,
952                relation_type: parse_relation_type(&r.get::<_, String>(2)?)?,
953                strength: r.get::<_, Option<f64>>(3)?.unwrap_or(1.0) as f32,
954                context: r.get(4)?,
955            },
956            updated_at: r.get(5)?,
957            updated_by: r.get(6)?,
958        })?;
959    }
960    Ok(())
961}
962
963fn export_distillations(conn: &Connection, emit: &mut Emit<'_>) -> Result<()> {
964    let mut stmt = conn.prepare(
965        "SELECT memory_id, distilled_id, distilled_at, model, status, updated_at, updated_by \
966         FROM distillation_log",
967    )?;
968    let mut rows = stmt.query([])?;
969    while let Some(r) = rows.next()? {
970        emit(SnapshotRow::Distillation {
971            row: Distillation {
972                memory_id: parse_uuid(&r.get::<_, String>(0)?)?,
973                distilled_id: r
974                    .get::<_, Option<String>>(1)?
975                    .map(|s| parse_uuid(&s))
976                    .transpose()?,
977                distilled_at: r
978                    .get::<_, Option<String>>(2)?
979                    .map(|s| parse_dt(&s))
980                    .transpose()?,
981                model: r.get(3)?,
982                status: r.get(4)?,
983            },
984            updated_at: r.get(5)?,
985            updated_by: r.get(6)?,
986        })?;
987    }
988    Ok(())
989}
990
991fn export_chunks(conn: &Connection, emit: &mut Emit<'_>) -> Result<()> {
992    let mut stmt = conn.prepare(
993        "SELECT memory_id, ord, text, embedding, content_hash FROM memory_chunks \
994         ORDER BY memory_id, ord",
995    )?;
996    let mut rows = stmt.query([])?;
997    let mut current: Option<(String, String, Vec<SnapshotChunk>)> = None;
998    while let Some(r) = rows.next()? {
999        let memory_id: String = r.get(0)?;
1000        let chunk = SnapshotChunk {
1001            ord: r.get::<_, i64>(1)? as usize,
1002            text: r.get(2)?,
1003            embedding: r.get::<_, Option<Vec<u8>>>(3)?.map(|blob| B64.encode(blob)),
1004        };
1005        let hash: String = r.get(4)?;
1006        match &mut current {
1007            Some((id, _, chunks)) if *id == memory_id => chunks.push(chunk),
1008            _ => {
1009                if let Some(done) = current.take() {
1010                    emit_chunks(done, emit)?;
1011                }
1012                current = Some((memory_id, hash, vec![chunk]));
1013            }
1014        }
1015    }
1016    if let Some(done) = current {
1017        emit_chunks(done, emit)?;
1018    }
1019    Ok(())
1020}
1021
1022fn emit_chunks(
1023    (memory_id, content_hash, chunks): (String, String, Vec<SnapshotChunk>),
1024    emit: &mut Emit<'_>,
1025) -> Result<()> {
1026    emit(SnapshotRow::Chunks {
1027        memory_id: parse_uuid(&memory_id)?,
1028        content_hash,
1029        chunks,
1030    })
1031}
1032
1033fn export_ingest_keys(conn: &Connection, emit: &mut Emit<'_>) -> Result<()> {
1034    let mut stmt = conn.prepare(
1035        "SELECT file_path, file_hash, ingested_at, memory_count, source, session_key, \
1036         content_hash, legacy_ids FROM ingest_log",
1037    )?;
1038    let mut rows = stmt.query([])?;
1039    while let Some(r) = rows.next()? {
1040        emit(SnapshotRow::IngestKey {
1041            file_path: r.get(0)?,
1042            file_hash: r.get(1)?,
1043            ingested_at: r.get(2)?,
1044            memory_count: r.get(3)?,
1045            source: r.get(4)?,
1046            session_key: r.get(5)?,
1047            content_hash: r.get(6)?,
1048            legacy_ids: r.get(7)?,
1049        })?;
1050    }
1051    Ok(())
1052}
1053
1054/// Rows above every live neighbour's ack: one any neighbour acked already
1055/// spreads through it.
1056fn export_tail(conn: &Connection, after: u64, seq: u64, emit: &mut Emit<'_>) -> Result<()> {
1057    let mut stmt = conn.prepare(
1058        "SELECT id, local_seq, origin_machine_id, origin_seq, op, node_type, node_id, \
1059         timestamp, data FROM sync_log WHERE local_seq > ?1 AND local_seq <= ?2 \
1060         ORDER BY local_seq",
1061    )?;
1062    let mut rows = stmt.query(params![i64::try_from(after)?, i64::try_from(seq)?])?;
1063    while let Some(r) = rows.next()? {
1064        let entry = super::sync_apply::row_to_sync_entry(r)?;
1065        emit(SnapshotRow::LogTail {
1066            record: record_of_entry(entry)?,
1067        })?;
1068    }
1069    Ok(())
1070}
1071
1072fn record_of_entry(entry: SyncEntry) -> Result<SyncRecord> {
1073    Ok(SyncRecord {
1074        local_seq: u64::try_from(entry.local_seq)?,
1075        origin_machine_id: entry.origin_machine_id,
1076        origin_seq: u64::try_from(entry.origin_seq)?,
1077        op: entry.op,
1078        node_type: entry.node_type,
1079        node_id: entry.node_id,
1080        timestamp: entry.timestamp,
1081        data: entry.data.map(|d| serde_json::from_str(&d)).transpose()?,
1082    })
1083}
1084
1085/// The stored ranges, then the new-layout keys still in the live log
1086/// coalesced into runs, so a receiver skips every record this store has
1087/// already settled.
1088fn export_dedup(conn: &Connection, emit: &mut Emit<'_>) -> Result<()> {
1089    let mut stmt = conn.prepare("SELECT origin, lo, hi FROM sync_dedup ORDER BY origin, lo")?;
1090    let mut rows = stmt.query([])?;
1091    while let Some(r) = rows.next()? {
1092        emit(SnapshotRow::Dedup {
1093            origin: r.get(0)?,
1094            lo: r.get(1)?,
1095            hi: r.get(2)?,
1096        })?;
1097    }
1098    let mut stmt = conn.prepare(
1099        "SELECT origin_machine_id, origin_seq FROM sync_log \
1100         ORDER BY origin_machine_id, origin_seq",
1101    )?;
1102    let mut rows = stmt.query([])?;
1103    let mut run: Option<(String, i64, i64)> = None;
1104    while let Some(r) = rows.next()? {
1105        let origin: String = r.get(0)?;
1106        let seq: i64 = r.get(1)?;
1107        if !clock::is_new_layout(&origin, seq) {
1108            continue;
1109        }
1110        match &mut run {
1111            Some((o, _, hi)) if *o == origin && seq <= hi.saturating_add(1) => {
1112                *hi = (*hi).max(seq);
1113            }
1114            _ => {
1115                if let Some((origin, lo, hi)) = run.take() {
1116                    emit(SnapshotRow::Dedup { origin, lo, hi })?;
1117                }
1118                run = Some((origin, seq, seq));
1119            }
1120        }
1121    }
1122    if let Some((origin, lo, hi)) = run {
1123        emit(SnapshotRow::Dedup { origin, lo, hi })?;
1124    }
1125    Ok(())
1126}
1127
1128#[cfg(test)]
1129mod tests {
1130    use super::*;
1131
1132    #[test]
1133    fn large_rows_flush_before_a_chunk_of_rows_fills() {
1134        let store =
1135            SqliteStore::in_memory("00000000-0000-4000-8000-000000000000".to_string()).unwrap();
1136        let mut merge = store
1137            .begin_snapshot_merge("s1", "00000000-0000-4000-8000-00000000000a", 0)
1138            .unwrap();
1139        let large = || SnapshotRow::Chunks {
1140            memory_id: Uuid::new_v4(),
1141            content_hash: String::new(),
1142            chunks: vec![SnapshotChunk {
1143                ord: 0,
1144                text: "x".repeat(MERGE_CHUNK_BYTES / 2 + 1),
1145                embedding: None,
1146            }],
1147        };
1148
1149        merge.push(large()).unwrap();
1150        assert_eq!(merge.progress().rows_done, 0);
1151        merge.push(large()).unwrap();
1152        assert_eq!(merge.progress().rows_done, 2, "the byte bound flushed");
1153    }
1154}