use std::collections::BTreeMap;
use anyhow::{Context, Result, bail};
use base64::Engine;
use base64::engine::general_purpose::STANDARD as B64;
use chrono::{DateTime, Utc};
use rusqlite::{Connection, OptionalExtension, Transaction, params};
use serde::{Deserialize, Serialize};
use uuid::Uuid;
use super::authored_order::{self, UPDATED_BY};
use super::{
MEMORY_COLS, SqliteStore, blob_to_embedding, clock, embedding_to_blob, floor, parse_dt,
parse_relation_type, parse_uuid, row_to_entity, row_to_memory, settled, sync_dedup,
};
use crate::schema::{
Conversation, Distillation, Entity, Memory, Relation, SyncEntry, SyncNodeType, SyncOp,
SyncRecord,
};
pub const MERGE_CHUNK: usize = 2_000;
pub const MERGE_CHUNK_BYTES: usize = 32 * 1024 * 1024;
const MERGE_PROGRESS: &str = "snapshot_merge";
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "t", rename_all = "snake_case")]
pub enum SnapshotRow {
MemoryTombstone {
id: String,
node_type: String,
deleted_at: String,
machine_id: Option<String>,
},
TypeTombstone {
node_type: String,
node_id: String,
deleted_at: i64,
origin: String,
},
Conversation {
row: Conversation,
updated_at: i64,
updated_by: String,
},
Entity {
row: Entity,
updated_at: i64,
updated_by: String,
},
Memory {
row: Memory,
embedding: Option<String>,
},
Relation {
row: Relation,
updated_at: i64,
updated_by: String,
},
Distillation {
row: Distillation,
updated_at: i64,
updated_by: String,
},
Chunks {
memory_id: Uuid,
content_hash: String,
chunks: Vec<SnapshotChunk>,
},
IngestKey {
file_path: String,
file_hash: String,
ingested_at: Option<String>,
memory_count: Option<i64>,
source: Option<String>,
session_key: Option<String>,
content_hash: Option<String>,
legacy_ids: i64,
},
LogTail {
record: SyncRecord,
},
Dedup {
origin: String,
lo: i64,
hi: i64,
},
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SnapshotChunk {
pub ord: usize,
pub text: String,
pub embedding: Option<String>,
}
impl SnapshotRow {
pub fn section(&self) -> &'static str {
match self {
SnapshotRow::MemoryTombstone { .. } => "tombstones",
SnapshotRow::TypeTombstone { .. } => "type_tombstones",
SnapshotRow::Conversation { .. } => "conversations",
SnapshotRow::Entity { .. } => "entities",
SnapshotRow::Memory { .. } => "memories",
SnapshotRow::Relation { .. } => "relations",
SnapshotRow::Distillation { .. } => "distillation_log",
SnapshotRow::Chunks { .. } => "memory_chunks",
SnapshotRow::IngestKey { .. } => "ingest_log",
SnapshotRow::LogTail { .. } => "log_tail",
SnapshotRow::Dedup { .. } => "sync_dedup",
}
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct SnapshotOptions {
pub with_chunks: bool,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct SnapshotState {
pub seq: u64,
pub cursors: BTreeMap<String, u64>,
pub counts: BTreeMap<String, u64>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct MergeProgress {
pub snapshot_id: String,
pub exporter: String,
pub seq: u64,
pub rows_done: u64,
pub next_n: u64,
pub changed: u64,
pub settled: u64,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct MergeOutcome {
pub rows: u64,
pub changed: u64,
pub settled: u64,
}
impl SqliteStore {
pub fn export_snapshot(
&self,
options: SnapshotOptions,
sink: &mut dyn FnMut(SnapshotRow) -> Result<()>,
) -> Result<SnapshotState> {
let conn = self.reader();
let tx = conn.unchecked_transaction()?;
let seq = u64::try_from(clock::state(&tx)?.last.max(0))?;
let mut state = SnapshotState {
seq,
cursors: export_cursors(&tx, self.machine_id())?,
counts: BTreeMap::new(),
};
let mut emit = |row: SnapshotRow| -> Result<()> {
*state.counts.entry(row.section().to_string()).or_insert(0) += 1;
sink(row)
};
export_tombstones(&tx, &mut emit)?;
export_conversations(&tx, &mut emit)?;
export_entities(&tx, &mut emit)?;
export_memories(self, &tx, &mut emit)?;
export_relations(&tx, &mut emit)?;
export_distillations(&tx, &mut emit)?;
if options.with_chunks {
export_chunks(&tx, &mut emit)?;
}
export_ingest_keys(&tx, &mut emit)?;
let tail_after = floor::max_live_ack(&tx, self.machine_id())?.unwrap_or(0);
export_tail(&tx, tail_after, seq, &mut emit)?;
export_dedup(&tx, &mut emit)?;
tx.commit()?;
Ok(state)
}
pub fn snapshot_merge_progress(&self) -> Result<Option<MergeProgress>> {
read_progress(&self.reader())
}
pub fn begin_snapshot_merge(
&self,
snapshot_id: &str,
exporter: &str,
seq: u64,
) -> Result<SnapshotMerge<'_>> {
let stored = read_progress(&self.reader())?;
let progress = match stored {
Some(progress) if progress.snapshot_id == snapshot_id => progress,
_ => {
let progress = MergeProgress {
snapshot_id: snapshot_id.to_string(),
exporter: exporter.to_string(),
seq,
..MergeProgress::default()
};
write_progress(&self.conn(), &progress)?;
progress
}
};
Ok(SnapshotMerge {
store: self,
progress,
index: 0,
batch: Vec::with_capacity(MERGE_CHUNK),
batch_bytes: 0,
})
}
pub fn abandon_snapshot_merge(&self) -> Result<()> {
self.conn().execute(
"DELETE FROM index_meta WHERE key = ?1",
params![MERGE_PROGRESS],
)?;
Ok(())
}
}
pub struct SnapshotMerge<'a> {
store: &'a SqliteStore,
progress: MergeProgress,
index: u64,
batch: Vec<SnapshotRow>,
batch_bytes: usize,
}
impl SnapshotMerge<'_> {
pub fn progress(&self) -> &MergeProgress {
&self.progress
}
pub fn push(&mut self, row: SnapshotRow) -> Result<()> {
let index = self.index;
self.index += 1;
if index < self.progress.rows_done {
return Ok(());
}
self.batch_bytes += approx_size(&row);
self.batch.push(row);
if self.batch.len() >= MERGE_CHUNK || self.batch_bytes >= MERGE_CHUNK_BYTES {
self.flush()?;
}
Ok(())
}
fn flush(&mut self) -> Result<()> {
if self.batch.is_empty() {
return Ok(());
}
let rows = std::mem::take(&mut self.batch);
self.batch_bytes = 0;
let store = self.store;
let mut progress = self.progress.clone();
let now = Utc::now().timestamp_micros();
store.with_txn(|tx| {
for row in rows {
let index = progress.rows_done;
merge_row(store, tx, &mut progress, index, row, now)?;
progress.rows_done += 1;
}
write_progress(tx, &progress)
})?;
self.progress = progress;
Ok(())
}
pub fn finish(mut self, cursors: &BTreeMap<String, u64>) -> Result<MergeOutcome> {
self.flush()?;
let progress = self.progress.clone();
let own = self.store.machine_id().to_string();
self.store.with_txn(|tx| {
raise_cursor(tx, &progress.exporter, progress.seq)?;
for (peer, seq) in cursors {
if *peer != own && *peer != progress.exporter {
raise_cursor(tx, peer, *seq)?;
}
}
tx.execute(
"INSERT INTO peer_in (peer_id, seq) VALUES (?1, ?2) \
ON CONFLICT(peer_id) DO UPDATE SET seq = MAX(peer_in.seq, excluded.seq)",
params![progress.exporter, i64::try_from(progress.seq)?],
)?;
tx.execute(
"DELETE FROM index_meta WHERE key = ?1",
params![MERGE_PROGRESS],
)?;
Ok(())
})?;
Ok(MergeOutcome {
rows: progress.rows_done,
changed: progress.changed,
settled: progress.settled,
})
}
}
fn read_progress(conn: &Connection) -> Result<Option<MergeProgress>> {
let value: Option<String> = conn
.query_row(
"SELECT value FROM index_meta WHERE key = ?1",
params![MERGE_PROGRESS],
|r| r.get(0),
)
.optional()?;
value
.map(|v| serde_json::from_str(&v).context("reading the snapshot merge's progress"))
.transpose()
}
fn write_progress(conn: &Connection, progress: &MergeProgress) -> Result<()> {
conn.execute(
"INSERT OR REPLACE INTO index_meta (key, value) VALUES (?1, ?2)",
params![MERGE_PROGRESS, serde_json::to_string(progress)?],
)?;
Ok(())
}
fn raise_cursor(tx: &Transaction<'_>, peer: &str, seq: u64) -> Result<()> {
tx.execute(
"INSERT INTO sync_state (peer_id, last_seq, last_sync_at) VALUES (?1, ?2, ?3) \
ON CONFLICT(peer_id) DO UPDATE SET last_seq = MAX(sync_state.last_seq, excluded.last_seq)",
params![peer, i64::try_from(seq)?, Utc::now().to_rfc3339()],
)?;
Ok(())
}
fn micros_to_time(micros: i64) -> DateTime<Utc> {
DateTime::from_timestamp_micros(micros).unwrap_or(DateTime::UNIX_EPOCH)
}
fn with_updated_by<T: Serialize>(value: &T, by: &str) -> Result<serde_json::Value> {
let mut json = serde_json::to_value(value)?;
if let Some(object) = json.as_object_mut() {
object.insert(UPDATED_BY.to_string(), by.into());
}
Ok(json)
}
fn encode_embedding(embedding: &[f32]) -> Option<String> {
(!embedding.is_empty()).then(|| B64.encode(embedding_to_blob(embedding)))
}
fn decode_embedding(encoded: Option<&str>) -> Result<Vec<f32>> {
match encoded {
Some(encoded) => Ok(blob_to_embedding(
&B64.decode(encoded)
.context("decoding a snapshot embedding")?,
)),
None => Ok(Vec::new()),
}
}
fn row_record(row: &SnapshotRow, origin: &str, n: u64) -> Result<Option<SyncRecord>> {
let record = |op, node_type, node_id: String, micros: i64, data| SyncRecord {
local_seq: 0,
origin_machine_id: origin.to_string(),
origin_seq: n,
op,
node_type,
node_id,
timestamp: micros_to_time(micros),
data,
};
Ok(Some(match row {
SnapshotRow::Conversation {
row,
updated_at,
updated_by,
} => record(
SyncOp::Update,
SyncNodeType::Conversation,
row.id.to_string(),
*updated_at,
Some(with_updated_by(row, updated_by)?),
),
SnapshotRow::Entity {
row,
updated_at,
updated_by,
} => record(
SyncOp::Update,
SyncNodeType::Entity,
row.id.to_string(),
*updated_at,
Some(with_updated_by(row, updated_by)?),
),
SnapshotRow::Memory { row, embedding } => {
let memory = Memory {
embedding: decode_embedding(embedding.as_deref())?,
..row.clone()
};
record(
SyncOp::Update,
SyncNodeType::Memory,
memory.id.to_string(),
memory.updated_at.timestamp_micros(),
Some(serde_json::to_value(&memory)?),
)
}
SnapshotRow::Relation {
row,
updated_at,
updated_by,
} => record(
SyncOp::Create,
SyncNodeType::Relation,
authored_order::relation_key(row),
*updated_at,
Some(with_updated_by(row, updated_by)?),
),
SnapshotRow::Distillation {
row,
updated_at,
updated_by,
} => record(
SyncOp::Create,
SyncNodeType::Distillation,
row.memory_id.to_string(),
*updated_at,
Some(with_updated_by(row, updated_by)?),
),
SnapshotRow::TypeTombstone {
node_type,
node_id,
deleted_at,
origin: by,
} => match node_type.as_str() {
"conversation" | "entity" => record(
SyncOp::Delete,
if node_type == "entity" {
SyncNodeType::Entity
} else {
SyncNodeType::Conversation
},
node_id.clone(),
*deleted_at,
Some(serde_json::json!({ UPDATED_BY: by })),
),
"relation" => {
let relation = relation_of_key(node_id)?;
record(
SyncOp::Delete,
SyncNodeType::Relation,
node_id.clone(),
*deleted_at,
Some(with_updated_by(&relation, by)?),
)
}
"distillation" => {
let dropped = distillation_of_key(node_id)?;
record(
SyncOp::Delete,
SyncNodeType::Distillation,
dropped.memory_id.to_string(),
*deleted_at,
Some(with_updated_by(&dropped, by)?),
)
}
_ => return Ok(None),
},
_ => return Ok(None),
}))
}
fn relation_of_key(key: &str) -> Result<Relation> {
let mut parts = key.splitn(3, ':');
let (Some(from), Some(to), Some(kind)) = (parts.next(), parts.next(), parts.next()) else {
bail!("relation tombstone key {key:?} is not from:to:type");
};
Ok(Relation {
from_id: parse_uuid(from)?,
to_id: parse_uuid(to)?,
relation_type: parse_relation_type(kind)?,
strength: 0.0,
context: None,
})
}
fn distillation_of_key(key: &str) -> Result<Distillation> {
let (memory, distilled) = key
.split_once(':')
.with_context(|| format!("distillation tombstone key {key:?} is not memory:distilled"))?;
Ok(Distillation {
memory_id: parse_uuid(memory)?,
distilled_id: (!distilled.is_empty())
.then(|| parse_uuid(distilled))
.transpose()?,
distilled_at: None,
model: None,
status: String::new(),
})
}
fn merge_row(
store: &SqliteStore,
tx: &Transaction<'_>,
progress: &mut MergeProgress,
index: u64,
row: SnapshotRow,
now: i64,
) -> Result<()> {
let origin = format!("snap:{}", progress.snapshot_id);
let mut attempt = progress.clone();
tx.execute_batch("SAVEPOINT merge_row")?;
match apply_row(store, tx, &mut attempt, &origin, &row) {
Ok(()) => {
tx.execute_batch("RELEASE merge_row")?;
*progress = attempt;
Ok(())
}
Err(e) => {
tx.execute_batch("ROLLBACK TO merge_row; RELEASE merge_row")?;
if settled::is_outer(&e) {
return Err(e);
}
progress.settled += 1;
match settle_record(&row, &origin, progress.next_n) {
Some(record) => settled::settle_on_tx(tx, &origin, index, &record, &e, now),
None => {
tracing::warn!(
kind = row.section(),
error = format!("{e:#}"),
"settled a snapshot row that cannot be merged"
);
Ok(())
}
}
}
}
}
fn apply_row(
store: &SqliteStore,
tx: &Transaction<'_>,
progress: &mut MergeProgress,
origin: &str,
row: &SnapshotRow,
) -> Result<()> {
match row {
SnapshotRow::MemoryTombstone {
id,
node_type,
deleted_at,
machine_id,
} => {
if merge_memory_tombstone(
store,
tx,
origin,
progress.next_n,
id,
node_type,
deleted_at,
machine_id.as_deref(),
)? {
progress.next_n += 1;
progress.changed += 1;
}
Ok(())
}
SnapshotRow::TypeTombstone {
node_type,
node_id,
deleted_at,
origin: by,
} if node_type == "distillation_slot" => merge_slot_floor(tx, node_id, *deleted_at, by),
SnapshotRow::Chunks {
memory_id,
content_hash,
chunks,
} => merge_chunks(tx, *memory_id, content_hash, chunks),
SnapshotRow::IngestKey {
file_path,
file_hash,
ingested_at,
memory_count,
source,
session_key,
content_hash,
legacy_ids,
} => {
tx.execute(
"INSERT OR IGNORE INTO ingest_log (file_path, file_hash, ingested_at, memory_count, \
source, session_key, content_hash, legacy_ids) VALUES (?1,?2,?3,?4,?5,?6,?7,?8)",
params![
file_path,
file_hash,
ingested_at,
memory_count,
source,
session_key,
content_hash,
legacy_ids
],
)?;
Ok(())
}
SnapshotRow::LogTail { record } => merge_tail_record(store, tx, progress, record),
SnapshotRow::Dedup { origin, lo, hi } => sync_dedup::merge_range(tx, origin, *lo, *hi),
_ => {
let Some(record) = row_record(row, origin, progress.next_n)? else {
return Ok(());
};
if store.apply_logged_on_tx(tx, &record)?? {
progress.next_n += 1;
progress.changed += 1;
}
Ok(())
}
}
}
fn settle_record(row: &SnapshotRow, origin: &str, n: u64) -> Option<SyncRecord> {
match row {
SnapshotRow::LogTail { record } => Some(record.clone()),
SnapshotRow::MemoryTombstone {
id,
node_type,
deleted_at,
..
} if node_type == "memory" => Some(memory_delete_record(
origin,
n,
id,
parse_dt(deleted_at).unwrap_or(DateTime::UNIX_EPOCH),
)),
_ => row_record(row, origin, n).ok().flatten(),
}
}
fn memory_delete_record(origin: &str, n: u64, id: &str, authored: DateTime<Utc>) -> SyncRecord {
SyncRecord {
local_seq: 0,
origin_machine_id: origin.to_string(),
origin_seq: n,
op: SyncOp::Delete,
node_type: SyncNodeType::Memory,
node_id: id.to_string(),
timestamp: authored,
data: None,
}
}
#[allow(clippy::too_many_arguments)]
fn merge_memory_tombstone(
store: &SqliteStore,
tx: &Transaction<'_>,
origin: &str,
n: u64,
id: &str,
node_type: &str,
deleted_at: &str,
machine_id: Option<&str>,
) -> Result<bool> {
let known: Option<i64> = tx
.query_row(
"SELECT 1 FROM tombstones WHERE node_id = ?1",
params![id],
|r| r.get(0),
)
.optional()?;
if known.is_some() {
return Ok(false);
}
tx.execute(
"INSERT INTO tombstones (node_id, node_type, deleted_at, machine_id) VALUES (?1,?2,?3,?4)",
params![id, node_type, deleted_at, machine_id],
)?;
if node_type != "memory" {
return Ok(false);
}
let record = memory_delete_record(origin, n, id, parse_dt(deleted_at)?);
store.apply_memory_delete(tx, &record)?;
Ok(true)
}
fn merge_slot_floor(
tx: &Transaction<'_>,
memory_id: &str,
deleted_at: i64,
by: &str,
) -> Result<()> {
let floor = authored_order::Tombstone {
deleted_at,
origin: by.to_string(),
};
if let Ok(memory) = Uuid::parse_str(memory_id)
&& let Some((_, row)) = authored_order::marker_stamp(tx, memory)?
&& floor.beats(&row)
{
tx.execute(
"DELETE FROM distillation_log WHERE memory_id = ?1",
params![memory_id],
)?;
}
authored_order::record_tombstone(tx, "distillation_slot", memory_id, &floor)?;
Ok(())
}
fn merge_chunks(
tx: &Transaction<'_>,
memory_id: Uuid,
content_hash: &str,
chunks: &[SnapshotChunk],
) -> Result<()> {
let id = memory_id.to_string();
let content: Option<String> = tx
.query_row(
"SELECT content FROM memories WHERE id = ?1 AND tier = 'hot' AND superseded_by IS NULL",
params![id],
|r| r.get(0),
)
.optional()?;
if content.is_none_or(|c| crate::chunk::content_hash(&c) != content_hash) {
return Ok(());
}
let embedded_here: i64 = tx.query_row(
"SELECT COUNT(*) FROM memory_chunks WHERE memory_id = ?1 AND content_hash = ?2 \
AND embedding IS NOT NULL",
params![id, content_hash],
|r| r.get(0),
)?;
if embedded_here as usize >= chunks.len() {
return Ok(());
}
let decoded = chunks
.iter()
.map(|c| {
Ok((
c.ord,
c.text.as_str(),
decode_embedding(c.embedding.as_deref())?,
))
})
.collect::<Result<Vec<_>>>()?;
SqliteStore::replace_chunk_rows(
tx,
&id,
content_hash,
decoded
.iter()
.map(|(ord, text, e)| (*ord, *text, e.as_slice())),
)?;
SqliteStore::append_outbox(tx, memory_id, "upsert")
}
fn merge_tail_record(
store: &SqliteStore,
tx: &Transaction<'_>,
progress: &mut MergeProgress,
record: &SyncRecord,
) -> Result<()> {
let origin = record.origin_machine_id.as_str();
let origin_seq = record.origin_seq as i64;
if sync_dedup::seen(tx, origin, origin_seq)? || origin == store.machine_id() {
return Ok(());
}
store.apply_on_tx(tx, record)?;
let data = record
.data
.as_ref()
.map(serde_json::to_string)
.transpose()?;
store.append_sync_log(
tx,
record.op,
record.node_type,
&record.node_id,
data.as_deref(),
Some((origin, origin_seq, record.timestamp)),
)?;
progress.changed += 1;
Ok(())
}
fn approx_size(row: &SnapshotRow) -> usize {
let text = |s: &Option<String>| s.as_ref().map_or(0, String::len);
match row {
SnapshotRow::Memory { row, embedding } => {
row.content.len() + text(&row.context) + text(embedding)
}
SnapshotRow::Chunks { chunks, .. } => chunks
.iter()
.map(|c| c.text.len() + text(&c.embedding))
.sum(),
SnapshotRow::LogTail { record } => record.data.as_ref().map_or(0, |d| d.to_string().len()),
_ => 0,
}
}
fn export_cursors(conn: &Connection, own: &str) -> Result<BTreeMap<String, u64>> {
let mut cursors = BTreeMap::new();
let mut add = |sql: &str| -> Result<()> {
let mut stmt = conn.prepare(sql)?;
let rows = stmt.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)?)))?;
for row in rows {
let (peer, seq) = row?;
if peer == own
|| peer.starts_with("push:")
|| crate::maintenance::is_stale_cursor(&peer)
{
continue;
}
let seq = u64::try_from(seq.max(0))?;
let entry = cursors.entry(peer).or_insert(0);
*entry = (*entry).max(seq);
}
Ok(())
};
add("SELECT peer_id, last_seq FROM sync_state")?;
add("SELECT peer_id, seq FROM peer_in")?;
Ok(cursors)
}
type Emit<'a> = dyn FnMut(SnapshotRow) -> Result<()> + 'a;
fn export_tombstones(conn: &Connection, emit: &mut Emit<'_>) -> Result<()> {
let mut stmt =
conn.prepare("SELECT node_id, node_type, deleted_at, machine_id FROM tombstones")?;
let mut rows = stmt.query([])?;
while let Some(r) = rows.next()? {
emit(SnapshotRow::MemoryTombstone {
id: r.get(0)?,
node_type: r
.get::<_, Option<String>>(1)?
.unwrap_or_else(|| "memory".to_string()),
deleted_at: r.get(2)?,
machine_id: r.get(3)?,
})?;
}
let mut stmt =
conn.prepare("SELECT node_type, node_id, deleted_at, origin FROM type_tombstones")?;
let mut rows = stmt.query([])?;
while let Some(r) = rows.next()? {
emit(SnapshotRow::TypeTombstone {
node_type: r.get(0)?,
node_id: r.get(1)?,
deleted_at: r.get(2)?,
origin: r.get(3)?,
})?;
}
Ok(())
}
fn export_conversations(conn: &Connection, emit: &mut Emit<'_>) -> Result<()> {
let mut stmt = conn.prepare(
"SELECT id, source, machine_id, started_at, project_path, updated_at, updated_by \
FROM conversations",
)?;
let mut rows = stmt.query([])?;
while let Some(r) = rows.next()? {
emit(SnapshotRow::Conversation {
row: Conversation {
id: parse_uuid(&r.get::<_, String>(0)?)?,
source: r.get(1)?,
machine_id: r.get(2)?,
started_at: parse_dt(&r.get::<_, String>(3)?)?,
project_path: r.get(4)?,
},
updated_at: r.get(5)?,
updated_by: r.get(6)?,
})?;
}
Ok(())
}
fn export_entities(conn: &Connection, emit: &mut Emit<'_>) -> Result<()> {
let mut stmt = conn.prepare(
"SELECT id, name, entity_type, embedding, aliases, updated_at, updated_by FROM entities",
)?;
let mut rows = stmt.query([])?;
while let Some(r) = rows.next()? {
emit(SnapshotRow::Entity {
row: row_to_entity(r)?,
updated_at: r.get(5)?,
updated_by: r.get(6)?,
})?;
}
Ok(())
}
fn export_memories(store: &SqliteStore, conn: &Connection, emit: &mut Emit<'_>) -> Result<()> {
let mut stmt = conn.prepare(&format!("SELECT {MEMORY_COLS}, embedding FROM memories"))?;
let mut rows = stmt.query([])?;
while let Some(r) = rows.next()? {
let stored = row_to_memory(r, true)?;
let mut memory = store.sync_form(conn, &stored)?.into_owned();
let embedding = encode_embedding(&memory.embedding);
memory.embedding = Vec::new();
emit(SnapshotRow::Memory {
row: memory,
embedding,
})?;
}
Ok(())
}
fn export_relations(conn: &Connection, emit: &mut Emit<'_>) -> Result<()> {
let mut stmt = conn.prepare(
"SELECT from_id, to_id, relation_type, strength, context, updated_at, updated_by \
FROM relations",
)?;
let mut rows = stmt.query([])?;
while let Some(r) = rows.next()? {
emit(SnapshotRow::Relation {
row: Relation {
from_id: parse_uuid(&r.get::<_, String>(0)?)?,
to_id: parse_uuid(&r.get::<_, String>(1)?)?,
relation_type: parse_relation_type(&r.get::<_, String>(2)?)?,
strength: r.get::<_, Option<f64>>(3)?.unwrap_or(1.0) as f32,
context: r.get(4)?,
},
updated_at: r.get(5)?,
updated_by: r.get(6)?,
})?;
}
Ok(())
}
fn export_distillations(conn: &Connection, emit: &mut Emit<'_>) -> Result<()> {
let mut stmt = conn.prepare(
"SELECT memory_id, distilled_id, distilled_at, model, status, updated_at, updated_by \
FROM distillation_log",
)?;
let mut rows = stmt.query([])?;
while let Some(r) = rows.next()? {
emit(SnapshotRow::Distillation {
row: Distillation {
memory_id: parse_uuid(&r.get::<_, String>(0)?)?,
distilled_id: r
.get::<_, Option<String>>(1)?
.map(|s| parse_uuid(&s))
.transpose()?,
distilled_at: r
.get::<_, Option<String>>(2)?
.map(|s| parse_dt(&s))
.transpose()?,
model: r.get(3)?,
status: r.get(4)?,
},
updated_at: r.get(5)?,
updated_by: r.get(6)?,
})?;
}
Ok(())
}
fn export_chunks(conn: &Connection, emit: &mut Emit<'_>) -> Result<()> {
let mut stmt = conn.prepare(
"SELECT memory_id, ord, text, embedding, content_hash FROM memory_chunks \
ORDER BY memory_id, ord",
)?;
let mut rows = stmt.query([])?;
let mut current: Option<(String, String, Vec<SnapshotChunk>)> = None;
while let Some(r) = rows.next()? {
let memory_id: String = r.get(0)?;
let chunk = SnapshotChunk {
ord: r.get::<_, i64>(1)? as usize,
text: r.get(2)?,
embedding: r.get::<_, Option<Vec<u8>>>(3)?.map(|blob| B64.encode(blob)),
};
let hash: String = r.get(4)?;
match &mut current {
Some((id, _, chunks)) if *id == memory_id => chunks.push(chunk),
_ => {
if let Some(done) = current.take() {
emit_chunks(done, emit)?;
}
current = Some((memory_id, hash, vec![chunk]));
}
}
}
if let Some(done) = current {
emit_chunks(done, emit)?;
}
Ok(())
}
fn emit_chunks(
(memory_id, content_hash, chunks): (String, String, Vec<SnapshotChunk>),
emit: &mut Emit<'_>,
) -> Result<()> {
emit(SnapshotRow::Chunks {
memory_id: parse_uuid(&memory_id)?,
content_hash,
chunks,
})
}
fn export_ingest_keys(conn: &Connection, emit: &mut Emit<'_>) -> Result<()> {
let mut stmt = conn.prepare(
"SELECT file_path, file_hash, ingested_at, memory_count, source, session_key, \
content_hash, legacy_ids FROM ingest_log",
)?;
let mut rows = stmt.query([])?;
while let Some(r) = rows.next()? {
emit(SnapshotRow::IngestKey {
file_path: r.get(0)?,
file_hash: r.get(1)?,
ingested_at: r.get(2)?,
memory_count: r.get(3)?,
source: r.get(4)?,
session_key: r.get(5)?,
content_hash: r.get(6)?,
legacy_ids: r.get(7)?,
})?;
}
Ok(())
}
fn export_tail(conn: &Connection, after: u64, seq: u64, emit: &mut Emit<'_>) -> Result<()> {
let mut stmt = conn.prepare(
"SELECT id, local_seq, origin_machine_id, origin_seq, op, node_type, node_id, \
timestamp, data FROM sync_log WHERE local_seq > ?1 AND local_seq <= ?2 \
ORDER BY local_seq",
)?;
let mut rows = stmt.query(params![i64::try_from(after)?, i64::try_from(seq)?])?;
while let Some(r) = rows.next()? {
let entry = super::sync_apply::row_to_sync_entry(r)?;
emit(SnapshotRow::LogTail {
record: record_of_entry(entry)?,
})?;
}
Ok(())
}
fn record_of_entry(entry: SyncEntry) -> Result<SyncRecord> {
Ok(SyncRecord {
local_seq: u64::try_from(entry.local_seq)?,
origin_machine_id: entry.origin_machine_id,
origin_seq: u64::try_from(entry.origin_seq)?,
op: entry.op,
node_type: entry.node_type,
node_id: entry.node_id,
timestamp: entry.timestamp,
data: entry.data.map(|d| serde_json::from_str(&d)).transpose()?,
})
}
fn export_dedup(conn: &Connection, emit: &mut Emit<'_>) -> Result<()> {
let mut stmt = conn.prepare("SELECT origin, lo, hi FROM sync_dedup ORDER BY origin, lo")?;
let mut rows = stmt.query([])?;
while let Some(r) = rows.next()? {
emit(SnapshotRow::Dedup {
origin: r.get(0)?,
lo: r.get(1)?,
hi: r.get(2)?,
})?;
}
let mut stmt = conn.prepare(
"SELECT origin_machine_id, origin_seq FROM sync_log \
ORDER BY origin_machine_id, origin_seq",
)?;
let mut rows = stmt.query([])?;
let mut run: Option<(String, i64, i64)> = None;
while let Some(r) = rows.next()? {
let origin: String = r.get(0)?;
let seq: i64 = r.get(1)?;
if !clock::is_new_layout(&origin, seq) {
continue;
}
match &mut run {
Some((o, _, hi)) if *o == origin && seq <= hi.saturating_add(1) => {
*hi = (*hi).max(seq);
}
_ => {
if let Some((origin, lo, hi)) = run.take() {
emit(SnapshotRow::Dedup { origin, lo, hi })?;
}
run = Some((origin, seq, seq));
}
}
}
if let Some((origin, lo, hi)) = run {
emit(SnapshotRow::Dedup { origin, lo, hi })?;
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn large_rows_flush_before_a_chunk_of_rows_fills() {
let store =
SqliteStore::in_memory("00000000-0000-4000-8000-000000000000".to_string()).unwrap();
let mut merge = store
.begin_snapshot_merge("s1", "00000000-0000-4000-8000-00000000000a", 0)
.unwrap();
let large = || SnapshotRow::Chunks {
memory_id: Uuid::new_v4(),
content_hash: String::new(),
chunks: vec![SnapshotChunk {
ord: 0,
text: "x".repeat(MERGE_CHUNK_BYTES / 2 + 1),
embedding: None,
}],
};
merge.push(large()).unwrap();
assert_eq!(merge.progress().rows_done, 0);
merge.push(large()).unwrap();
assert_eq!(merge.progress().rows_done, 2, "the byte bound flushed");
}
}