use std::collections::HashMap;
use std::path::Path;
use anyhow::Result;
use chrono::{DateTime, Utc};
use rusqlite::{OptionalExtension, Transaction, params};
use uuid::Uuid;
use super::{
MEMORY_COLS, SqliteStore, embedding_to_blob, parse_dt, parse_relation_type, parse_uuid,
relation_type_to_str, row_to_entity, row_to_memory, sync_op_to_str,
};
use crate::machine::MachineIdentity;
use crate::project_path;
use crate::schema::{
CompactStats, Conversation, Distillation, Entity, IngestLogEntry, IngestRecord, Memory,
Relation, RelationType, SyncEntry, SyncNodeType, SyncOp, SyncRecord, SyncState,
distillable_source, explicit_sources_list,
};
use crate::store::{
AmbiguousSlug, ApplyOutcome, ConsolidationStats, ExplicitRestoreStats, PendingDistillation,
RefiledSlug, SkipReason, Store, StoreAdmin, StraySlugReport,
};
pub(super) const UNCONSOLIDATED_PREDICATE: &str = concat!(
distillable_source!("source"),
" AND tier = 'hot' AND superseded_by IS NULL \
AND id NOT IN (SELECT memory_id FROM distillation_log)"
);
const RETIRABLE_PREDICATE: &str = concat!(
"m.tier = 'hot' AND m.superseded_by IS NULL AND ",
distillable_source!("m.source"),
" AND (dl.memory_id = dl.distilled_id \
OR EXISTS (SELECT 1 FROM memories d WHERE d.id = dl.distilled_id))"
);
fn parse_record_data<T: serde::de::DeserializeOwned>(record: &SyncRecord, what: &str) -> Result<T> {
let data = record
.data
.clone()
.ok_or_else(|| anyhow::anyhow!("missing data for {what}"))?;
Ok(serde_json::from_value(data)?)
}
fn normalize_incoming(memory: &mut Memory) {
if memory.updated_at == DateTime::UNIX_EPOCH {
memory.updated_at = memory.created_at;
}
}
impl SqliteStore {
fn apply_on_tx(&self, tx: &Transaction<'_>, record: &SyncRecord) -> Result<ApplyOutcome> {
let redacted = super::payload_redaction::redacted_record(record);
let record = redacted.as_ref().unwrap_or(record);
let origin = record.origin_machine_id.as_str();
let origin_seq = record.origin_seq as i64;
match (&record.op, &record.node_type) {
(SyncOp::Create, SyncNodeType::Memory) => {
let mut memory: Memory = parse_record_data(record, "memory create")?;
normalize_incoming(&mut memory);
if Self::tombstoned(tx, memory.id)? {
return Ok(ApplyOutcome::Skipped(SkipReason::Tombstoned));
}
if Self::memory_exists(tx, memory.id)? {
Self::fill_session_at(tx, &memory)?;
return Ok(ApplyOutcome::Skipped(SkipReason::AlreadyPresent));
}
Self::insert_memory(tx, &memory)?;
Self::append_outbox(tx, memory.id, "upsert")?;
Self::log_applied(tx, record, SyncOp::Create, origin, origin_seq)?;
Ok(ApplyOutcome::Created)
}
(SyncOp::Update, SyncNodeType::Memory) => {
let mut incoming: Memory = parse_record_data(record, "memory update")?;
normalize_incoming(&mut incoming);
match Self::existing_lww_key(tx, incoming.id)? {
None => {
if Self::tombstoned(tx, incoming.id)? {
return Ok(ApplyOutcome::Skipped(SkipReason::Tombstoned));
}
Self::insert_memory(tx, &incoming)?;
Self::append_outbox(tx, incoming.id, "upsert")?;
Self::log_applied(tx, record, SyncOp::Create, origin, origin_seq)?;
Ok(ApplyOutcome::Created)
}
Some((updated_at, access_count, machine_id)) => {
let incoming_wins = if incoming.updated_at != updated_at {
incoming.updated_at > updated_at
} else if incoming.access_count != access_count {
incoming.access_count > access_count
} else {
incoming.machine_id > machine_id
};
if !incoming_wins {
Self::fill_session_at(tx, &incoming)?;
return Ok(ApplyOutcome::Skipped(SkipReason::StaleWrite));
}
if incoming.session_at.is_none() {
incoming.session_at = Self::stored_session_at(tx, incoming.id)?;
}
Self::insert_memory(tx, &incoming)?;
Self::append_outbox(tx, incoming.id, "upsert")?;
Self::log_applied(tx, record, SyncOp::Update, origin, origin_seq)?;
Ok(ApplyOutcome::Updated)
}
}
}
(SyncOp::Delete, SyncNodeType::Memory) => {
if Self::tombstone_row_exists(tx, &record.node_id)? {
return Ok(ApplyOutcome::Skipped(SkipReason::Tombstoned));
}
tx.execute(
"DELETE FROM memories WHERE id = ?1",
params![record.node_id],
)?;
tx.execute(
"DELETE FROM relations WHERE from_id = ?1 OR to_id = ?1",
params![record.node_id],
)?;
Self::upsert_tombstone(tx, &record.node_id, "memory", origin)?;
if let Ok(id) = Uuid::parse_str(&record.node_id) {
Self::append_outbox(tx, id, "delete")?;
}
Self::log_applied(tx, record, SyncOp::Delete, origin, origin_seq)?;
Ok(ApplyOutcome::Deleted)
}
(SyncOp::Create, SyncNodeType::Conversation) => {
let conversation: Conversation = parse_record_data(record, "conversation create")?;
if Self::row_exists(tx, "conversations", &conversation.id.to_string())? {
return Ok(ApplyOutcome::Skipped(SkipReason::AlreadyPresent));
}
Self::insert_conversation(tx, &conversation)?;
Self::log_applied(tx, record, SyncOp::Create, origin, origin_seq)?;
Ok(ApplyOutcome::Created)
}
(SyncOp::Update, SyncNodeType::Conversation) => {
let conversation: Conversation = parse_record_data(record, "conversation update")?;
Self::insert_conversation(tx, &conversation)?;
Self::log_applied(tx, record, SyncOp::Update, origin, origin_seq)?;
Ok(ApplyOutcome::Updated)
}
(SyncOp::Delete, SyncNodeType::Conversation) => {
if !Self::row_exists(tx, "conversations", &record.node_id)? {
return Ok(ApplyOutcome::Skipped(SkipReason::AlreadyPresent));
}
tx.execute(
"DELETE FROM conversations WHERE id = ?1",
params![record.node_id],
)?;
tx.execute(
"DELETE FROM relations WHERE to_id = ?1",
params![record.node_id],
)?;
Self::log_applied(tx, record, SyncOp::Delete, origin, origin_seq)?;
Ok(ApplyOutcome::Deleted)
}
(SyncOp::Create, SyncNodeType::Entity) => {
let entity: Entity = parse_record_data(record, "entity create")?;
if Self::row_exists(tx, "entities", &entity.id.to_string())? {
return Ok(ApplyOutcome::Skipped(SkipReason::AlreadyPresent));
}
Self::insert_entity(tx, &entity)?;
Self::log_applied(tx, record, SyncOp::Create, origin, origin_seq)?;
Ok(ApplyOutcome::Created)
}
(SyncOp::Update, SyncNodeType::Entity) => {
let entity: Entity = parse_record_data(record, "entity update")?;
Self::insert_entity(tx, &entity)?;
Self::log_applied(tx, record, SyncOp::Update, origin, origin_seq)?;
Ok(ApplyOutcome::Updated)
}
(SyncOp::Delete, SyncNodeType::Entity) => {
if !Self::row_exists(tx, "entities", &record.node_id)? {
return Ok(ApplyOutcome::Skipped(SkipReason::AlreadyPresent));
}
tx.execute(
"DELETE FROM entities WHERE id = ?1",
params![record.node_id],
)?;
tx.execute(
"DELETE FROM relations WHERE to_id = ?1",
params![record.node_id],
)?;
Self::log_applied(tx, record, SyncOp::Delete, origin, origin_seq)?;
Ok(ApplyOutcome::Deleted)
}
(SyncOp::Create, SyncNodeType::Distillation) => {
let entry: Distillation = parse_record_data(record, "distillation create")?;
let existing: Option<i64> = tx
.query_row(
"SELECT 1 FROM distillation_log WHERE memory_id = ?1",
params![record.node_id],
|r| r.get(0),
)
.optional()?;
if existing.is_some() {
return Ok(ApplyOutcome::Skipped(SkipReason::AlreadyPresent));
}
Self::insert_distillation(tx, &entry)?;
Self::log_applied(tx, record, SyncOp::Create, origin, origin_seq)?;
Ok(ApplyOutcome::Created)
}
(SyncOp::Update, SyncNodeType::Distillation) => {
Ok(ApplyOutcome::Skipped(SkipReason::Unsupported))
}
(SyncOp::Delete, SyncNodeType::Distillation) => {
let existing: Option<i64> = tx
.query_row(
"SELECT 1 FROM distillation_log WHERE memory_id = ?1",
params![record.node_id],
|r| r.get(0),
)
.optional()?;
if existing.is_none() {
return Ok(ApplyOutcome::Skipped(SkipReason::AlreadyPresent));
}
tx.execute(
"DELETE FROM distillation_log WHERE memory_id = ?1",
params![record.node_id],
)?;
Self::log_applied(tx, record, SyncOp::Delete, origin, origin_seq)?;
Ok(ApplyOutcome::Deleted)
}
(SyncOp::Create, SyncNodeType::Relation) => {
let relation: Relation = parse_record_data(record, "relation create")?;
if !Self::relation_endpoints_exist(tx, &relation)? {
return Ok(ApplyOutcome::Skipped(SkipReason::DanglingEndpoint));
}
Self::insert_relation(tx, &relation)?;
Self::log_applied(tx, record, SyncOp::Create, origin, origin_seq)?;
Ok(ApplyOutcome::Created)
}
(SyncOp::Update, SyncNodeType::Relation) => {
Ok(ApplyOutcome::Skipped(SkipReason::Unsupported))
}
(SyncOp::Delete, SyncNodeType::Relation) => {
let Some(relation) = record
.data
.as_ref()
.and_then(|d| serde_json::from_value::<Relation>(d.clone()).ok())
else {
return Ok(ApplyOutcome::Skipped(SkipReason::Unsupported));
};
let existing: Option<i64> = tx
.query_row(
"SELECT 1 FROM relations WHERE from_id = ?1 AND to_id = ?2 AND relation_type = ?3",
params![
relation.from_id.to_string(),
relation.to_id.to_string(),
relation_type_to_str(relation.relation_type),
],
|r| r.get(0),
)
.optional()?;
if existing.is_none() {
return Ok(ApplyOutcome::Skipped(SkipReason::AlreadyPresent));
}
tx.execute(
"DELETE FROM relations WHERE from_id = ?1 AND to_id = ?2 AND relation_type = ?3",
params![
relation.from_id.to_string(),
relation.to_id.to_string(),
relation_type_to_str(relation.relation_type),
],
)?;
Self::log_applied(tx, record, SyncOp::Delete, origin, origin_seq)?;
Ok(ApplyOutcome::Deleted)
}
}
}
fn log_applied(
tx: &Transaction<'_>,
record: &SyncRecord,
op: SyncOp,
origin: &str,
origin_seq: i64,
) -> Result<()> {
let data = record
.data
.as_ref()
.map(serde_json::to_string)
.transpose()?;
Self::append_sync_log(
tx,
op,
record.node_type,
&record.node_id,
data.as_deref(),
origin,
Some(origin_seq),
)
}
fn memory_exists(tx: &Transaction<'_>, id: Uuid) -> Result<bool> {
Self::row_exists(tx, "memories", &id.to_string())
}
fn row_exists(tx: &Transaction<'_>, table: &str, id: &str) -> Result<bool> {
let hit: Option<i64> = tx
.query_row(
&format!("SELECT 1 FROM {table} WHERE id = ?1"),
params![id],
|r| r.get(0),
)
.optional()?;
Ok(hit.is_some())
}
fn existing_lww_key(
tx: &Transaction<'_>,
id: Uuid,
) -> Result<Option<(DateTime<Utc>, u32, String)>> {
tx.query_row(
"SELECT updated_at, access_count, machine_id FROM memories WHERE id = ?1",
params![id.to_string()],
|r| {
Ok((
parse_dt(&r.get::<_, String>(0)?)?,
r.get::<_, i64>(1)? as u32,
r.get::<_, String>(2)?,
))
},
)
.optional()
.map_err(Into::into)
}
fn stored_session_at(tx: &Transaction<'_>, id: Uuid) -> Result<Option<DateTime<Utc>>> {
let stored: Option<Option<String>> = tx
.query_row(
"SELECT session_at FROM memories WHERE id = ?1",
params![id.to_string()],
|r| r.get(0),
)
.optional()?;
Ok(stored.flatten().map(|s| parse_dt(&s)).transpose()?)
}
fn fill_session_at(tx: &Transaction<'_>, incoming: &Memory) -> Result<()> {
if let Some(at) = incoming.session_at {
tx.execute(
"UPDATE memories SET session_at = ?2 WHERE id = ?1 AND session_at IS NULL",
params![incoming.id.to_string(), at.to_rfc3339()],
)?;
}
Ok(())
}
fn insert_conversation(tx: &Transaction<'_>, conversation: &Conversation) -> Result<()> {
tx.execute(
"INSERT OR REPLACE INTO conversations (id, source, machine_id, started_at, project_path) \
VALUES (?1,?2,?3,?4,?5)",
params![
conversation.id.to_string(),
conversation.source,
conversation.machine_id,
conversation.started_at.to_rfc3339(),
conversation.project_path,
],
)?;
Ok(())
}
fn insert_entity(tx: &Transaction<'_>, entity: &Entity) -> Result<()> {
let embedding = if entity.embedding.is_empty() {
None
} else {
Some(embedding_to_blob(&entity.embedding))
};
tx.execute(
"INSERT OR REPLACE INTO entities (id, name, entity_type, embedding, aliases) \
VALUES (?1,?2,?3,?4,?5)",
params![
entity.id.to_string(),
entity.name,
entity.entity_type,
embedding,
serde_json::to_string(&entity.aliases)?,
],
)?;
Ok(())
}
fn insert_relation(tx: &Transaction<'_>, relation: &Relation) -> Result<()> {
tx.execute(
"INSERT INTO relations (from_id, to_id, relation_type, strength, context, created_at) \
VALUES (?1,?2,?3,?4,?5,?6) \
ON CONFLICT(from_id, to_id, relation_type) DO UPDATE SET \
strength = excluded.strength, context = excluded.context",
params![
relation.from_id.to_string(),
relation.to_id.to_string(),
relation_type_to_str(relation.relation_type),
relation.strength as f64,
relation.context,
Utc::now().to_rfc3339(),
],
)?;
Ok(())
}
fn relation_endpoints_exist(tx: &Transaction<'_>, relation: &Relation) -> Result<bool> {
if !Self::memory_exists(tx, relation.from_id)? {
return Ok(false);
}
let target_table = match relation.relation_type {
RelationType::Mentions => "entities",
RelationType::DerivedFrom => "conversations",
_ => "memories",
};
Self::row_exists(tx, target_table, &relation.to_id.to_string())
}
fn sync_entries(&self, after_seq: u64, limit: Option<usize>) -> Result<Vec<SyncEntry>> {
let conn = self.reader();
let limit_clause = match limit {
Some(n) => format!(" LIMIT {n}"),
None => String::new(),
};
let mut stmt = conn.prepare(&format!(
"SELECT id, local_seq, origin_machine_id, origin_seq, op, node_type, node_id, \
timestamp, data FROM sync_log WHERE local_seq > ?1 ORDER BY local_seq{limit_clause}"
))?;
let rows = stmt.query_map(params![after_seq as i64], |r| {
let op = match r.get::<_, String>(4)?.as_str() {
"update" => SyncOp::Update,
"delete" => SyncOp::Delete,
_ => SyncOp::Create,
};
let node_type = match r.get::<_, String>(5)?.as_str() {
"entity" => SyncNodeType::Entity,
"conversation" => SyncNodeType::Conversation,
"relation" => SyncNodeType::Relation,
"distillation" => SyncNodeType::Distillation,
_ => SyncNodeType::Memory,
};
Ok(SyncEntry {
id: r.get(0)?,
local_seq: r.get(1)?,
origin_machine_id: r.get(2)?,
origin_seq: r.get(3)?,
op,
node_type,
node_id: r.get(6)?,
timestamp: parse_dt(&r.get::<_, String>(7)?)?,
data: r.get::<_, Option<String>>(8)?.filter(|d| !d.is_empty()),
})
})?;
Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
}
}
impl StoreAdmin for SqliteStore {
fn register_machine(&self, identity: &MachineIdentity) -> Result<()> {
let conn = self.conn();
conn.execute(
"INSERT INTO machines (id, name) VALUES (?1, ?2) \
ON CONFLICT(id) DO UPDATE SET name = excluded.name",
params![identity.id, identity.name],
)?;
let backfilled = conn.execute(
"UPDATE memories SET machine_id = ?1 WHERE machine_id = ''",
params![identity.id],
)?;
if backfilled > 0 {
tracing::info!("backfilled machine_id on {backfilled} existing memories");
}
Ok(())
}
fn get_all_machines(&self) -> Result<HashMap<String, String>> {
let conn = self.reader();
let mut stmt = conn.prepare("SELECT id, COALESCE(name, '') FROM machines")?;
let rows = stmt.query_map([], |r| Ok((r.get(0)?, r.get(1)?)))?;
Ok(rows.collect::<rusqlite::Result<HashMap<_, _>>>()?)
}
fn diagnostic(&self) -> Result<String> {
let conn = self.reader();
let total: i64 = conn.query_row("SELECT COUNT(*) FROM memories", [], |r| r.get(0))?;
let with_emb: i64 = conn.query_row(
"SELECT COUNT(*) FROM memories WHERE embedding IS NOT NULL",
[],
|r| r.get(0),
)?;
let max_seq: i64 = conn.query_row(
"SELECT COALESCE(MAX(local_seq), 0) FROM sync_log",
[],
|r| r.get(0),
)?;
Ok(format!(
"backend: sqlite\ntotal memories: {total}\nwith embeddings: {with_emb}\nmax sync seq: {max_seq}"
))
}
fn find_or_create_entity(&self, name: &str, entity_type: &str) -> Result<Entity> {
if let Some(existing) = self.find_entity_by_name(name)? {
return Ok(existing);
}
let entity = Entity::new(name.to_string(), entity_type.to_string());
self.store_entity(&entity)?;
Ok(entity)
}
fn memory_content_exists(&self, content_prefix: &str) -> Result<bool> {
SqliteStore::memory_content_exists(self, content_prefix)
}
fn store_memory_if_absent(&self, memory: &Memory) -> Result<bool> {
self.with_txn(|tx| {
if Self::memory_exists(tx, memory.id)? {
return Ok(false);
}
self.store_memory_on_tx(tx, memory)
})
}
fn store_conversation_if_absent(&self, conversation: &Conversation) -> Result<bool> {
let data = serde_json::to_string(conversation).ok();
self.with_txn(|tx| {
let id = conversation.id.to_string();
if Self::row_exists(tx, "conversations", &id)? {
return Ok(false);
}
Self::insert_conversation(tx, conversation)?;
Self::append_sync_log(
tx,
SyncOp::Create,
SyncNodeType::Conversation,
&id,
data.as_deref(),
&self.machine_id,
None,
)?;
Ok(true)
})
}
fn memory_id_taken(&self, id: Uuid) -> Result<bool> {
let conn = self.reader();
let hit: Option<i64> = conn
.query_row(
"SELECT 1 FROM memories WHERE id = ?1 \
UNION ALL SELECT 1 FROM tombstones WHERE node_id = ?1 LIMIT 1",
params![id.to_string()],
|r| r.get(0),
)
.optional()?;
Ok(hit.is_some())
}
fn is_file_changed(&self, file_path: &str, file_hash: &str) -> Result<bool> {
let conn = self.reader();
let stored: Option<String> = conn
.query_row(
"SELECT file_hash FROM ingest_log WHERE file_path = ?1",
params![file_path],
|r| r.get(0),
)
.optional()?;
Ok(match stored {
Some(hash) => hash != file_hash,
None => true,
})
}
fn mark_ingested(
&self,
file_path: &str,
file_hash: &str,
memory_count: usize,
source: &str,
) -> Result<()> {
let conn = self.conn();
conn.execute(
"INSERT INTO ingest_log (file_path, file_hash, ingested_at, memory_count, source) \
VALUES (?1,?2,?3,?4,?5) \
ON CONFLICT(file_path) DO UPDATE SET file_hash = excluded.file_hash, \
ingested_at = excluded.ingested_at, memory_count = excluded.memory_count, \
source = excluded.source, content_hash = NULL",
params![
file_path,
file_hash,
Utc::now().to_rfc3339(),
memory_count as i64,
source,
],
)?;
Ok(())
}
fn ingested_file_count(&self) -> Result<usize> {
let conn = self.reader();
let count: i64 = conn.query_row("SELECT COUNT(*) FROM ingest_log", [], |r| r.get(0))?;
Ok(count as usize)
}
fn clear_ingest_log(&self) -> Result<usize> {
let conn = self.conn();
let count = conn.execute("DELETE FROM ingest_log", [])?;
Ok(count)
}
fn ingest_log_entry(&self, file_path: &str) -> Result<Option<IngestLogEntry>> {
let conn = self.reader();
conn.query_row(
&format!("SELECT {INGEST_LOG_COLS} FROM ingest_log WHERE file_path = ?1"),
params![file_path],
row_to_ingest_log_entry,
)
.optional()
.map_err(Into::into)
}
fn ingested_session(
&self,
session_key: &str,
content_hash: &str,
) -> Result<Option<IngestLogEntry>> {
let conn = self.reader();
conn.query_row(
&format!(
"SELECT {INGEST_LOG_COLS} FROM ingest_log \
WHERE session_key = ?1 AND content_hash = ?2 \
ORDER BY legacy_ids DESC LIMIT 1"
),
params![session_key, content_hash],
row_to_ingest_log_entry,
)
.optional()
.map_err(Into::into)
}
fn legacy_ingest_entries_with_stem(&self, stem: &str) -> Result<Vec<IngestLogEntry>> {
let conn = self.reader();
let mut stmt = conn.prepare(&format!(
"SELECT {INGEST_LOG_COLS} FROM ingest_log \
WHERE legacy_ids = 1 AND content_hash IS NULL AND instr(file_path, ?1) > 0"
))?;
let rows = stmt
.query_map(params![stem], row_to_ingest_log_entry)?
.collect::<rusqlite::Result<Vec<_>>>()?;
Ok(rows
.into_iter()
.filter(|e| {
Path::new(&e.file_path)
.file_stem()
.is_some_and(|s| s.to_string_lossy() == stem)
})
.collect())
}
fn upgrade_ingest_hash(
&self,
file_path: &str,
content_hash: &str,
session_key: Option<&str>,
) -> Result<()> {
let conn = self.conn();
conn.execute(
"UPDATE ingest_log SET file_hash = ?2, content_hash = ?2, session_key = ?3 \
WHERE file_path = ?1",
params![file_path, content_hash, session_key],
)?;
Ok(())
}
fn record_ingested(&self, record: &IngestRecord<'_>) -> Result<()> {
let conn = self.conn();
conn.execute(
"INSERT INTO ingest_log (file_path, file_hash, ingested_at, memory_count, source, \
session_key, content_hash, legacy_ids) VALUES (?1,?2,?3,?4,?5,?6,?2,?7) \
ON CONFLICT(file_path) DO UPDATE SET file_hash = excluded.file_hash, \
content_hash = excluded.content_hash, session_key = excluded.session_key, \
ingested_at = excluded.ingested_at, memory_count = excluded.memory_count, \
source = excluded.source, legacy_ids = MAX(legacy_ids, excluded.legacy_ids)",
params![
record.file_path,
record.content_hash,
Utc::now().to_rfc3339(),
record.memory_count as i64,
record.source,
record.session_key,
record.legacy_ids,
],
)?;
Ok(())
}
fn clear_reparseable_ingest_log(&self) -> Result<usize> {
let conn = self.conn();
let count = conn.execute("DELETE FROM ingest_log WHERE legacy_ids = 0", [])?;
Ok(count)
}
fn delete_memories_by_source(&self, source: &str) -> Result<usize> {
self.purge_memories_where("source = ?1", source)
}
fn delete_memories_by_source_id(&self, source_id: &str) -> Result<usize> {
self.purge_memories_where("source_id = ?1", source_id)
}
fn rebuild_vector_index(&self) -> Result<()> {
Ok(())
}
fn backfill_project_paths(&self) -> Result<u64> {
let from_source_id = self.backfill_project_paths_from_source_id()?;
let from_relations: Vec<(Uuid, String)> = {
let conn = self.reader();
let mut stmt = conn.prepare(
"SELECT m.id, (SELECT c.project_path FROM conversations c \
JOIN relations r ON r.to_id = c.id \
WHERE r.from_id = m.id AND r.relation_type = 'derived_from' \
AND c.project_path IS NOT NULL AND c.project_path <> '' \
LIMIT 1) \
FROM memories m WHERE (m.project_path IS NULL OR m.project_path = '')",
)?;
let rows = stmt.query_map([], |r| {
Ok((
parse_uuid(&r.get::<_, String>(0)?)?,
r.get::<_, Option<String>>(1)?,
))
})?;
rows.filter_map(|row| match row {
Ok((id, Some(project))) => Some(Ok((id, project))),
Ok((_, None)) => None,
Err(e) => Some(Err(e)),
})
.collect::<rusqlite::Result<Vec<_>>>()?
};
Ok(from_source_id + self.refile_all(&from_relations)?)
}
fn backfill_sync_log(&self) -> Result<u64> {
let mut logged = 0u64;
let memory_ids: Vec<Uuid> = {
let conn = self.reader();
let mut stmt = conn.prepare(
"SELECT id FROM memories WHERE id NOT IN ( \
SELECT node_id FROM sync_log WHERE node_type = 'memory')",
)?;
let rows = stmt.query_map([], |r| parse_uuid(&r.get::<_, String>(0)?))?;
rows.collect::<rusqlite::Result<Vec<_>>>()?
};
for id in memory_ids {
let Some(memory) = self.get_memory_with_embedding(id)? else {
continue;
};
let data = serde_json::to_string(&memory).ok();
self.with_txn(|tx| {
Self::append_sync_log(
tx,
SyncOp::Create,
SyncNodeType::Memory,
&id.to_string(),
data.as_deref(),
&memory.machine_id,
None,
)
})?;
logged += 1;
}
let conversations: Vec<Conversation> = {
let conn = self.reader();
let mut stmt = conn.prepare(
"SELECT id, source, machine_id, started_at, project_path FROM conversations \
WHERE id NOT IN ( \
SELECT node_id FROM sync_log WHERE node_type = 'conversation')",
)?;
let rows = stmt.query_map([], |r| {
Ok(Conversation {
id: parse_uuid(&r.get::<_, String>(0)?)?,
source: r.get(1)?,
machine_id: r.get(2)?,
started_at: parse_dt(&r.get::<_, String>(3)?)?,
project_path: r.get(4)?,
})
})?;
rows.collect::<rusqlite::Result<Vec<_>>>()?
};
for conversation in conversations {
let data = serde_json::to_string(&conversation).ok();
let machine_id = conversation.machine_id.clone();
self.with_txn(|tx| {
Self::append_sync_log(
tx,
SyncOp::Create,
SyncNodeType::Conversation,
&conversation.id.to_string(),
data.as_deref(),
&machine_id,
None,
)
})?;
logged += 1;
}
let entities: Vec<Entity> = {
let conn = self.reader();
let mut stmt = conn.prepare(
"SELECT id, name, entity_type, embedding, aliases FROM entities \
WHERE id NOT IN ( \
SELECT node_id FROM sync_log WHERE node_type = 'entity')",
)?;
let rows = stmt.query_map([], row_to_entity)?;
rows.collect::<rusqlite::Result<Vec<_>>>()?
};
for entity in entities {
let data = serde_json::to_string(&entity).ok();
self.with_txn(|tx| {
Self::append_sync_log(
tx,
SyncOp::Create,
SyncNodeType::Entity,
&entity.id.to_string(),
data.as_deref(),
&self.machine_id,
None,
)
})?;
logged += 1;
}
let relations: Vec<Relation> = {
let conn = self.reader();
let mut stmt = conn.prepare(
"SELECT from_id, to_id, relation_type, strength, context FROM relations \
WHERE from_id || ':' || to_id || ':' || relation_type NOT IN ( \
SELECT node_id FROM sync_log WHERE node_type = 'relation')",
)?;
let rows = stmt.query_map([], |r| {
Ok(Relation {
from_id: parse_uuid(&r.get::<_, String>(0)?)?,
to_id: parse_uuid(&r.get::<_, String>(1)?)?,
relation_type: parse_relation_type(&r.get::<_, String>(2)?)?,
strength: r.get::<_, f64>(3)? as f32,
context: r.get(4)?,
})
})?;
rows.collect::<rusqlite::Result<Vec<_>>>()?
};
for relation in relations {
let node_id = format!(
"{}:{}:{}",
relation.from_id,
relation.to_id,
relation_type_to_str(relation.relation_type)
);
let data = serde_json::to_string(&relation).ok();
self.with_txn(|tx| {
Self::append_sync_log(
tx,
SyncOp::Create,
SyncNodeType::Relation,
&node_id,
data.as_deref(),
&self.machine_id,
None,
)
})?;
logged += 1;
}
let distillations: Vec<Distillation> = {
let conn = self.reader();
let mut stmt = conn.prepare(
"SELECT memory_id, distilled_id, distilled_at, model, status \
FROM distillation_log WHERE memory_id NOT IN ( \
SELECT node_id FROM sync_log WHERE node_type = 'distillation')",
)?;
let rows = stmt.query_map([], |r| {
Ok(Distillation {
memory_id: parse_uuid(&r.get::<_, String>(0)?)?,
distilled_id: r
.get::<_, Option<String>>(1)?
.map(|v| parse_uuid(&v))
.transpose()?,
distilled_at: r
.get::<_, Option<String>>(2)?
.map(|v| parse_dt(&v))
.transpose()?,
model: r.get(3)?,
status: r.get(4)?,
})
})?;
rows.collect::<rusqlite::Result<Vec<_>>>()?
};
for entry in distillations {
let data = serde_json::to_string(&entry).ok();
self.with_txn(|tx| {
Self::append_sync_log(
tx,
SyncOp::Create,
SyncNodeType::Distillation,
&entry.memory_id.to_string(),
data.as_deref(),
&self.machine_id,
None,
)
})?;
logged += 1;
}
Ok(logged)
}
fn backfill_tombstones(&self) -> Result<u64> {
let node_ids: Vec<String> = {
let conn = self.reader();
let mut stmt = conn.prepare(
"SELECT node_id FROM tombstones WHERE node_type = 'memory' \
AND node_id NOT IN ( \
SELECT node_id FROM sync_log \
WHERE node_type = 'memory' AND op = 'delete')",
)?;
let rows = stmt.query_map([], |r| r.get::<_, String>(0))?;
rows.collect::<rusqlite::Result<Vec<_>>>()?
};
let mut logged = 0u64;
for node_id in node_ids {
self.with_txn(|tx| {
Self::append_sync_log(
tx,
SyncOp::Delete,
SyncNodeType::Memory,
&node_id,
None,
&self.machine_id,
None,
)
})?;
logged += 1;
}
Ok(logged)
}
fn prune_dangling_relations(&self) -> Result<u64> {
let dangling: Vec<Relation> = {
let conn = self.reader();
let mut stmt = conn.prepare(
"SELECT from_id, to_id, relation_type, strength, context FROM relations \
WHERE from_id NOT IN (SELECT id FROM memories) \
OR (relation_type = 'mentions' \
AND to_id NOT IN (SELECT id FROM entities)) \
OR (relation_type = 'derived_from' \
AND to_id NOT IN (SELECT id FROM conversations)) \
OR (relation_type NOT IN ('mentions', 'derived_from') \
AND to_id NOT IN (SELECT id FROM memories))",
)?;
let rows = stmt.query_map([], |r| {
Ok(Relation {
from_id: parse_uuid(&r.get::<_, String>(0)?)?,
to_id: parse_uuid(&r.get::<_, String>(1)?)?,
relation_type: parse_relation_type(&r.get::<_, String>(2)?)?,
strength: r.get::<_, f64>(3)? as f32,
context: r.get(4)?,
})
})?;
rows.collect::<rusqlite::Result<Vec<_>>>()?
};
let mut pruned = 0u64;
for relation in dangling {
if self.with_txn(|tx| self.delete_relation_on_tx(tx, &relation))? {
pruned += 1;
}
}
Ok(pruned)
}
fn compact_sync_log(&self, ops: &[SyncOp], dry_run: bool) -> Result<CompactStats> {
if ops.is_empty() {
return Ok(CompactStats::default());
}
let op_strs: Vec<&'static str> = ops.iter().map(|op| sync_op_to_str(*op)).collect();
let in_list = op_strs.iter().map(|_| "?").collect::<Vec<_>>().join(",");
let mut conn = self.conn();
conn.execute_batch("PRAGMA cache_size = -262144;")?;
conn.execute_batch(
"CREATE INDEX IF NOT EXISTS idx_sync_log_type_id_op_seq \
ON sync_log(node_type, node_id, op, local_seq);",
)?;
let max_before: i64 = conn.query_row(
"SELECT COALESCE(MAX(local_seq), 0) FROM sync_log",
[],
|r| r.get(0),
)?;
conn.execute_batch("DROP TABLE IF EXISTS compact_victims;")?;
conn.execute(
&format!(
"CREATE TEMP TABLE compact_victims AS \
SELECT s.local_seq, s.node_type, s.node_id, s.op \
FROM sync_log s \
JOIN (SELECT node_type, node_id, op, MAX(local_seq) AS keep_seq \
FROM sync_log \
WHERE op IN ({in_list}) \
GROUP BY node_type, node_id, op) k \
ON s.node_type = k.node_type AND s.node_id = k.node_id AND s.op = k.op \
WHERE s.op IN ({in_list}) AND s.local_seq < k.keep_seq"
),
rusqlite::params_from_iter(op_strs.iter().chain(op_strs.iter())),
)?;
conn.execute_batch("CREATE INDEX idx_compact_victims_seq ON compact_victims(local_seq);")?;
let groups_with_duplicates: i64 = conn.query_row(
"SELECT COUNT(*) FROM (SELECT DISTINCT node_type, node_id, op FROM compact_victims)",
[],
|r| r.get(0),
)?;
let victim_count: i64 =
conn.query_row("SELECT COUNT(*) FROM compact_victims", [], |r| r.get(0))?;
let mut rows_deleted: i64 = 0;
if !dry_run {
let mut cursor: i64 = 0;
loop {
let chunk: Vec<i64> = {
let mut stmt = conn.prepare(
"SELECT local_seq FROM compact_victims WHERE local_seq > ?1 \
ORDER BY local_seq LIMIT 20000",
)?;
let rows = stmt.query_map(params![cursor], |r| r.get::<_, i64>(0))?;
rows.collect::<rusqlite::Result<Vec<_>>>()?
};
if chunk.is_empty() {
break;
}
cursor = *chunk.last().expect("just checked non-empty");
let placeholders = chunk.iter().map(|_| "?").collect::<Vec<_>>().join(",");
let tx = conn.transaction()?;
let n = tx.execute(
&format!("DELETE FROM sync_log WHERE local_seq IN ({placeholders})"),
rusqlite::params_from_iter(chunk.iter()),
)?;
tx.commit()?;
rows_deleted += n as i64;
}
}
conn.execute_batch("DROP TABLE IF EXISTS compact_victims;")?;
let max_after: i64 = conn.query_row(
"SELECT COALESCE(MAX(local_seq), 0) FROM sync_log",
[],
|r| r.get(0),
)?;
if max_after != max_before {
anyhow::bail!(
"compact_sync_log changed MAX(local_seq) from {max_before} to {max_after}; \
a peer's sync_state cursor may now be invalid, aborting rather than \
silently risking replication loss"
);
}
Ok(CompactStats {
groups_with_duplicates: groups_with_duplicates as u64,
rows_deleted: if dry_run {
victim_count as u64
} else {
rows_deleted as u64
},
max_local_seq_before: max_before as u64,
max_local_seq_after: max_after as u64,
})
}
fn memories_by_project_path(&self, project_path: &str) -> Result<Vec<Memory>> {
let conn = self.reader();
let mut stmt = conn.prepare(&format!(
"SELECT {MEMORY_COLS} FROM memories \
WHERE project_path = ?1 AND tier = 'hot' AND superseded_by IS NULL \
ORDER BY created_at DESC LIMIT 20"
))?;
let rows = stmt.query_map(params![project_path], |r| row_to_memory(r, false))?;
Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
}
fn distinct_project_paths(&self) -> Result<Vec<(String, u64)>> {
let conn = self.reader();
let mut stmt = conn.prepare(
"SELECT project_path, COUNT(*) FROM memories \
WHERE project_path IS NOT NULL AND project_path <> '' \
AND tier = 'hot' AND superseded_by IS NULL \
GROUP BY project_path ORDER BY COUNT(*) DESC",
)?;
let rows = stmt.query_map([], |r| {
let count: i64 = r.get(1)?;
Ok((r.get::<_, String>(0)?, count as u64))
})?;
Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
}
fn all_memories_with_embeddings(&self) -> Result<Vec<Memory>> {
let conn = self.reader();
let mut stmt = conn.prepare(&format!("SELECT {MEMORY_COLS}, embedding FROM memories"))?;
let rows = stmt.query_map([], |r| row_to_memory(r, true))?;
Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
}
fn consolidation_stats(&self) -> Result<ConsolidationStats> {
let conn = self.reader();
let count = |sql: &str| -> Result<u64> {
let n: i64 = conn.query_row(sql, [], |r| r.get(0))?;
Ok(n as u64)
};
Ok(ConsolidationStats {
distilled_live: count(
"SELECT COUNT(*) FROM memories WHERE source = 'consolidation' \
AND tier = 'hot' AND superseded_by IS NULL",
)?,
raw_pending: count(&format!(
"SELECT COUNT(*) FROM memories WHERE {UNCONSOLIDATED_PREDICATE}"
))?,
raw_retired: count(
"SELECT COUNT(*) FROM distillation_log dl JOIN memories m ON m.id = dl.memory_id \
WHERE m.tier = 'archived'",
)?,
raw_retire_pending: count(&format!(
"SELECT COUNT(*) FROM distillation_log dl JOIN memories m ON m.id = dl.memory_id \
WHERE {RETIRABLE_PREDICATE}"
))?,
})
}
fn unconsolidated_memories(&self, limit: usize) -> Result<Vec<Memory>> {
let conn = self.reader();
let mut stmt = conn.prepare(&format!(
"SELECT {MEMORY_COLS} FROM memories WHERE {UNCONSOLIDATED_PREDICATE} \
ORDER BY source_id, created_at LIMIT ?1"
))?;
let rows = stmt.query_map(params![limit as i64], |r| row_to_memory(r, false))?;
Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
}
fn mark_consolidated(&self, raw_id: Uuid, distilled_id: Uuid, model: &str) -> Result<()> {
self.with_txn(|tx| self.mark_consolidated_on_tx(tx, raw_id, distilled_id, model))
}
fn retire_consolidated_raws(&self) -> Result<usize> {
const BATCH: usize = 500;
let ids: Vec<Uuid> = {
let conn = self.reader();
let mut stmt = conn.prepare(&format!(
"SELECT m.id FROM distillation_log dl JOIN memories m ON m.id = dl.memory_id \
WHERE {RETIRABLE_PREDICATE} ORDER BY m.created_at"
))?;
let rows = stmt.query_map([], |r| parse_uuid(&r.get::<_, String>(0)?))?;
rows.collect::<rusqlite::Result<Vec<_>>>()?
};
let mut retired = 0usize;
for chunk in ids.chunks(BATCH) {
retired += self.with_txn(|tx| {
let mut n = 0usize;
for id in chunk {
if self.archive_on_tx(tx, *id)? {
n += 1;
}
}
Ok(n)
})?;
}
Ok(retired)
}
fn memories_created_between(
&self,
start: &DateTime<Utc>,
end: &DateTime<Utc>,
) -> Result<Vec<Memory>> {
let conn = self.reader();
let mut stmt = conn.prepare(&format!(
"SELECT {MEMORY_COLS} FROM memories WHERE created_at >= ?1 AND created_at < ?2 \
ORDER BY created_at ASC"
))?;
let rows = stmt.query_map(params![start.to_rfc3339(), end.to_rfc3339()], |r| {
row_to_memory(r, false)
})?;
Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
}
fn sync_log_since(&self, after_seq: u64) -> Result<Vec<SyncEntry>> {
self.sync_entries(after_seq, None)
}
fn sync_log_page(&self, after_seq: u64, limit: Option<usize>) -> Result<Vec<SyncEntry>> {
self.sync_entries(after_seq, limit)
}
fn max_sync_seq(&self) -> Result<u64> {
SqliteStore::max_sync_seq(self)
}
fn get_sync_state(&self, peer_id: &str) -> Result<Option<SyncState>> {
let conn = self.reader();
conn.query_row(
"SELECT peer_id, last_seq, last_sync_at FROM sync_state WHERE peer_id = ?1",
params![peer_id],
|r| {
Ok(SyncState {
peer_id: r.get(0)?,
last_seq: r.get::<_, i64>(1)? as u64,
last_sync_at: parse_dt(&r.get::<_, String>(2)?)?,
})
},
)
.optional()
.map_err(Into::into)
}
fn set_sync_state(&self, state: &SyncState) -> Result<()> {
let conn = self.conn();
conn.execute(
"INSERT OR REPLACE INTO sync_state (peer_id, last_seq, last_sync_at) VALUES (?1,?2,?3)",
params![
state.peer_id,
state.last_seq as i64,
state.last_sync_at.to_rfc3339(),
],
)?;
Ok(())
}
fn get_all_sync_states(&self) -> Result<Vec<SyncState>> {
let conn = self.reader();
let mut stmt = conn.prepare("SELECT peer_id, last_seq, last_sync_at FROM sync_state")?;
let rows = stmt.query_map([], |r| {
Ok(SyncState {
peer_id: r.get(0)?,
last_seq: r.get::<_, i64>(1)? as u64,
last_sync_at: parse_dt(&r.get::<_, String>(2)?)?,
})
})?;
Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
}
fn tombstone_exists(&self, id: Uuid) -> Result<bool> {
SqliteStore::tombstone_exists(self, id)
}
fn apply_record(&self, record: &SyncRecord) -> Result<ApplyOutcome> {
self.with_txn(|tx| self.apply_on_tx(tx, record))
}
fn apply_batch(&self, records: &[SyncRecord]) -> Result<Vec<ApplyOutcome>> {
self.with_txn(|tx| {
let mut outcomes = Vec::with_capacity(records.len());
for record in records {
outcomes.push(self.apply_on_tx(tx, record)?);
}
Ok(outcomes)
})
}
}
impl SqliteStore {
pub fn pending_distillation(&self) -> Result<PendingDistillation> {
let conn = self.reader();
let (count, oldest): (i64, Option<String>) = conn.query_row(
&format!(
"SELECT COUNT(*), MIN(created_at) FROM memories WHERE {UNCONSOLIDATED_PREDICATE}"
),
[],
|r| Ok((r.get(0)?, r.get(1)?)),
)?;
Ok(PendingDistillation {
count: count as u64,
oldest_created_at: oldest.as_deref().map(parse_dt).transpose()?,
})
}
}
impl SqliteStore {
fn backfill_project_paths_from_source_id(&self) -> Result<u64> {
let untagged: Vec<(Uuid, String)> = {
let conn = self.reader();
let mut stmt = conn.prepare(
"SELECT id, source_id FROM memories \
WHERE (project_path IS NULL OR project_path = '') \
AND source_id LIKE '/%'",
)?;
let rows = stmt.query_map([], |r| {
Ok((parse_uuid(&r.get::<_, String>(0)?)?, r.get::<_, String>(1)?))
})?;
rows.collect::<rusqlite::Result<Vec<_>>>()?
};
let derived: Vec<(Uuid, String)> = untagged
.into_iter()
.filter_map(|(id, source_id)| {
project_path::from_transcript(&source_id).map(|slug| (id, slug))
})
.collect();
self.refile_all(&derived)
}
fn refile_all(&self, rows: &[(Uuid, String)]) -> Result<u64> {
const BATCH: usize = 500;
let mut changed = 0u64;
for chunk in rows.chunks(BATCH) {
changed += self.with_txn(|tx| {
let mut n = 0u64;
for (id, project) in chunk {
if self.refile_on_tx(tx, *id, project)? {
n += 1;
}
}
Ok(n)
})?;
}
Ok(changed)
}
pub fn restore_explicit_memories(&self) -> Result<ExplicitRestoreStats> {
const BATCH: usize = 500;
let marked: Vec<(Uuid, Uuid)> = {
let conn = self.reader();
let mut stmt = conn.prepare(concat!(
"SELECT dl.memory_id, dl.distilled_id FROM distillation_log dl \
JOIN memories m ON m.id = dl.memory_id WHERE m.source IN (",
explicit_sources_list!(),
") ORDER BY m.created_at, m.id"
))?;
let rows = stmt.query_map([], |r| {
Ok((
parse_uuid(&r.get::<_, String>(0)?)?,
parse_uuid(&r.get::<_, String>(1)?)?,
))
})?;
rows.collect::<rusqlite::Result<Vec<_>>>()?
};
let mut all_explicit: Vec<(Uuid, Uuid)> = Vec::new();
{
let conn = self.reader();
let mut seen = std::collections::HashSet::new();
for (raw_id, distilled_id) in &marked {
if raw_id == distilled_id || !seen.insert(*distilled_id) {
continue;
}
let other: i64 = conn.query_row(
concat!(
"SELECT COUNT(*) FROM distillation_log dl \
JOIN memories m ON m.id = dl.memory_id \
WHERE dl.distilled_id = ?1 AND m.source NOT IN (",
explicit_sources_list!(),
")"
),
params![distilled_id.to_string()],
|r| r.get(0),
)?;
if other == 0 {
all_explicit.push((*distilled_id, *raw_id));
}
}
}
let mut stats = ExplicitRestoreStats::default();
for chunk in marked.chunks(BATCH) {
let (restored, dropped) = self.with_txn(|tx| {
let mut restored = 0u64;
let mut dropped = 0u64;
for (raw_id, _) in chunk {
if self.unarchive_on_tx(tx, *raw_id)? {
restored += 1;
}
if self.drop_marker_on_tx(tx, *raw_id)? {
dropped += 1;
}
}
Ok((restored, dropped))
})?;
stats.raws_restored += restored;
stats.markers_dropped += dropped;
}
for chunk in all_explicit.chunks(BATCH) {
stats.distillates_superseded += self.with_txn(|tx| {
let mut n = 0u64;
for (distilled_id, by) in chunk {
if self.supersede_on_tx(tx, *distilled_id, *by)? {
n += 1;
}
}
Ok(n)
})?;
}
Ok(stats)
}
fn unarchive_on_tx(&self, tx: &Transaction<'_>, memory_id: Uuid) -> Result<bool> {
let changed = tx.execute(
"UPDATE memories SET tier = 'hot', updated_at = ?2 \
WHERE id = ?1 AND tier = 'archived' AND archived_segment IS NULL \
AND retired_reason IS NULL",
params![memory_id.to_string(), Utc::now().to_rfc3339()],
)?;
if changed == 0 {
return Ok(false);
}
Self::append_outbox(tx, memory_id, "upsert")?;
if let Some(memory) = Self::memory_on_tx(tx, memory_id)? {
self.log_memory_update(tx, &memory)?;
}
Ok(true)
}
fn drop_marker_on_tx(&self, tx: &Transaction<'_>, raw_id: Uuid) -> Result<bool> {
let n = tx.execute(
"DELETE FROM distillation_log WHERE memory_id = ?1",
params![raw_id.to_string()],
)?;
if n == 0 {
return Ok(false);
}
Self::append_sync_log(
tx,
SyncOp::Delete,
SyncNodeType::Distillation,
&raw_id.to_string(),
None,
&self.machine_id,
None,
)?;
Ok(true)
}
pub(crate) fn mark_consolidated_on_tx(
&self,
tx: &Transaction<'_>,
raw_id: Uuid,
distilled_id: Uuid,
model: &str,
) -> Result<()> {
self.mark_distillation_on_tx(tx, raw_id, distilled_id, model, "legacy")
}
pub(super) fn mark_distillation_on_tx(
&self,
tx: &Transaction<'_>,
raw_id: Uuid,
distilled_id: Uuid,
model: &str,
status: &str,
) -> Result<()> {
let entry = Distillation {
memory_id: raw_id,
distilled_id: Some(distilled_id),
distilled_at: Some(Utc::now()),
model: Some(model.to_string()),
status: status.to_string(),
};
let data = serde_json::to_string(&entry).ok();
Self::insert_distillation(tx, &entry)?;
Self::append_sync_log(
tx,
SyncOp::Create,
SyncNodeType::Distillation,
&raw_id.to_string(),
data.as_deref(),
&self.machine_id,
None,
)
}
pub fn refile_nested_transcript_projects(&self) -> Result<u64> {
const NESTED: &str = "('subagents', 'agent-transcripts')";
let raws: Vec<(Uuid, String)> = {
let conn = self.reader();
let mut stmt = conn.prepare(&format!(
"SELECT id, source_id FROM memories \
WHERE project_path IN {NESTED} AND source <> 'consolidation'"
))?;
let rows = stmt.query_map([], |r| {
Ok((parse_uuid(&r.get::<_, String>(0)?)?, r.get::<_, String>(1)?))
})?;
rows.collect::<rusqlite::Result<Vec<_>>>()?
};
let raws: Vec<(Uuid, String)> = raws
.into_iter()
.filter_map(|(id, source_id)| {
project_path::from_transcript(&source_id).map(|s| (id, s))
})
.collect();
let mut changed = self.refile_all(&raws)?;
let distillates: Vec<(Uuid, Option<String>, String)> = {
let conn = self.reader();
let mut stmt = conn.prepare(&format!(
"SELECT d.id, \
(SELECT r.project_path FROM distillation_log dl \
JOIN memories r ON r.id = dl.memory_id \
WHERE dl.distilled_id = d.id AND dl.memory_id <> dl.distilled_id \
AND r.project_path IS NOT NULL AND r.project_path NOT IN {NESTED} \
GROUP BY r.project_path ORDER BY COUNT(*) DESC, r.project_path LIMIT 1), \
d.source_id \
FROM memories d \
WHERE d.source = 'consolidation' AND d.project_path IN {NESTED}"
))?;
let rows = stmt.query_map([], |r| {
Ok((
parse_uuid(&r.get::<_, String>(0)?)?,
r.get::<_, Option<String>>(1)?,
r.get::<_, String>(2)?,
))
})?;
rows.collect::<rusqlite::Result<Vec<_>>>()?
};
let distillates: Vec<(Uuid, String)> = distillates
.into_iter()
.filter_map(|(id, slug, source_id)| {
slug.or_else(|| project_path::from_transcript(&source_id))
.map(|s| (id, s))
})
.collect();
changed += self.refile_all(&distillates)?;
Ok(changed)
}
pub fn refile_stray_project_slugs(&self) -> Result<StraySlugReport> {
let slugs: Vec<(String, u64, bool, bool)> = {
let conn = self.reader();
let mut stmt = conn.prepare(concat!(
"SELECT project_path, \
SUM(tier = 'hot' AND superseded_by IS NULL), \
MAX(source IN (",
explicit_sources_list!(),
")), MAX(",
distillable_source!("source"),
") FROM memories \
WHERE project_path IS NOT NULL AND project_path <> '' \
GROUP BY project_path ORDER BY project_path"
))?;
let rows = stmt.query_map([], |r| {
Ok((
r.get::<_, String>(0)?,
r.get::<_, i64>(1)? as u64,
r.get::<_, i64>(2)? != 0,
r.get::<_, i64>(3)? != 0,
))
})?;
rows.collect::<rusqlite::Result<Vec<_>>>()?
};
let mut report = StraySlugReport::default();
for (stray, _, has_explicit, has_transcripts) in &slugs {
if !has_explicit || stray.starts_with('-') || stray.contains('/') {
continue;
}
let suffix = format!("-{}", project_path::encode(stray));
let mut projects: Vec<Vec<(&String, u64)>> = Vec::new();
for (slug, live, _, _) in &slugs {
if slug == stray || !slug.ends_with(&suffix) {
continue;
}
let bare = slug.trim_start_matches('-');
match projects
.iter_mut()
.find(|p| p[0].0.trim_start_matches('-') == bare)
{
Some(project) => project.push((slug, *live)),
None => projects.push(vec![(slug, *live)]),
}
}
let target = match projects.as_slice() {
[] => {
if !has_transcripts {
report.unresolved.push(stray.clone());
}
continue;
}
[spellings] => spellings
.iter()
.max_by_key(|(slug, live)| (*live, slug.starts_with('-')))
.map(|(slug, _)| (*slug).clone())
.expect("a project group is never empty"),
_ => {
report.ambiguous.push(AmbiguousSlug {
slug: stray.clone(),
candidates: projects
.iter()
.flat_map(|p| p.iter().map(|(slug, _)| (*slug).clone()))
.collect(),
});
continue;
}
};
let ids: Vec<Uuid> = {
let conn = self.reader();
let mut stmt = conn.prepare(concat!(
"SELECT id FROM memories WHERE project_path = ?1 AND source IN (",
explicit_sources_list!(),
")"
))?;
let rows =
stmt.query_map(params![stray], |r| parse_uuid(&r.get::<_, String>(0)?))?;
rows.collect::<rusqlite::Result<Vec<_>>>()?
};
let rows: Vec<(Uuid, String)> =
ids.into_iter().map(|id| (id, target.clone())).collect();
let moved = self.refile_all(&rows)?;
if moved > 0 {
report.memories_refiled += moved;
report.refiled.push(RefiledSlug {
from: stray.clone(),
to: target,
memories: moved,
});
}
}
Ok(report)
}
pub fn refile_memory(&self, id: Uuid, project_path: &str) -> Result<bool> {
self.with_txn(|tx| self.refile_on_tx(tx, id, project_path))
}
fn refile_on_tx(&self, tx: &Transaction<'_>, id: Uuid, project_path: &str) -> Result<bool> {
let changed = tx.execute(
"UPDATE memories SET project_path = ?2, updated_at = ?3 \
WHERE id = ?1 AND project_path IS NOT ?2",
params![id.to_string(), project_path, Utc::now().to_rfc3339()],
)?;
if changed == 0 {
return Ok(false);
}
Self::append_outbox(tx, id, "upsert")?;
if let Some(memory) = Self::memory_on_tx(tx, id)? {
self.log_memory_update(tx, &memory)?;
}
Ok(true)
}
fn purge_memories_where(&self, predicate: &str, value: &str) -> Result<usize> {
self.with_txn(|tx| {
let ids: Vec<String> = {
let mut stmt = tx.prepare(&format!("SELECT id FROM memories WHERE {predicate}"))?;
let rows = stmt.query_map(params![value], |r| r.get::<_, String>(0))?;
rows.collect::<rusqlite::Result<Vec<_>>>()?
};
for id in &ids {
tx.execute(
"DELETE FROM relations WHERE from_id = ?1 OR to_id = ?1",
params![id],
)?;
if let Ok(uuid) = Uuid::parse_str(id) {
Self::append_outbox(tx, uuid, "delete")?;
}
Self::upsert_tombstone(tx, id, "memory", &self.machine_id)?;
Self::append_sync_log(
tx,
SyncOp::Delete,
SyncNodeType::Memory,
id,
None,
&self.machine_id,
None,
)?;
}
tx.execute(
&format!("DELETE FROM memories WHERE {predicate}"),
params![value],
)?;
Ok(ids.len())
})
}
}
const INGEST_LOG_COLS: &str = "file_path, file_hash, content_hash, session_key, legacy_ids";
fn row_to_ingest_log_entry(r: &rusqlite::Row<'_>) -> rusqlite::Result<IngestLogEntry> {
Ok(IngestLogEntry {
file_path: r.get(0)?,
file_hash: r.get(1)?,
content_hash: r.get(2)?,
session_key: r.get(3)?,
legacy_ids: r.get::<_, i64>(4)? != 0,
})
}
#[cfg(test)]
mod tests {
use super::*;
use crate::schema::{MemoryType, Tier};
use crate::sqlite_store::TRAVERSE_MAX_RESULTS;
fn store() -> SqliteStore {
SqliteStore::in_memory("machine-a".to_string()).unwrap()
}
fn memory(content: &str) -> Memory {
Memory::new(
content.to_string(),
MemoryType::Semantic,
"test".to_string(),
String::new(),
)
}
fn record(
op: SyncOp,
node_type: SyncNodeType,
node_id: String,
data: Option<serde_json::Value>,
) -> SyncRecord {
SyncRecord {
local_seq: 1,
origin_machine_id: "machine-b".to_string(),
origin_seq: 1,
op,
node_type,
node_id,
timestamp: Utc::now(),
data,
}
}
fn deletes_for(s: &SqliteStore, id: Uuid) -> usize {
s.sync_log_since(0)
.unwrap()
.into_iter()
.filter(|e| e.op == SyncOp::Delete && e.node_id == id.to_string())
.count()
}
fn outbox_rows(s: &SqliteStore) -> i64 {
s.reader()
.query_row("SELECT COUNT(*) FROM index_outbox", [], |r| r.get(0))
.unwrap()
}
fn orphan_tombstone(s: &SqliteStore, id: Uuid) {
s.with_txn(|tx| SqliteStore::upsert_tombstone(tx, &id.to_string(), "memory", "machine-a"))
.unwrap();
}
#[test]
fn skip_reasons_distinguish_convergence_from_data_loss() {
let s = store();
let m = memory("already here");
s.store_memory(&m).unwrap();
let dup = record(
SyncOp::Create,
SyncNodeType::Memory,
m.id.to_string(),
Some(serde_json::to_value(&m).unwrap()),
);
assert_eq!(
s.apply_record(&dup).unwrap(),
ApplyOutcome::Skipped(SkipReason::AlreadyPresent),
"a re-sent record the store already has is routine convergence"
);
let ghost = Relation {
from_id: Uuid::new_v4(),
to_id: Uuid::new_v4(),
relation_type: RelationType::Mentions,
strength: 1.0,
context: None,
};
let edge = record(
SyncOp::Create,
SyncNodeType::Relation,
format!("{}:{}:mentions", ghost.from_id, ghost.to_id),
Some(serde_json::to_value(&ghost).unwrap()),
);
assert_eq!(
s.apply_record(&edge).unwrap(),
ApplyOutcome::Skipped(SkipReason::DanglingEndpoint),
"an edge that can never resolve must be distinguishable from convergence"
);
}
#[test]
fn unconsolidated_memories_orders_by_source_id_then_created_at() {
let s = store();
let mk = |content: &str, source_id: &str, created_at: chrono::DateTime<Utc>| {
let mut m = memory(content);
m.source_id = source_id.to_string();
m.created_at = created_at;
m
};
let base = Utc::now();
let b_second = mk("b turn 2", "conv-b", base + chrono::Duration::seconds(10));
let a_second = mk("a turn 2", "conv-a", base + chrono::Duration::seconds(10));
let b_first = mk("b turn 1", "conv-b", base);
let a_first = mk("a turn 1", "conv-a", base);
for m in [&b_second, &a_second, &b_first, &a_first] {
s.store_memory(m).unwrap();
}
let fetched = s.unconsolidated_memories(100).unwrap();
let order: Vec<&str> = fetched.iter().map(|m| m.content.as_str()).collect();
assert_eq!(
order,
vec!["a turn 1", "a turn 2", "b turn 1", "b turn 2"],
"fetch must group by source_id and order each group by created_at"
);
}
#[test]
fn consolidation_state_replicates_so_a_peer_does_not_redistill() {
let source = store();
let raw = memory("raw memory to distill");
source.store_memory(&raw).unwrap();
let distilled = memory("distilled summary");
source.store_memory(&distilled).unwrap();
let peer = SqliteStore::in_memory("machine-peer".to_string()).unwrap();
peer.store_memory(&raw).unwrap();
let pending = |st: &SqliteStore| {
st.unconsolidated_memories(100)
.unwrap()
.iter()
.any(|m| m.id == raw.id)
};
assert!(
pending(&peer),
"precondition: the peer still sees the raw memory as work to do"
);
source
.mark_consolidated(raw.id, distilled.id, "gemma4-e2b")
.unwrap();
assert!(!pending(&source));
for entry in source.sync_log_since(0).unwrap() {
if entry.node_type != SyncNodeType::Distillation {
continue;
}
let data = entry
.data
.as_ref()
.and_then(|d| serde_json::from_str::<serde_json::Value>(d).ok());
let rec = record(entry.op, entry.node_type, entry.node_id.clone(), data);
peer.apply_record(&rec).unwrap();
}
assert!(
!pending(&peer),
"the peer must learn the memory was already distilled, or it burns GPU redoing it"
);
assert_eq!(peer.consolidation_stats().unwrap().raw_retire_pending, 0);
assert_eq!(
peer.integrity_report(None)
.unwrap()
.orphaned_distillation_markers,
0
);
peer.store_memory(&distilled).unwrap();
assert_eq!(peer.consolidation_stats().unwrap().raw_retire_pending, 1);
}
#[test]
fn replaying_a_distillation_record_is_idempotent() {
let s = store();
let raw = memory("raw");
s.store_memory(&raw).unwrap();
let dist = memory("distilled");
s.store_memory(&dist).unwrap();
s.mark_consolidated(raw.id, dist.id, "gemma4-e2b").unwrap();
let entry = s
.sync_log_since(0)
.unwrap()
.into_iter()
.find(|e| e.node_type == SyncNodeType::Distillation)
.expect("mark_consolidated must log a distillation record");
let data = entry
.data
.as_ref()
.and_then(|d| serde_json::from_str::<serde_json::Value>(d).ok());
let rec = record(
SyncOp::Create,
SyncNodeType::Distillation,
entry.node_id,
data,
);
assert_eq!(
s.apply_record(&rec).unwrap(),
ApplyOutcome::Skipped(SkipReason::AlreadyPresent)
);
assert_eq!(s.consolidation_stats().unwrap().raw_retire_pending, 1);
}
#[test]
fn retire_is_idempotent_and_logs_one_update_per_raw() {
let s = store();
let raw = memory("raw memory to distill");
s.store_memory(&raw).unwrap();
let dist = memory("distilled summary");
s.store_memory(&dist).unwrap();
s.mark_consolidated(raw.id, dist.id, "gemma4-e2b").unwrap();
assert_eq!(s.retire_consolidated_raws().unwrap(), 1);
assert_eq!(
s.get_memory(raw.id).unwrap().unwrap().tier,
crate::schema::Tier::Archived
);
assert_eq!(deletes_for(&s, raw.id), 0, "retirement never deletes");
let log_after_first = s.sync_log_since(0).unwrap().len();
let outbox_after_first = outbox_rows(&s);
assert_eq!(
s.retire_consolidated_raws().unwrap(),
0,
"a second pass has no hot raws left to retire"
);
assert_eq!(
s.sync_log_since(0).unwrap().len(),
log_after_first,
"re-running retire must not re-log an update"
);
assert_eq!(
outbox_rows(&s),
outbox_after_first,
"re-running retire must not re-queue index work"
);
}
#[test]
fn deleting_an_already_deleted_memory_records_nothing_new() {
let s = store();
let m = memory("deleted once, asked for twice");
s.store_memory(&m).unwrap();
s.delete_memory(m.id).unwrap();
let log_len = s.sync_log_since(0).unwrap().len();
let outbox = outbox_rows(&s);
s.delete_memory(m.id).unwrap();
assert_eq!(
s.sync_log_since(0).unwrap().len(),
log_len,
"the deletion is already asserted to peers; repeating it adds no information"
);
assert_eq!(outbox_rows(&s), outbox);
assert!(
s.tombstone_exists(m.id).unwrap(),
"the tombstone must survive the repeat delete"
);
}
#[test]
fn deleting_a_memory_this_machine_never_had_records_nothing() {
let s = store();
let absent = Uuid::new_v4();
let outbox = outbox_rows(&s);
assert!(!s.remove_memory(absent).unwrap());
assert!(
!s.tombstone_exists(absent).unwrap(),
"nothing was removed, so there is nothing to tombstone or tell peers"
);
assert_eq!(deletes_for(&s, absent), 0);
assert_eq!(outbox_rows(&s), outbox);
}
#[test]
fn remove_memory_reports_whether_a_row_existed() {
let s = store();
let m = memory("held here");
s.store_memory(&m).unwrap();
assert!(s.remove_memory(m.id).unwrap());
assert!(s.tombstone_exists(m.id).unwrap());
assert_eq!(deletes_for(&s, m.id), 1);
assert!(!s.remove_memory(m.id).unwrap());
assert_eq!(deletes_for(&s, m.id), 1);
}
#[test]
fn backfill_covers_distillation_rows_written_before_they_were_syncable() {
let s = store();
let raw = memory("raw");
s.store_memory(&raw).unwrap();
let dist = memory("distilled");
s.store_memory(&dist).unwrap();
s.mark_consolidated(raw.id, dist.id, "gemma4-e2b").unwrap();
s.with_txn(|tx| {
tx.execute("DELETE FROM sync_log", [])?;
Ok(())
})
.unwrap();
s.backfill_sync_log().unwrap();
let distillations = s
.sync_log_since(0)
.unwrap()
.into_iter()
.filter(|e| e.node_type == SyncNodeType::Distillation)
.count();
assert_eq!(
distillations, 1,
"the 3577 rows already on disk must become syncable, not stay stranded"
);
}
#[test]
fn distinct_project_paths_counts_live_memories_per_project() {
let s = store();
let mut a1 = memory("a1");
a1.project_path = Some("project-a".to_string());
s.store_memory(&a1).unwrap();
let mut a2 = memory("a2");
a2.project_path = Some("project-a".to_string());
s.store_memory(&a2).unwrap();
let mut b1 = memory("b1");
b1.project_path = Some("project-b".to_string());
s.store_memory(&b1).unwrap();
s.store_memory(&memory("no project")).unwrap();
let mut archived = memory("a3, consolidated away");
archived.project_path = Some("project-a".to_string());
archived.tier = Tier::Archived;
s.store_memory(&archived).unwrap();
let mut superseded = memory("b2, superseded");
superseded.project_path = Some("project-b".to_string());
superseded.superseded_by = Some(b1.id);
s.store_memory(&superseded).unwrap();
let mut only_archived = memory("c1, nothing live left");
only_archived.project_path = Some("project-c".to_string());
only_archived.tier = Tier::Archived;
s.store_memory(&only_archived).unwrap();
let mut counts = s.distinct_project_paths().unwrap();
counts.sort_by(|a, b| a.0.cmp(&b.0));
assert_eq!(
counts,
vec![("project-a".to_string(), 2), ("project-b".to_string(), 1),]
);
}
#[test]
fn a_source_scope_admits_distillates_of_that_source() {
let s = store();
let raw = explicit("cursor transcript chunk about wombats", "cursor");
let mut distillate = explicit("wombats, distilled", "consolidation");
distillate.embedding = vec![1.0; 4];
let mut other = explicit("wombats from another source", "claude-code");
other.embedding = vec![1.0; 4];
for m in [&raw, &distillate, &other] {
s.store_memory(m).unwrap();
}
s.mark_consolidated(raw.id, distillate.id, "model").unwrap();
assert!(s.source_admits(&distillate, "cursor").unwrap());
assert!(!s.source_admits(&other, "cursor").unwrap());
assert!(!s.source_admits(&distillate, "claude-code").unwrap());
assert_eq!(s.scoped_count(Some("cursor"), None, None).unwrap(), 2);
let kept = s
.retain_in_scope(
&[raw.id, distillate.id, other.id],
Some("cursor"),
None,
None,
)
.unwrap();
assert!(kept.contains(&distillate.id) && kept.contains(&raw.id));
assert!(!kept.contains(&other.id));
let text: Vec<Uuid> = s
.scoped_text_search("wombats", Some("cursor"), None, None, 10)
.unwrap()
.into_iter()
.map(|m| m.id)
.collect();
assert!(text.contains(&distillate.id) && !text.contains(&other.id));
let dense: Vec<Uuid> = s
.filtered_vector_search(&[1.0; 4], Some("cursor"), None, None, 10)
.unwrap()
.into_iter()
.map(|(m, _)| m.id)
.collect();
assert_eq!(dense, vec![distillate.id]);
}
#[test]
fn record_accesses_bumps_counts_without_touching_confidence() {
let s = store();
let mut a = memory("read twice");
a.confidence = 0.5;
let b = memory("read once");
let untouched = memory("never read");
for m in [&a, &b, &untouched] {
s.store_memory(m).unwrap();
}
let log_len = s.sync_log_since(0).unwrap().len();
let before = Utc::now();
s.record_accesses(&[a.id, b.id]).unwrap();
s.record_accesses(&[a.id, Uuid::new_v4()]).unwrap();
let a = s.get_memory(a.id).unwrap().unwrap();
assert_eq!(a.access_count, 2);
assert_eq!(a.confidence, 0.5);
assert!(a.last_accessed >= before);
let b = s.get_memory(b.id).unwrap().unwrap();
assert_eq!(b.access_count, 1);
let untouched = s.get_memory(untouched.id).unwrap().unwrap();
assert_eq!(untouched.access_count, 0);
assert_eq!(
s.sync_log_since(0).unwrap().len(),
log_len,
"access counts are local statistics, never synced"
);
}
fn in_project(content: &str, project: &str, created: DateTime<Utc>) -> Memory {
let mut m = memory(content);
m.project_path = Some(project.to_string());
m.created_at = created;
m
}
#[test]
fn memories_by_project_path_returns_live_rows_newest_first() {
let s = store();
let t0 = Utc::now() - chrono::Duration::days(3);
let old = in_project("old", "loom", t0);
let new = in_project("new", "loom", t0 + chrono::Duration::days(1));
let mut archived = in_project("archived", "loom", t0 + chrono::Duration::days(2));
archived.tier = Tier::Archived;
let mut superseded = in_project("superseded", "loom", t0 + chrono::Duration::days(2));
superseded.superseded_by = Some(new.id);
let elsewhere = in_project("elsewhere", "other", t0 + chrono::Duration::days(2));
for m in [&old, &new, &archived, &superseded, &elsewhere] {
s.store_memory(m).unwrap();
}
let ids: Vec<Uuid> = s
.memories_by_project_path("loom")
.unwrap()
.into_iter()
.map(|m| m.id)
.collect();
assert_eq!(ids, vec![new.id, old.id]);
}
#[test]
fn memories_by_project_path_keeps_the_twenty_row_bound() {
let s = store();
let t0 = Utc::now() - chrono::Duration::days(1);
for i in 0..25 {
s.store_memory(&in_project(
&format!("row {i}"),
"loom",
t0 + chrono::Duration::minutes(i),
))
.unwrap();
}
let rows = s.memories_by_project_path("loom").unwrap();
assert_eq!(rows.len(), 20);
assert_eq!(rows[0].content, "row 24");
}
fn mentions(memory_id: Uuid, entity_id: Uuid) -> Relation {
Relation {
from_id: memory_id,
to_id: entity_id,
relation_type: RelationType::Mentions,
strength: 1.0,
context: None,
}
}
#[test]
fn traverse_reaches_live_memories_through_shared_entities_conversations_and_edges() {
let s = store();
let entity = Entity::new("wombat".to_string(), "animal".to_string());
s.store_entity(&entity).unwrap();
let conversation = Uuid::new_v4();
let mut start = memory("start");
start.conversation_id = Some(conversation);
let via_entity = memory("shares the wombat entity");
let mut via_conversation = memory("same conversation");
via_conversation.conversation_id = Some(conversation);
let via_reverse_edge = memory("relates_to start, edge pointing at start");
let mut archived_peer = memory("archived, shares the entity");
archived_peer.tier = Tier::Archived;
let mut superseded_peer = memory("superseded, same conversation");
superseded_peer.conversation_id = Some(conversation);
superseded_peer.superseded_by = Some(start.id);
let unrelated = memory("nothing in common");
for m in [
&start,
&via_entity,
&via_conversation,
&via_reverse_edge,
&archived_peer,
&superseded_peer,
&unrelated,
] {
s.store_memory(m).unwrap();
}
s.store_relation(&mentions(start.id, entity.id)).unwrap();
s.store_relation(&mentions(via_entity.id, entity.id))
.unwrap();
s.store_relation(&mentions(archived_peer.id, entity.id))
.unwrap();
s.store_relation(&Relation {
from_id: via_reverse_edge.id,
to_id: start.id,
relation_type: RelationType::RelatesTo,
strength: 1.0,
context: None,
})
.unwrap();
let mut got: Vec<Uuid> = s
.traverse(start.id, 1)
.unwrap()
.into_iter()
.map(|(m, _)| m.id)
.collect();
got.sort();
let mut want = vec![via_entity.id, via_conversation.id, via_reverse_edge.id];
want.sort();
assert_eq!(got, want);
}
#[test]
fn traverse_depth_bounds_how_far_a_shared_entity_carries() {
let s = store();
let near = Entity::new("near".to_string(), "thing".to_string());
let far = Entity::new("far".to_string(), "thing".to_string());
s.store_entity(&near).unwrap();
s.store_entity(&far).unwrap();
let start = memory("start");
let hop = memory("one hop away");
let two_hops = memory("two hops away");
for m in [&start, &hop, &two_hops] {
s.store_memory(m).unwrap();
}
s.store_relation(&mentions(start.id, near.id)).unwrap();
s.store_relation(&mentions(hop.id, near.id)).unwrap();
s.store_relation(&mentions(hop.id, far.id)).unwrap();
s.store_relation(&mentions(two_hops.id, far.id)).unwrap();
let ids = |depth: u32| -> Vec<Uuid> {
let mut v: Vec<Uuid> = s
.traverse(start.id, depth)
.unwrap()
.into_iter()
.map(|(m, _)| m.id)
.collect();
v.sort();
v
};
assert_eq!(ids(1), vec![hop.id]);
let mut want = vec![hop.id, two_hops.id];
want.sort();
assert_eq!(ids(2), want);
assert!(!ids(2).contains(&start.id));
}
#[test]
fn traverse_result_count_is_bounded() {
let s = store();
let hub = Entity::new("hub".to_string(), "thing".to_string());
s.store_entity(&hub).unwrap();
let start = memory("start");
s.store_memory(&start).unwrap();
s.store_relation(&mentions(start.id, hub.id)).unwrap();
for i in 0..(TRAVERSE_MAX_RESULTS + 10) {
let m = memory(&format!("peer {i}"));
s.store_memory(&m).unwrap();
s.store_relation(&mentions(m.id, hub.id)).unwrap();
}
assert_eq!(s.traverse(start.id, 1).unwrap().len(), TRAVERSE_MAX_RESULTS);
}
#[test]
fn scoped_text_search_returns_the_newest_matches_first() {
let s = store();
let t0 = Utc::now() - chrono::Duration::days(3);
let oldest = in_project("wombat sighting one", "zoo", t0);
let middle = in_project("wombat sighting two", "zoo", t0 + chrono::Duration::days(1));
let newest = in_project(
"wombat sighting three",
"zoo",
t0 + chrono::Duration::days(2),
);
for m in [&oldest, &middle, &newest] {
s.store_memory(m).unwrap();
}
let ids: Vec<Uuid> = s
.scoped_text_search("wombat", None, None, Some("zoo"), 2)
.unwrap()
.into_iter()
.map(|m| m.id)
.collect();
assert_eq!(ids, vec![newest.id, middle.id]);
}
fn explicit(content: &str, source: &str) -> Memory {
let mut m = memory(content);
m.source = source.to_string();
m
}
#[test]
fn explicit_memories_are_neither_distilled_nor_retired() {
let s = store();
let remembered = explicit("remembered on purpose", "mcp-direct");
let via_api = explicit("posted to /v1/remember", "api");
let memory_file = explicit("from a claude memory file", "claude-memory");
let from_cli = explicit("sb remember from a terminal", "cli");
let transcript = explicit("a transcript chunk", "claude-code");
for m in [&remembered, &via_api, &memory_file, &from_cli, &transcript] {
s.store_memory(m).unwrap();
}
let pending: Vec<Uuid> = s
.unconsolidated_memories(10)
.unwrap()
.into_iter()
.map(|m| m.id)
.collect();
assert_eq!(pending, vec![transcript.id]);
assert_eq!(s.consolidation_stats().unwrap().raw_pending, 1);
let dist = explicit("a paraphrase", "consolidation");
s.store_memory(&dist).unwrap();
s.mark_consolidated(remembered.id, dist.id, "model")
.unwrap();
s.mark_consolidated(transcript.id, dist.id, "model")
.unwrap();
assert_eq!(s.consolidation_stats().unwrap().raw_retire_pending, 1);
assert_eq!(s.retire_consolidated_raws().unwrap(), 1);
assert_eq!(
s.get_memory(remembered.id).unwrap().unwrap().tier,
crate::schema::Tier::Hot,
"a marker on an explicit memory must never retire it"
);
assert_eq!(
s.get_memory(transcript.id).unwrap().unwrap().tier,
crate::schema::Tier::Archived
);
}
#[test]
fn the_undistillable_source_sql_list_matches_the_rust_lists() {
let sql = crate::schema::distillable_source!("source");
let lists = [
crate::schema::EXPLICIT_SOURCES,
crate::schema::DOCUMENT_SOURCES,
];
for source in lists.concat() {
assert!(sql.contains(&format!("'{source}'")), "{sql}");
}
assert_eq!(
sql.matches('\'').count() / 2,
lists.concat().len() + 1,
"the SQL list holds exactly the explicit and document sources plus 'consolidation'"
);
}
fn filed(content: &str, source: &str, project: &str) -> Memory {
explicit(content, source).with_project_path(Some(project.to_string()))
}
#[test]
fn stray_short_slugs_move_to_the_one_project_they_name() {
let s = store();
let rows = [
filed("wardrobe transcript", "claude-code", "-mnt-data-wardrobe"),
filed("wardrobe remembered", "mcp-direct", "wardrobe"),
filed(
"accounting transcript",
"claude-code",
"home-anish-alayacare-services-api-accounting",
),
filed("accounting remembered", "api", "api-accounting"),
filed("loom transcript", "claude-code", "-mnt-data-loom"),
filed("loom via cursor", "cursor", "mnt-data-loom"),
filed("loom remembered", "mcp-direct", "loom"),
filed("harness transcript", "claude-code", "-mnt-data-harness"),
filed(
"my-harness transcript",
"claude-code",
"-mnt-data-my-harness",
),
filed("harness remembered", "mcp-direct", "harness"),
filed("probe remembered", "mcp-direct", "tmp-skillprobe-1"),
filed("a transcript under a bare slug", "cursor", "wardrobe"),
];
for m in &rows {
s.store_memory(m).unwrap();
}
let project = |m: &Memory| s.get_memory(m.id).unwrap().unwrap().project_path.unwrap();
let log_before = s.max_sync_seq().unwrap();
let report = s.refile_stray_project_slugs().unwrap();
assert_eq!(project(&rows[1]), "-mnt-data-wardrobe");
assert_eq!(
project(&rows[3]),
"home-anish-alayacare-services-api-accounting"
);
assert_eq!(
project(&rows[6]),
"-mnt-data-loom",
"two spellings of one directory are one project, not an ambiguity"
);
assert_eq!(
project(&rows[9]),
"harness",
"an ambiguous name is not guessed"
);
assert_eq!(project(&rows[10]), "tmp-skillprobe-1");
assert_eq!(
project(&rows[11]),
"wardrobe",
"only explicit memories were filed by the unresolved remember"
);
assert_eq!(report.memories_refiled, 3);
let ambiguous: Vec<&str> = report.ambiguous.iter().map(|a| a.slug.as_str()).collect();
assert_eq!(ambiguous, vec!["harness"]);
assert_eq!(
report.ambiguous[0].candidates,
vec![
"-mnt-data-harness".to_string(),
"-mnt-data-my-harness".to_string()
]
);
assert_eq!(s.max_sync_seq().unwrap(), log_before + 3);
assert_eq!(report.unresolved, vec!["tmp-skillprobe-1".to_string()]);
let again = s.refile_stray_project_slugs().unwrap();
assert_eq!(again.memories_refiled, 0);
assert_eq!(
again.unresolved,
vec!["tmp-skillprobe-1".to_string()],
"a transcript project that received refiled memories is not itself a stray"
);
assert_eq!(s.max_sync_seq().unwrap(), log_before + 3);
}
#[test]
fn restore_explicit_memories_leaves_a_forgotten_memory_retired() {
let s = store();
let forgotten = explicit("explicit, then forgotten", "mcp-direct");
let distillate = explicit("distilled from the forgotten one", "consolidation");
s.store_memory(&forgotten).unwrap();
s.store_memory(&distillate).unwrap();
s.mark_consolidated(forgotten.id, distillate.id, "model")
.unwrap();
s.retire_memory(forgotten.id, "wrong").unwrap();
let stats = s.restore_explicit_memories().unwrap();
assert_eq!(stats.raws_restored, 0);
assert_eq!(
s.get_memory(forgotten.id).unwrap().unwrap().tier,
crate::schema::Tier::Archived
);
}
#[test]
fn restore_explicit_memories_returns_them_to_recall_and_supersedes_their_distillates() {
let s = store();
let e1 = explicit("explicit, archived", "mcp-direct");
let e2 = explicit("explicit, marked but still hot", "claude-memory");
let e3 = explicit(
"explicit, archived, shares a distillate with a transcript",
"api",
);
let raw = explicit("transcript chunk", "claude-code");
let all_explicit = explicit("distilled from e1 and e2 only", "consolidation");
let mixed = explicit("distilled from e3 and a transcript", "consolidation");
for m in [&e1, &e2, &e3, &raw, &all_explicit, &mixed] {
s.store_memory(m).unwrap();
}
for (r, d) in [
(&e1, &all_explicit),
(&e2, &all_explicit),
(&e3, &mixed),
(&raw, &mixed),
] {
s.mark_consolidated(r.id, d.id, "model").unwrap();
}
for id in [e1.id, e3.id, raw.id] {
s.archive_memory(id).unwrap();
}
let log_before = s.sync_log_since(0).unwrap().len();
let outbox_before = outbox_rows(&s);
let stats = s.restore_explicit_memories().unwrap();
assert_eq!(
stats.raws_restored, 2,
"e1 and e3 were archived; e2 was still hot"
);
assert_eq!(stats.markers_dropped, 3);
assert_eq!(stats.distillates_superseded, 1);
for id in [e1.id, e2.id, e3.id] {
let m = s.get_memory(id).unwrap().unwrap();
assert_eq!(m.tier, crate::schema::Tier::Hot);
assert!(m.superseded_by.is_none());
}
let marked: i64 = s
.reader()
.query_row(
"SELECT COUNT(*) FROM distillation_log WHERE memory_id IN (?1, ?2, ?3)",
params![e1.id.to_string(), e2.id.to_string(), e3.id.to_string()],
|r| r.get(0),
)
.unwrap();
assert_eq!(marked, 0, "a restored memory must not be retired again");
assert_eq!(
s.get_memory(raw.id).unwrap().unwrap().tier,
crate::schema::Tier::Archived,
"a transcript raw stays retired"
);
let all_explicit_after = s.get_memory(all_explicit.id).unwrap().unwrap();
assert!(
[Some(e1.id), Some(e2.id)].contains(&all_explicit_after.superseded_by),
"a distillate made only of explicit memories yields to one of them"
);
assert!(
s.get_memory(mixed.id)
.unwrap()
.unwrap()
.superseded_by
.is_none(),
"a distillate still standing in for a transcript raw stays live"
);
let new_entries: Vec<SyncEntry> = s.sync_log_since(log_before as u64).unwrap();
let updates: Vec<String> = new_entries
.iter()
.filter(|e| e.op == SyncOp::Update && e.node_type == SyncNodeType::Memory)
.map(|e| e.node_id.clone())
.collect();
for id in [e1.id, e3.id, all_explicit.id] {
assert!(updates.contains(&id.to_string()), "{id} must replicate");
}
let marker_deletes = new_entries
.iter()
.filter(|e| e.op == SyncOp::Delete && e.node_type == SyncNodeType::Distillation)
.count();
assert_eq!(marker_deletes, 3, "peers must forget the markers too");
assert_eq!(
outbox_rows(&s) - outbox_before,
3,
"restored rows and the superseded distillate re-enter the indexes"
);
let again = s.restore_explicit_memories().unwrap();
assert_eq!(
(
again.raws_restored,
again.markers_dropped,
again.distillates_superseded
),
(0, 0, 0)
);
}
#[test]
fn refile_nested_transcript_projects_moves_raws_and_their_distillates() {
let s = store();
let raw = |content: &str, source_id: &str, project: &str| {
let mut m = memory(content);
m.source = "claude-code".to_string();
m.source_id = source_id.to_string();
m.project_path = Some(project.to_string());
s.store_memory(&m).unwrap();
m
};
let loom = "/home/u/.claude/projects/-mnt-data-loom/sess/subagents/agent-1.jsonl";
let kiln = "/home/u/.claude/projects/-mnt-data-kiln/sess/subagents/agent-2.jsonl";
let sub_a = raw("loom subagent a", loom, "subagents");
let sub_b = raw("loom subagent b", loom, "subagents");
let sub_c = raw("kiln subagent", kiln, "subagents");
let cursor = raw(
"cursor agent transcript",
"/home/u/.cursor/projects/mnt-data-harness/agent-transcripts/s1/s1.jsonl",
"agent-transcripts",
);
let rootless = raw(
"no projects root in its path",
"/srv/dump/agent-3.jsonl",
"subagents",
);
let filed = raw("already filed", loom, "-mnt-data-loom");
let mut dist = memory("distilled from two loom raws and one kiln raw");
dist.source = "consolidation".to_string();
dist.project_path = Some("subagents".to_string());
s.store_memory(&dist).unwrap();
for r in [&sub_a, &sub_b, &sub_c] {
s.mark_consolidated(r.id, dist.id, "model").unwrap();
}
let log_before = s.sync_log_since(0).unwrap().len();
let outbox_before = outbox_rows(&s);
assert_eq!(s.refile_nested_transcript_projects().unwrap(), 5);
let project = |id: Uuid| s.get_memory(id).unwrap().unwrap().project_path;
for id in [sub_a.id, sub_b.id, filed.id] {
assert_eq!(project(id).as_deref(), Some("-mnt-data-loom"));
}
assert_eq!(project(sub_c.id).as_deref(), Some("-mnt-data-kiln"));
assert_eq!(project(cursor.id).as_deref(), Some("mnt-data-harness"));
assert_eq!(
project(dist.id).as_deref(),
Some("-mnt-data-loom"),
"a distillate follows the majority project of its raws"
);
assert_eq!(
project(rootless.id).as_deref(),
Some("subagents"),
"a path without a projects root cannot be re-filed"
);
let updates = s
.sync_log_since(log_before as u64)
.unwrap()
.into_iter()
.filter(|e| e.op == SyncOp::Update)
.count();
assert_eq!(updates, 5, "every re-filed row is replicated as an update");
assert_eq!(outbox_rows(&s) - outbox_before, 5);
assert_eq!(s.refile_nested_transcript_projects().unwrap(), 0);
}
#[test]
fn refile_nested_transcript_projects_files_an_orphaned_distillate_by_its_source_id() {
let s = store();
let distillate = |content: &str, source_id: &str| {
let mut m = memory(content);
m.source = "consolidation".to_string();
m.source_id = source_id.to_string();
m.project_path = Some("subagents".to_string());
s.store_memory(&m).unwrap();
m
};
let orphan = distillate(
"its raws were purged",
"/home/u/.claude/projects/-mnt-data-loom/sess/",
);
let raw = {
let mut m = memory("a raw that is later purged");
m.source = "claude-code".to_string();
m.project_path = Some("-mnt-data-kiln".to_string());
s.store_memory(&m).unwrap();
m
};
let purged = distillate(
"distilled from a raw that is gone",
"/home/u/.claude/projects/-mnt-data-kiln/sess/",
);
s.mark_consolidated(raw.id, purged.id, "model").unwrap();
s.delete_memory(raw.id).unwrap();
let rootless = distillate("no projects root", "/srv/dump/");
assert_eq!(s.refile_nested_transcript_projects().unwrap(), 2);
let project = |id: Uuid| s.get_memory(id).unwrap().unwrap().project_path;
assert_eq!(project(orphan.id).as_deref(), Some("-mnt-data-loom"));
assert_eq!(project(purged.id).as_deref(), Some("-mnt-data-kiln"));
assert_eq!(project(rootless.id).as_deref(), Some("subagents"));
assert_eq!(s.refile_nested_transcript_projects().unwrap(), 0);
}
#[test]
fn backfill_project_paths_files_by_the_project_directory_and_replicates() {
let s = store();
let mut with_path = memory("has a path-shaped source_id");
with_path.source_id = "/home/user/.claude/projects/-my-project/abc.jsonl".to_string();
s.store_memory(&with_path).unwrap();
let mut already_tagged = memory("already has a project_path");
already_tagged.source_id = "/home/user/.claude/projects/-my-project/def.jsonl".to_string();
already_tagged.project_path = Some("keep-me".to_string());
s.store_memory(&already_tagged).unwrap();
let unrooted = memory("source_id is not path-shaped");
s.store_memory(&unrooted).unwrap();
let mut nested = memory("a subagent transcript chunk");
nested.source_id =
"/home/user/.claude/projects/-my-project/abc/subagents/agent-1.jsonl".to_string();
s.store_memory(&nested).unwrap();
let log_before = s.max_sync_seq().unwrap();
let outbox_before = outbox_rows(&s);
let changed = s.backfill_project_paths().unwrap();
assert_eq!(changed, 2, "only the untagged, path-shaped rows backfill");
assert_eq!(
s.get_memory(nested.id).unwrap().unwrap().project_path,
Some("-my-project".to_string()),
"a nested transcript is filed under the project directory, not `subagents`"
);
assert_eq!(
s.max_sync_seq().unwrap() - log_before,
2,
"every backfilled row is replicated"
);
assert_eq!(outbox_rows(&s) - outbox_before, 2);
assert_eq!(s.backfill_project_paths().unwrap(), 0);
assert_eq!(
s.get_memory(with_path.id).unwrap().unwrap().project_path,
Some("-my-project".to_string())
);
assert_eq!(
s.get_memory(already_tagged.id)
.unwrap()
.unwrap()
.project_path,
Some("keep-me".to_string()),
"an existing project_path must not be overwritten"
);
assert_eq!(
s.get_memory(unrooted.id).unwrap().unwrap().project_path,
None
);
}
#[test]
fn prune_dangling_relations_drops_only_unresolvable_edges() {
let s = store();
let m = memory("live endpoint");
s.store_memory(&m).unwrap();
let ent = Entity {
id: Uuid::new_v4(),
name: "kuzu".to_string(),
entity_type: "tool".to_string(),
embedding: vec![],
aliases: vec![],
};
s.store_entity(&ent).unwrap();
s.store_relation(&Relation {
from_id: m.id,
to_id: ent.id,
relation_type: RelationType::Mentions,
strength: 1.0,
context: None,
})
.unwrap();
s.store_relation(&Relation {
from_id: Uuid::new_v4(),
to_id: ent.id,
relation_type: RelationType::Mentions,
strength: 1.0,
context: None,
})
.unwrap();
assert_eq!(s.all_relations().unwrap().len(), 2);
assert_eq!(s.prune_dangling_relations().unwrap(), 1);
let left = s.all_relations().unwrap();
assert_eq!(left.len(), 1);
assert_eq!(left[0].from_id, m.id, "the valid edge must survive");
assert_eq!(
s.prune_dangling_relations().unwrap(),
0,
"prune must be idempotent"
);
}
#[test]
fn pruned_relation_deletion_reaches_a_peer() {
let s = store();
let ent = Entity {
id: Uuid::new_v4(),
name: "kuzu".to_string(),
entity_type: "tool".to_string(),
embedding: vec![],
aliases: vec![],
};
s.store_entity(&ent).unwrap();
let ghost = Uuid::new_v4();
let dangling = Relation {
from_id: ghost,
to_id: ent.id,
relation_type: RelationType::Mentions,
strength: 1.0,
context: None,
};
s.store_relation(&dangling).unwrap();
let peer = SqliteStore::in_memory("machine-peer".to_string()).unwrap();
let mut ghost_memory = memory("endpoint alive on the peer");
ghost_memory.id = ghost;
peer.store_memory(&ghost_memory).unwrap();
peer.store_entity(&ent).unwrap();
peer.store_relation(&dangling).unwrap();
assert_eq!(peer.all_relations().unwrap().len(), 1);
let before = s.sync_log_since(0).unwrap().len() as u64;
s.prune_dangling_relations().unwrap();
for entry in s.sync_log_since(before).unwrap() {
if entry.op != SyncOp::Delete || entry.node_type != SyncNodeType::Relation {
continue;
}
let data = entry
.data
.as_ref()
.and_then(|d| serde_json::from_str::<serde_json::Value>(d).ok());
let rec = record(
SyncOp::Delete,
SyncNodeType::Relation,
entry.node_id.clone(),
data,
);
peer.apply_record(&rec).unwrap();
}
assert_eq!(
peer.all_relations().unwrap().len(),
0,
"the prune must propagate, not just clean the local store"
);
}
#[test]
fn backfill_covers_entities_conversations_and_relations_not_just_memories() {
let s = store();
let m = memory("endpoint memory");
s.store_memory(&m).unwrap();
let conv = Conversation {
id: Uuid::new_v4(),
source: "claude-code".to_string(),
machine_id: "machine-a".to_string(),
started_at: Utc::now(),
project_path: Some("/tmp/p".to_string()),
};
s.store_conversation(&conv).unwrap();
let ent = Entity {
id: Uuid::new_v4(),
name: "kuzu".to_string(),
entity_type: "tool".to_string(),
embedding: vec![],
aliases: vec![],
};
s.store_entity(&ent).unwrap();
s.store_relation(&Relation {
from_id: m.id,
to_id: conv.id,
relation_type: RelationType::DerivedFrom,
strength: 1.0,
context: None,
})
.unwrap();
s.with_txn(|tx| {
tx.execute("DELETE FROM sync_log", [])?;
Ok(())
})
.unwrap();
assert_eq!(s.sync_log_since(0).unwrap().len(), 0);
assert_eq!(s.backfill_sync_log().unwrap(), 4);
let by_type = |nt: SyncNodeType| {
s.sync_log_since(0)
.unwrap()
.into_iter()
.filter(|e| e.node_type == nt)
.count()
};
assert_eq!(by_type(SyncNodeType::Memory), 1);
assert_eq!(by_type(SyncNodeType::Conversation), 1);
assert_eq!(by_type(SyncNodeType::Entity), 1);
assert_eq!(by_type(SyncNodeType::Relation), 1);
assert_eq!(
s.backfill_sync_log().unwrap(),
0,
"backfill must be idempotent across every node type"
);
}
#[test]
fn backfilled_relation_applies_on_a_peer_once_its_endpoints_exist() {
let s = store();
let m = memory("endpoint memory");
s.store_memory(&m).unwrap();
let conv = Conversation {
id: Uuid::new_v4(),
source: "claude-code".to_string(),
machine_id: "machine-a".to_string(),
started_at: Utc::now(),
project_path: None,
};
s.store_conversation(&conv).unwrap();
s.store_relation(&Relation {
from_id: m.id,
to_id: conv.id,
relation_type: RelationType::DerivedFrom,
strength: 1.0,
context: None,
})
.unwrap();
s.with_txn(|tx| {
tx.execute("DELETE FROM sync_log", [])?;
Ok(())
})
.unwrap();
s.backfill_sync_log().unwrap();
let peer = SqliteStore::in_memory("machine-peer".to_string()).unwrap();
for entry in s.sync_log_since(0).unwrap() {
let data = entry
.data
.as_ref()
.and_then(|d| serde_json::from_str::<serde_json::Value>(d).ok());
let rec = record(entry.op, entry.node_type, entry.node_id.clone(), data);
peer.apply_record(&rec).unwrap();
}
assert_eq!(
peer.all_relations().unwrap().len(),
1,
"a backfilled relation must apply once its endpoints have been backfilled too"
);
}
#[test]
fn purging_by_source_id_tombstones_and_logs_each_delete() {
let s = store();
let mut m = memory("ingested from a file that later changed");
m.source_id = "/tmp/transcript.jsonl".to_string();
s.store_memory(&m).unwrap();
assert_eq!(
s.delete_memories_by_source_id("/tmp/transcript.jsonl")
.unwrap(),
1
);
let deletes: Vec<_> = s
.sync_log_since(0)
.unwrap()
.into_iter()
.filter(|e| e.op == SyncOp::Delete)
.collect();
assert_eq!(
deletes.len(),
1,
"a purge must log the delete or it can never reach a peer"
);
assert_eq!(deletes[0].node_id, m.id.to_string());
assert_eq!(
s.backfill_tombstones().unwrap(),
0,
"the purge must also leave a tombstone, or the peer copy is resurrected"
);
}
#[test]
fn purged_memory_is_deleted_on_a_peer_rather_than_resurrected() {
let source = store();
let mut m = memory("superseded by a newer version of the file");
m.source_id = "/tmp/transcript.jsonl".to_string();
source.store_memory(&m).unwrap();
let peer = SqliteStore::in_memory("machine-peer".to_string()).unwrap();
peer.store_memory(&m).unwrap();
source
.delete_memories_by_source_id("/tmp/transcript.jsonl")
.unwrap();
for entry in source.sync_log_since(0).unwrap() {
if entry.op != SyncOp::Delete {
continue;
}
let rec = record(
SyncOp::Delete,
SyncNodeType::Memory,
entry.node_id.clone(),
None,
);
peer.apply_record(&rec).unwrap();
}
assert!(
peer.get_memory(m.id).unwrap().is_none(),
"the peer must drop its copy of a purged memory"
);
let recreate = record(
SyncOp::Create,
SyncNodeType::Memory,
m.id.to_string(),
Some(serde_json::to_value(&m).unwrap()),
);
peer.apply_record(&recreate).unwrap();
assert!(
peer.get_memory(m.id).unwrap().is_none(),
"the tombstone must stop the purged memory coming back on the next sync"
);
}
#[test]
fn backfill_tombstones_emits_a_delete_for_every_orphaned_tombstone() {
let s = store();
let m = memory("deleted without a logged delete");
s.store_memory(&m).unwrap();
s.with_txn(|tx| {
tx.execute(
"DELETE FROM memories WHERE id = ?1",
params![m.id.to_string()],
)?;
Ok(())
})
.unwrap();
orphan_tombstone(&s, m.id);
let deletes = |s: &SqliteStore| {
s.sync_log_since(0)
.unwrap()
.into_iter()
.filter(|e| e.op == SyncOp::Delete)
.collect::<Vec<_>>()
};
assert!(
deletes(&s).is_empty(),
"precondition: tombstone has no delete entry"
);
assert_eq!(s.backfill_tombstones().unwrap(), 1);
let got = deletes(&s);
assert_eq!(got.len(), 1);
assert_eq!(got[0].node_id, m.id.to_string());
assert_eq!(got[0].node_type, SyncNodeType::Memory);
assert_eq!(
s.backfill_tombstones().unwrap(),
0,
"must be idempotent, not re-emit a delete on every run"
);
}
#[test]
fn backfill_tombstones_skips_tombstones_that_already_logged_a_delete() {
let s = store();
let m = memory("deleted through the normal path");
s.store_memory(&m).unwrap();
s.delete_memory(m.id).unwrap();
let before = s.sync_log_since(0).unwrap().len();
assert_eq!(s.backfill_tombstones().unwrap(), 0);
assert_eq!(s.sync_log_since(0).unwrap().len(), before);
}
#[test]
fn backfilled_delete_survives_export_and_deletes_on_a_peer() {
let source = store();
let m = memory("purged here, still alive on the peer");
source.store_memory(&m).unwrap();
source
.with_txn(|tx| {
tx.execute(
"DELETE FROM memories WHERE id = ?1",
params![m.id.to_string()],
)?;
Ok(())
})
.unwrap();
orphan_tombstone(&source, m.id);
assert_eq!(source.backfill_tombstones().unwrap(), 1);
let peer = SqliteStore::in_memory("machine-peer".to_string()).unwrap();
peer.store_memory(&m).unwrap();
assert!(peer.get_memory(m.id).unwrap().is_some());
for entry in source.sync_log_since(0).unwrap() {
if entry.op != SyncOp::Delete {
continue;
}
let rec = record(
SyncOp::Delete,
SyncNodeType::Memory,
entry.node_id.clone(),
None,
);
peer.apply_record(&rec).unwrap();
}
assert!(
peer.get_memory(m.id).unwrap().is_none(),
"the backfilled delete must actually remove the memory on the peer"
);
}
#[test]
fn create_is_idempotent_and_respects_tombstones() {
let s = store();
let m = memory("incoming create");
let rec = record(
SyncOp::Create,
SyncNodeType::Memory,
m.id.to_string(),
Some(serde_json::to_value(&m).unwrap()),
);
assert!(matches!(
s.apply_record(&rec).unwrap(),
ApplyOutcome::Created
));
assert!(matches!(
s.apply_record(&rec).unwrap(),
ApplyOutcome::Skipped(SkipReason::AlreadyPresent)
));
s.delete_memory(m.id).unwrap();
assert!(
matches!(
s.apply_record(&rec).unwrap(),
ApplyOutcome::Skipped(SkipReason::Tombstoned)
),
"a tombstoned id must never be resurrected by a replayed create"
);
}
#[test]
fn update_lww_prefers_newer_updated_at() {
let s = store();
let mut m = memory("lww base");
s.store_memory(&m).unwrap();
let mut older = m.clone();
older.content = "stale writer".to_string();
older.updated_at = m.updated_at - chrono::Duration::seconds(60);
let stale = record(
SyncOp::Update,
SyncNodeType::Memory,
m.id.to_string(),
Some(serde_json::to_value(&older).unwrap()),
);
assert!(matches!(
s.apply_record(&stale).unwrap(),
ApplyOutcome::Skipped(SkipReason::StaleWrite)
));
assert_eq!(s.get_memory(m.id).unwrap().unwrap().content, "lww base");
m.content = "fresh writer".to_string();
m.updated_at += chrono::Duration::seconds(60);
let fresh = record(
SyncOp::Update,
SyncNodeType::Memory,
m.id.to_string(),
Some(serde_json::to_value(&m).unwrap()),
);
assert!(matches!(
s.apply_record(&fresh).unwrap(),
ApplyOutcome::Updated
));
assert_eq!(s.get_memory(m.id).unwrap().unwrap().content, "fresh writer");
}
fn session(at: &str) -> DateTime<Utc> {
at.parse().unwrap()
}
fn sync_log_rows(s: &SqliteStore) -> i64 {
s.reader()
.query_row("SELECT COUNT(*) FROM sync_log", [], |r| r.get(0))
.unwrap()
}
#[test]
fn an_update_that_wins_lww_without_a_date_keeps_the_stored_date() {
let s = store();
let mut m = memory("dated here");
m.session_at = Some(session("2026-07-23T09:00:00Z"));
s.store_memory(&m).unwrap();
let mut newer = m.clone();
newer.session_at = None;
newer.content = "edited on a peer that has no date".to_string();
newer.updated_at += chrono::Duration::seconds(60);
let update = record(
SyncOp::Update,
SyncNodeType::Memory,
m.id.to_string(),
Some(serde_json::to_value(&newer).unwrap()),
);
assert!(matches!(
s.apply_record(&update).unwrap(),
ApplyOutcome::Updated
));
let stored = s.get_memory(m.id).unwrap().unwrap();
assert_eq!(stored.content, "edited on a peer that has no date");
assert_eq!(stored.session_at, m.session_at);
}
#[test]
fn an_update_that_loses_lww_fills_only_a_missing_date() {
let s = store();
let m = memory("undated here");
s.store_memory(&m).unwrap();
let superseder = memory("the newer fact");
s.store_memory(&superseder).unwrap();
s.supersede_memory(m.id, superseder.id).unwrap();
let before = s.get_memory(m.id).unwrap().unwrap();
let log_before = sync_log_rows(&s);
let outbox_before = outbox_rows(&s);
let mut stale = m.clone();
stale.session_at = Some(session("2026-07-23T09:00:00Z"));
stale.content = "stale content must not land".to_string();
let update = record(
SyncOp::Update,
SyncNodeType::Memory,
m.id.to_string(),
Some(serde_json::to_value(&stale).unwrap()),
);
assert!(matches!(
s.apply_record(&update).unwrap(),
ApplyOutcome::Skipped(SkipReason::StaleWrite)
));
let after = s.get_memory(m.id).unwrap().unwrap();
assert_eq!(after.session_at, stale.session_at);
assert_eq!(after.content, before.content);
assert_eq!(after.updated_at, before.updated_at);
assert_eq!(after.tier, before.tier);
assert_eq!(after.superseded_by, Some(superseder.id));
assert_eq!(sync_log_rows(&s), log_before, "a fill is not relogged");
assert_eq!(outbox_rows(&s), outbox_before, "a date is not indexed");
let mut other = stale.clone();
other.session_at = Some(session("2026-01-01T00:00:00Z"));
s.apply_record(&record(
SyncOp::Update,
SyncNodeType::Memory,
m.id.to_string(),
Some(serde_json::to_value(&other).unwrap()),
))
.unwrap();
assert_eq!(
s.get_memory(m.id).unwrap().unwrap().session_at,
stale.session_at,
"a losing update never overwrites a known date"
);
}
#[test]
fn batch_is_atomic_across_a_failing_record() {
let s = store();
let good = memory("good batch record");
let records = vec![
record(
SyncOp::Create,
SyncNodeType::Memory,
good.id.to_string(),
Some(serde_json::to_value(&good).unwrap()),
),
record(
SyncOp::Create,
SyncNodeType::Memory,
Uuid::new_v4().to_string(),
None,
),
];
assert!(s.apply_batch(&records).is_err());
assert!(
s.get_memory(good.id).unwrap().is_none(),
"a failed batch must roll back every record, including the good prefix"
);
assert!(
matches!(s.apply_record(&records[0]).unwrap(), ApplyOutcome::Created),
"per-record fallback after a rolled-back batch must not double-apply"
);
}
#[test]
fn relation_with_missing_endpoint_is_skipped() {
let s = store();
let a = memory("relation endpoint present");
s.store_memory(&a).unwrap();
let rel = Relation {
from_id: a.id,
to_id: Uuid::new_v4(),
relation_type: RelationType::RelatesTo,
strength: 1.0,
context: None,
};
let rec = record(
SyncOp::Create,
SyncNodeType::Relation,
format!("{}:{}:relates_to", rel.from_id, rel.to_id),
Some(serde_json::to_value(&rel).unwrap()),
);
assert!(matches!(
s.apply_record(&rec).unwrap(),
ApplyOutcome::Skipped(SkipReason::DanglingEndpoint)
));
}
fn inject_duplicate_delete(s: &SqliteStore, node_id: &str, machine_id: &str) -> i64 {
s.with_txn(|tx| {
let local_seq: i64 = tx.query_row(
"SELECT COALESCE(MAX(local_seq), 0) + 1 FROM sync_log",
[],
|r| r.get(0),
)?;
let id = format!("{machine_id}:{local_seq}");
tx.execute(
"INSERT INTO sync_log \
(id, local_seq, origin_machine_id, origin_seq, op, node_type, node_id, \
timestamp, data) \
VALUES (?1,?2,?3,?4,'delete','memory',?5,?6,NULL)",
params![
id,
local_seq,
machine_id,
local_seq,
node_id,
Utc::now().to_rfc3339()
],
)?;
Ok(local_seq)
})
.unwrap()
}
#[test]
fn compact_sync_log_collapses_duplicates_keeping_the_latest_and_leaving_max_seq_unchanged() {
let s = store();
let m = memory("consolidated raw memory, re-deleted every cleanup cycle");
s.store_memory(&m).unwrap();
s.delete_memory(m.id).unwrap();
assert_eq!(deletes_for(&s, m.id), 1, "precondition: one real delete");
let mut last_injected = 0;
for _ in 0..4 {
last_injected = inject_duplicate_delete(&s, &m.id.to_string(), "machine-a");
}
assert_eq!(
deletes_for(&s, m.id),
5,
"precondition: 1 real + 4 re-logged duplicates"
);
let max_before = s.max_sync_seq().unwrap();
assert_eq!(
max_before as i64, last_injected,
"precondition: the last injected duplicate holds the current global max"
);
let stats = s.compact_sync_log(&[SyncOp::Delete], false).unwrap();
assert_eq!(stats.groups_with_duplicates, 1);
assert_eq!(stats.rows_deleted, 4);
assert_eq!(stats.max_local_seq_before, stats.max_local_seq_after);
assert_eq!(s.max_sync_seq().unwrap(), max_before);
assert_eq!(
deletes_for(&s, m.id),
1,
"must collapse to exactly one delete row"
);
let surviving_seq: i64 = s
.reader()
.query_row(
"SELECT local_seq FROM sync_log WHERE node_id = ?1 AND op = 'delete'",
params![m.id.to_string()],
|r| r.get(0),
)
.unwrap();
assert_eq!(
surviving_seq, last_injected,
"the surviving row must be the highest local_seq, or the next write reuses \
a sequence a peer already considers seen"
);
}
#[test]
fn compact_sync_log_is_idempotent() {
let s = store();
let m = memory("re-deleted repeatedly");
s.store_memory(&m).unwrap();
s.delete_memory(m.id).unwrap();
for _ in 0..3 {
inject_duplicate_delete(&s, &m.id.to_string(), "machine-a");
}
let first = s.compact_sync_log(&[SyncOp::Delete], false).unwrap();
assert_eq!(first.rows_deleted, 3);
let second = s.compact_sync_log(&[SyncOp::Delete], false).unwrap();
assert_eq!(second.rows_deleted, 0);
assert_eq!(second.groups_with_duplicates, 0);
assert_eq!(deletes_for(&s, m.id), 1);
}
#[test]
fn compact_sync_log_dry_run_reports_without_deleting() {
let s = store();
let m = memory("dry-run candidate");
s.store_memory(&m).unwrap();
s.delete_memory(m.id).unwrap();
inject_duplicate_delete(&s, &m.id.to_string(), "machine-a");
inject_duplicate_delete(&s, &m.id.to_string(), "machine-a");
let stats = s.compact_sync_log(&[SyncOp::Delete], true).unwrap();
assert_eq!(stats.rows_deleted, 2, "dry run reports the victim count");
assert_eq!(
deletes_for(&s, m.id),
3,
"dry run must not actually delete anything"
);
}
#[test]
fn compact_sync_log_leaves_a_single_delete_and_its_create_untouched() {
let s = store();
let untouched = memory("created once, never deleted");
s.store_memory(&untouched).unwrap();
let deleted_once = memory("created once, deleted once, never duplicated");
s.store_memory(&deleted_once).unwrap();
s.delete_memory(deleted_once.id).unwrap();
let stats = s.compact_sync_log(&[SyncOp::Delete], false).unwrap();
assert_eq!(
stats.rows_deleted, 0,
"nothing here is a duplicate, compaction must be a no-op"
);
let creates = |id: Uuid| {
s.sync_log_since(0)
.unwrap()
.into_iter()
.filter(|e| e.op == SyncOp::Create && e.node_id == id.to_string())
.count()
};
assert_eq!(creates(untouched.id), 1);
assert_eq!(creates(deleted_once.id), 1);
assert_eq!(deletes_for(&s, deleted_once.id), 1);
}
#[test]
fn compacted_sync_log_replays_to_the_same_tombstone_set_as_uncompacted() {
let source = store();
let m = memory("replicated to two peers, one before compaction, one after");
source.store_memory(&m).unwrap();
source.delete_memory(m.id).unwrap();
for _ in 0..5 {
inject_duplicate_delete(&source, &m.id.to_string(), "machine-a");
}
let apply_all = |peer: &SqliteStore| {
for entry in source.sync_log_since(0).unwrap() {
let data = entry
.data
.as_ref()
.and_then(|d| serde_json::from_str::<serde_json::Value>(d).ok());
let rec = record(entry.op, entry.node_type, entry.node_id.clone(), data);
let _ = peer.apply_record(&rec);
}
};
let peer_before = SqliteStore::in_memory("machine-peer-before".to_string()).unwrap();
apply_all(&peer_before);
source.compact_sync_log(&[SyncOp::Delete], false).unwrap();
let peer_after = SqliteStore::in_memory("machine-peer-after".to_string()).unwrap();
apply_all(&peer_after);
assert_eq!(
peer_before.tombstone_exists(m.id).unwrap(),
peer_after.tombstone_exists(m.id).unwrap(),
"compaction must not change what a peer converges to"
);
assert!(
peer_after.tombstone_exists(m.id).unwrap(),
"precondition: the delete must actually have propagated"
);
}
#[test]
fn delete_import_is_idempotent_and_does_not_relog() {
let s = store();
let m = memory("locally deleted, then re-sent by a peer as a duplicate");
s.store_memory(&m).unwrap();
s.delete_memory(m.id).unwrap();
assert_eq!(deletes_for(&s, m.id), 1, "precondition: one real delete");
let outbox_before = outbox_rows(&s);
let rec = record(SyncOp::Delete, SyncNodeType::Memory, m.id.to_string(), None);
assert_eq!(
s.apply_record(&rec).unwrap(),
ApplyOutcome::Skipped(SkipReason::Tombstoned),
"a delete for an already-tombstoned node must be skipped, matching \
how Create and Update already treat AlreadyPresent/Tombstoned"
);
assert_eq!(
deletes_for(&s, m.id),
1,
"a redundant delete import must not mint a new sync_log row"
);
assert_eq!(
outbox_rows(&s),
outbox_before,
"a redundant delete import must not requeue an index eviction"
);
}
#[test]
fn compacted_delete_does_not_regrow_when_the_same_record_is_reimported() {
let s = store();
let m = memory("regression test for the 2026-08-24 sync_log reinflation incident");
s.store_memory(&m).unwrap();
s.delete_memory(m.id).unwrap();
for _ in 0..4 {
inject_duplicate_delete(&s, &m.id.to_string(), "machine-a");
}
assert_eq!(
deletes_for(&s, m.id),
5,
"precondition: 1 real + 4 duplicates"
);
let pre_compaction_entries: Vec<_> = s
.sync_log_since(0)
.unwrap()
.into_iter()
.filter(|e| e.op == SyncOp::Delete && e.node_id == m.id.to_string())
.collect();
let stats = s.compact_sync_log(&[SyncOp::Delete], false).unwrap();
assert_eq!(stats.rows_deleted, 4);
assert_eq!(
deletes_for(&s, m.id),
1,
"precondition: compaction collapsed to one row"
);
for entry in &pre_compaction_entries {
let data = entry
.data
.as_ref()
.and_then(|d| serde_json::from_str::<serde_json::Value>(d).ok());
let rec = SyncRecord {
local_seq: entry.local_seq as u64,
origin_machine_id: entry.origin_machine_id.clone(),
origin_seq: entry.origin_seq as u64,
op: entry.op,
node_type: entry.node_type,
node_id: entry.node_id.clone(),
timestamp: entry.timestamp,
data,
};
let outcome = s.apply_record(&rec).unwrap();
assert!(
matches!(outcome, ApplyOutcome::Skipped(SkipReason::Tombstoned)),
"a compacted-away duplicate must be rejected on reimport, not re-logged; got {outcome:?}"
);
}
assert_eq!(
deletes_for(&s, m.id),
1,
"reimporting compacted-away duplicates must not regrow sync_log"
);
}
#[test]
fn backfill_tombstones_after_compaction_finds_nothing_to_backfill() {
let s = store();
let m = memory("compacted then checked for backfill gaps");
s.store_memory(&m).unwrap();
s.delete_memory(m.id).unwrap();
for _ in 0..2 {
inject_duplicate_delete(&s, &m.id.to_string(), "machine-a");
}
s.compact_sync_log(&[SyncOp::Delete], false).unwrap();
assert_eq!(
s.backfill_tombstones().unwrap(),
0,
"the compacted store must still have exactly one delete entry per tombstone"
);
}
#[test]
fn pending_distillation_counts_waiting_raws_and_dates_the_oldest() {
let s = store();
assert_eq!(
s.pending_distillation().unwrap(),
PendingDistillation::default()
);
let mut older = explicit("an older transcript chunk", "claude-code");
older.created_at = Utc::now() - chrono::Duration::days(3);
let newer = explicit("a newer transcript chunk", "claude-code");
let note = explicit("remembered on purpose", "mcp-direct");
let dist = explicit("a paraphrase", "consolidation");
for m in [&older, &newer, ¬e, &dist] {
s.store_memory(m).unwrap();
}
assert_eq!(
s.pending_distillation().unwrap(),
PendingDistillation {
count: 2,
oldest_created_at: Some(older.created_at),
}
);
s.mark_consolidated(older.id, dist.id, "model").unwrap();
assert_eq!(
s.pending_distillation().unwrap(),
PendingDistillation {
count: 1,
oldest_created_at: Some(newer.created_at),
}
);
}
}