use std::collections::HashMap;
use std::collections::hash_map::Entry;
use anyhow::{Context, Result};
use chrono::DateTime;
use rusqlite::types::Value;
use rusqlite::{OptionalExtension, Statement, Transaction, params, params_from_iter};
struct AuthoredTable {
node_type: &'static str,
keys: &'static str,
update: &'static str,
}
const AUTHORED_TABLES: &[AuthoredTable] = &[
AuthoredTable {
node_type: "entity",
keys: "SELECT id FROM entities",
update: "UPDATE entities SET updated_at = ?1, updated_by = ?2 WHERE id = ?3",
},
AuthoredTable {
node_type: "conversation",
keys: "SELECT id FROM conversations",
update: "UPDATE conversations SET updated_at = ?1, updated_by = ?2 WHERE id = ?3",
},
AuthoredTable {
node_type: "relation",
keys: "SELECT from_id, to_id, relation_type FROM relations",
update: "UPDATE relations SET updated_at = ?1, updated_by = ?2 \
WHERE from_id = ?3 AND to_id = ?4 AND relation_type = ?5",
},
AuthoredTable {
node_type: "distillation",
keys: "SELECT memory_id FROM distillation_log",
update: "UPDATE distillation_log SET updated_at = ?1, updated_by = ?2 WHERE memory_id = ?3",
},
];
pub(super) fn seed(tx: &Transaction<'_>) -> Result<()> {
seed_authored_columns(tx)?;
seed_type_tombstones(tx)?;
raise_authored_clock(tx)
}
fn raise_authored_clock(tx: &Transaction<'_>) -> Result<()> {
let newest: Option<i64> = tx.query_row(
"SELECT MAX(m) FROM ( \
SELECT MAX(updated_at) AS m FROM entities \
UNION ALL SELECT MAX(updated_at) FROM conversations \
UNION ALL SELECT MAX(updated_at) FROM relations \
UNION ALL SELECT MAX(updated_at) FROM distillation_log \
UNION ALL SELECT MAX(deleted_at) FROM type_tombstones)",
[],
|r| r.get(0),
)?;
match newest {
Some(micros) => super::clock::observe_authored(tx, micros),
None => Ok(()),
}
}
fn seed_authored_columns(tx: &Transaction<'_>) -> Result<()> {
let mut newest = NewestWrite::prepare(tx)?;
for table in AUTHORED_TABLES {
let keys = row_keys(tx, table.keys)?;
let mut update = tx.prepare(table.update)?;
for key in keys {
let Some((updated_at, updated_by)) = newest.find(table.node_type, &key.join(":"))?
else {
continue;
};
let values = [Value::Integer(updated_at), Value::Text(updated_by)]
.into_iter()
.chain(key.into_iter().map(Value::Text));
update
.execute(params_from_iter(values))
.with_context(|| format!("seeding authored time for a {}", table.node_type))?;
}
}
Ok(())
}
fn row_keys(tx: &Transaction<'_>, sql: &str) -> Result<Vec<Vec<String>>> {
let mut stmt = tx.prepare(sql)?;
let width = stmt.column_count();
let mut rows = stmt.query([])?;
let mut keys = Vec::new();
while let Some(row) = rows.next()? {
let key = (0..width)
.map(|i| row.get(i))
.collect::<rusqlite::Result<Option<Vec<String>>>>()?;
keys.extend(key);
}
Ok(keys)
}
struct NewestWrite<'t> {
seq_by_op: Statement<'t>,
row: Statement<'t>,
}
impl<'t> NewestWrite<'t> {
fn prepare(tx: &'t Transaction<'_>) -> Result<Self> {
Ok(Self {
seq_by_op: tx.prepare(
"SELECT local_seq FROM sync_log WHERE node_type = ?1 AND node_id = ?2 AND op = ?3 \
ORDER BY local_seq DESC LIMIT 1",
)?,
row: tx.prepare(
"SELECT timestamp, origin_machine_id, data FROM sync_log WHERE local_seq = ?1",
)?,
})
}
fn find(&mut self, node_type: &str, node_id: &str) -> Result<Option<(i64, String)>> {
let mut newest: Option<i64> = None;
for op in ["create", "update"] {
let seq = self
.seq_by_op
.query_row(params![node_type, node_id, op], |r| r.get(0))
.optional()?;
newest = newest.max(seq);
}
let Some(seq) = newest else {
return Ok(None);
};
let (timestamp, origin, data): (String, String, Option<String>) = self
.row
.query_row(params![seq], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))?;
let data = parse_data(data.as_deref());
Ok(Some((micros(×tamp), author(data.as_ref(), origin))))
}
}
fn seed_type_tombstones(tx: &Transaction<'_>) -> Result<()> {
let newest = newest_deletes(tx)?;
let mut live = LiveRow::prepare(tx)?;
let mut insert = tx.prepare(
"INSERT INTO type_tombstones (node_type, node_id, deleted_at, origin) \
VALUES (?1, ?2, ?3, ?4)",
)?;
for ((node_type, node_id), (deleted_at, origin)) in newest {
if !live.exists(&node_type, &node_id)? {
insert
.execute(params![node_type, node_id, deleted_at, origin])
.with_context(|| format!("seeding a {node_type} tombstone"))?;
}
}
Ok(())
}
fn newest_deletes(tx: &Transaction<'_>) -> Result<HashMap<(String, String), (i64, String)>> {
let mut stmt = tx.prepare(
"SELECT node_type, node_id, timestamp, origin_machine_id, data FROM sync_log \
WHERE op = 'delete' AND node_type IN ('entity', 'conversation', 'relation', 'distillation')",
)?;
let mut rows = stmt.query([])?;
let mut newest: HashMap<(String, String), (i64, String)> = HashMap::new();
while let Some(row) = rows.next()? {
let node_type: String = row.get(0)?;
let node_id: String = row.get(1)?;
let timestamp: String = row.get(2)?;
let origin: String = row.get(3)?;
let data = parse_data(row.get::<_, Option<String>>(4)?.as_deref());
let node_id = if node_type == "distillation" {
match data.as_ref().and_then(|d| d.get("distilled_id")) {
Some(serde_json::Value::Null) => format!("{node_id}:"),
Some(serde_json::Value::String(distilled_id)) => {
format!("{node_id}:{distilled_id}")
}
_ => continue,
}
} else {
node_id
};
let candidate = (micros(×tamp), author(data.as_ref(), origin));
match newest.entry((node_type, node_id)) {
Entry::Occupied(mut kept) => {
if candidate > *kept.get() {
kept.insert(candidate);
}
}
Entry::Vacant(slot) => {
slot.insert(candidate);
}
}
}
Ok(newest)
}
struct LiveRow<'t> {
entity: Statement<'t>,
conversation: Statement<'t>,
relation: Statement<'t>,
marker: Statement<'t>,
}
impl<'t> LiveRow<'t> {
fn prepare(tx: &'t Transaction<'_>) -> Result<Self> {
Ok(Self {
entity: tx.prepare("SELECT 1 FROM entities WHERE id = ?1")?,
conversation: tx.prepare("SELECT 1 FROM conversations WHERE id = ?1")?,
relation: tx.prepare(
"SELECT 1 FROM relations WHERE from_id = ?1 AND to_id = ?2 AND relation_type = ?3",
)?,
marker: tx.prepare(
"SELECT 1 FROM distillation_log WHERE memory_id = ?1 AND distilled_id IS ?2",
)?,
})
}
fn exists(&mut self, node_type: &str, node_id: &str) -> Result<bool> {
let found = match node_type {
"entity" => self.entity.exists(params![node_id])?,
"conversation" => self.conversation.exists(params![node_id])?,
"relation" => {
let mut parts = node_id.splitn(3, ':');
match (parts.next(), parts.next(), parts.next()) {
(Some(from_id), Some(to_id), Some(relation_type)) => self
.relation
.exists(params![from_id, to_id, relation_type])?,
_ => false,
}
}
"distillation" => {
let (memory_id, distilled_id) = node_id.split_once(':').unwrap_or((node_id, ""));
let distilled_id = (!distilled_id.is_empty()).then_some(distilled_id);
self.marker.exists(params![memory_id, distilled_id])?
}
_ => false,
};
Ok(found)
}
}
fn parse_data(data: Option<&str>) -> Option<serde_json::Value> {
data.and_then(|d| serde_json::from_str(d).ok())
}
fn micros(timestamp: &str) -> i64 {
DateTime::parse_from_rfc3339(timestamp).map_or(0, |t| t.timestamp_micros())
}
fn author(data: Option<&serde_json::Value>, origin: String) -> String {
data.and_then(|d| d.get("updated_by"))
.and_then(serde_json::Value::as_str)
.map_or(origin, str::to_owned)
}
#[cfg(test)]
mod tests {
use rusqlite::{Connection, params};
use super::super::ddl::{SCHEMA_VERSION, create_at_version, init_schema, schema_version};
const RELATION_R1: &str = r#"{"from_id":"R1F","to_id":"R1T","relation_type":"related_to"}"#;
type LogRow = (
i64,
&'static str,
&'static str,
&'static str,
&'static str,
&'static str,
Option<&'static str>,
);
const LOG: &[LogRow] = &[
(
1,
"pi",
"create",
"entity",
"E1",
"2026-01-01T00:00:00+00:00",
Some(r#"{"name":"rust"}"#),
),
(
2,
"mac",
"update",
"conversation",
"C1",
"2026-01-03T00:00:00+00:00",
None,
),
(
3,
"pi",
"update",
"entity",
"E1",
"2026-01-02T00:00:00.000001+00:00",
Some(r#"{"updated_by":"laptop"}"#),
),
(
4,
"pi",
"delete",
"relation",
"R1F:R1T:related_to",
"2026-01-05T00:00:00+00:00",
Some(RELATION_R1),
),
(
5,
"desk",
"create",
"conversation",
"C1",
"2026-01-01T12:00:00Z",
None,
),
(
6,
"pi",
"create",
"relation",
"R1F:R1T:related_to",
"not a time",
Some(RELATION_R1),
),
(
7,
"laptop",
"create",
"distillation",
"M1",
"2026-01-04T00:00:00+00:00",
Some(r#"{"memory_id":"M1","distilled_id":"D1","updated_by":"mac"}"#),
),
(
8,
"pi",
"delete",
"entity",
"E9",
"2026-01-06T00:00:00+00:00",
None,
),
(
9,
"mac",
"delete",
"entity",
"E9",
"2026-01-05T00:00:00+00:00",
Some(r#"{"updated_by":"zed"}"#),
),
(
10,
"desk",
"delete",
"conversation",
"C9",
"2026-01-07T00:00:00+00:00",
Some(r#"{"updated_by":"laptop"}"#),
),
(
11,
"pi",
"delete",
"conversation",
"C9",
"2026-01-07T00:00:00+00:00",
None,
),
(
12,
"pi",
"delete",
"distillation",
"M2",
"2026-01-08T00:00:00+00:00",
None,
),
(
13,
"pi",
"delete",
"distillation",
"M3",
"2026-01-08T00:00:00+00:00",
Some(r#"{"memory_id":"M3","distilled_id":null,"updated_by":"mac"}"#),
),
(
14,
"pi",
"delete",
"distillation",
"M1",
"2026-01-03T00:00:00+00:00",
Some(r#"{"memory_id":"M1","distilled_id":"D0"}"#),
),
(
15,
"pi",
"delete",
"distillation",
"M1",
"2026-01-02T00:00:00+00:00",
Some(r#"{"memory_id":"M1","distilled_id":"D1"}"#),
),
(
16,
"pi",
"delete",
"memory",
"MEM1",
"2026-01-08T00:00:00+00:00",
None,
),
(
17,
"pi",
"delete",
"relation",
"A:B:contradicts",
"2026-01-09T00:00:00+00:00",
Some(r#"{"from_id":"A","to_id":"B","relation_type":"contradicts"}"#),
),
(
18,
"pi",
"delete",
"distillation",
"M4",
"2026-01-08T00:00:00+00:00",
Some(r#"{"memory_id":"M4"}"#),
),
];
fn migrated_store() -> Connection {
let conn = Connection::open_in_memory().unwrap();
create_at_version(&conn, 9).unwrap();
conn.execute_batch(
"INSERT INTO entities (id, name, entity_type) VALUES ('E1', 'rust', 'topic');
INSERT INTO entities (id, name, entity_type) VALUES ('E2', 'sqlite', 'topic');
INSERT INTO conversations (id, source, machine_id, started_at)
VALUES ('C1', 'claude-code', 'desk', '2026-01-01T00:00:00Z');
INSERT INTO relations (from_id, to_id, relation_type, strength)
VALUES ('R1F', 'R1T', 'related_to', 0.5);
INSERT INTO distillation_log (memory_id, distilled_id, status)
VALUES ('M1', 'D1', 'done');",
)
.unwrap();
for (seq, origin, op, node_type, node_id, timestamp, data) in LOG {
conn.execute(
"INSERT INTO sync_log (id, local_seq, origin_machine_id, origin_seq, op, \
node_type, node_id, timestamp, data) VALUES (?1, ?2, ?3, ?2, ?4, ?5, ?6, ?7, ?8)",
params![
format!("{origin}:{seq}"),
seq,
origin,
op,
node_type,
node_id,
timestamp,
data
],
)
.unwrap();
}
init_schema(&conn).unwrap();
assert_eq!(schema_version(&conn).unwrap(), SCHEMA_VERSION);
conn
}
fn authored(conn: &Connection, sql: &str) -> Vec<(String, i64, String)> {
let mut stmt = conn.prepare(sql).unwrap();
stmt.query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))
.unwrap()
.collect::<rusqlite::Result<_>>()
.unwrap()
}
fn row(id: &str, updated_at: i64, updated_by: &str) -> (String, i64, String) {
(id.to_string(), updated_at, updated_by.to_string())
}
#[test]
fn rows_take_the_time_and_author_of_their_newest_logged_write() {
let conn = migrated_store();
assert_eq!(
authored(
&conn,
"SELECT id, updated_at, updated_by FROM entities ORDER BY id"
),
vec![row("E1", 1_767_312_000_000_001, "laptop"), row("E2", 0, "")]
);
assert_eq!(
authored(
&conn,
"SELECT id, updated_at, updated_by FROM conversations"
),
vec![row("C1", 1_767_268_800_000_000, "desk")]
);
assert_eq!(
authored(
&conn,
"SELECT from_id || ':' || to_id || ':' || relation_type, updated_at, updated_by \
FROM relations"
),
vec![row("R1F:R1T:related_to", 0, "pi")]
);
assert_eq!(
authored(
&conn,
"SELECT memory_id, updated_at, updated_by FROM distillation_log"
),
vec![row("M1", 1_767_484_800_000_000, "mac")]
);
}
#[test]
fn the_next_local_write_is_authored_after_every_seeded_stamp() {
let conn = migrated_store();
let authored: String = conn
.query_row(
"SELECT value FROM index_meta WHERE key = 'clock_authored_last'",
[],
|r| r.get(0),
)
.unwrap();
assert_eq!(authored, "1767916800000000");
}
#[test]
fn a_store_created_before_the_lookup_index_gets_it() {
let conn = Connection::open_in_memory().unwrap();
create_at_version(&conn, 9).unwrap();
conn.execute("DROP INDEX idx_sync_log_type_id_op_seq", [])
.unwrap();
init_schema(&conn).unwrap();
let indexed: i64 = conn
.query_row(
"SELECT COUNT(*) FROM sqlite_master \
WHERE type = 'index' AND name = 'idx_sync_log_type_id_op_seq'",
[],
|r| r.get(0),
)
.unwrap();
assert_eq!(indexed, 1);
}
#[test]
fn a_legacy_row_with_a_null_key_does_not_stop_the_seed() {
let conn = Connection::open_in_memory().unwrap();
create_at_version(&conn, 9).unwrap();
conn.execute(
"INSERT INTO entities (id, name, entity_type) VALUES (NULL, 'orphan', 'topic')",
[],
)
.unwrap();
init_schema(&conn).unwrap();
assert_eq!(schema_version(&conn).unwrap(), SCHEMA_VERSION);
}
#[test]
fn only_deletes_without_a_live_row_become_type_tombstones() {
let conn = migrated_store();
let mut stmt = conn
.prepare(
"SELECT node_type, node_id, deleted_at, origin FROM type_tombstones \
ORDER BY node_type, node_id",
)
.unwrap();
let tombstones: Vec<(String, String, i64, String)> = stmt
.query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)))
.unwrap()
.collect::<rusqlite::Result<_>>()
.unwrap();
let tombstone = |node_type: &str, node_id: &str, deleted_at: i64, origin: &str| {
(
node_type.to_string(),
node_id.to_string(),
deleted_at,
origin.to_string(),
)
};
assert_eq!(
tombstones,
vec![
tombstone("conversation", "C9", 1_767_744_000_000_000, "pi"),
tombstone("distillation", "M1:D0", 1_767_398_400_000_000, "pi"),
tombstone("distillation", "M3:", 1_767_830_400_000_000, "mac"),
tombstone("entity", "E9", 1_767_657_600_000_000, "pi"),
tombstone("relation", "A:B:contradicts", 1_767_916_800_000_000, "pi"),
]
);
}
}