use std::collections::BTreeSet;
use rusqlite::Connection;
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub enum ConstraintOrigin {
PrimaryKey,
UniqueConstraint,
UniqueIndex,
}
impl ConstraintOrigin {
fn parse(origin: &str) -> Option<Self> {
match origin {
"pk" => Some(Self::PrimaryKey),
"u" => Some(Self::UniqueConstraint),
"c" => Some(Self::UniqueIndex),
_ => None,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
pub struct NamespaceTable {
pub name: String,
pub virtual_table: bool,
}
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
pub struct NamespaceConstraint {
pub table: String,
pub index: String,
pub origin: ConstraintOrigin,
pub columns: Vec<Option<String>>,
pub partial: bool,
}
impl NamespaceConstraint {
pub fn columns_are_nameable(&self) -> bool {
self.columns.iter().all(Option::is_some)
}
}
#[derive(Debug, Clone, Default)]
pub struct NamespaceCensus {
pub database: String,
pub tables: Vec<NamespaceTable>,
pub constraints: Vec<NamespaceConstraint>,
pub unenumerable: Vec<String>,
}
impl NamespaceCensus {
pub fn table_names(&self) -> Vec<&str> {
self.tables.iter().map(|t| t.name.as_str()).collect()
}
pub fn vector_tables(&self) -> Vec<&str> {
self.tables
.iter()
.filter(|t| t.name.starts_with("vec_"))
.map(|t| t.name.as_str())
.collect()
}
pub fn constraints_on(&self, table: &str) -> Vec<&NamespaceConstraint> {
self.constraints
.iter()
.filter(|c| c.table == table)
.collect()
}
}
pub(crate) fn quote_ident(name: &str) -> String {
format!("\"{}\"", name.replace('"', "\"\""))
}
pub fn namespace_tables(conn: &Connection) -> rusqlite::Result<Vec<NamespaceTable>> {
let mut stmt = conn.prepare(
"SELECT name, COALESCE(sql, '') FROM sqlite_master \
WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name",
)?;
let candidates: Vec<(String, String)> = stmt
.query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
.collect::<rusqlite::Result<_>>()?;
let mut tables = Vec::new();
for (name, sql) in candidates {
if !table_has_namespace_column(conn, &name)? {
continue;
}
let mut head = sql.split_whitespace();
let virtual_table = matches!(
(head.next(), head.next(), head.next()),
(Some(create), Some(virt), Some(table))
if create.eq_ignore_ascii_case("CREATE")
&& virt.eq_ignore_ascii_case("VIRTUAL")
&& table.eq_ignore_ascii_case("TABLE")
);
tables.push(NamespaceTable {
name,
virtual_table,
});
}
Ok(tables)
}
fn table_has_namespace_column(conn: &Connection, table: &str) -> rusqlite::Result<bool> {
let sql = format!("PRAGMA table_info({})", quote_ident(table));
let mut stmt = conn.prepare(&sql)?;
let mut rows = stmt.query([])?;
while let Some(row) = rows.next()? {
let column: String = row.get(1)?;
if column.eq_ignore_ascii_case("namespace") {
return Ok(true);
}
}
Ok(false)
}
pub fn census(conn: &Connection) -> rusqlite::Result<NamespaceCensus> {
let tables = namespace_tables(conn)?;
let mut constraints = Vec::new();
let mut unenumerable = Vec::new();
for table in &tables {
match unique_constraints_naming_namespace(conn, &table.name) {
Ok(found) if found.is_empty() && table.virtual_table => {
unenumerable.push(table.name.clone())
}
Ok(found) => constraints.extend(found),
Err(_) if table.virtual_table => unenumerable.push(table.name.clone()),
Err(error) => return Err(error),
}
}
constraints.sort();
Ok(NamespaceCensus {
database: main_database_file(conn)?,
tables,
constraints,
unenumerable,
})
}
fn main_database_file(conn: &Connection) -> rusqlite::Result<String> {
let mut stmt = conn.prepare("PRAGMA database_list")?;
let mut rows = stmt.query([])?;
while let Some(row) = rows.next()? {
let name: String = row.get(1)?;
if name == "main" {
return Ok(row.get::<_, Option<String>>(2)?.unwrap_or_default());
}
}
Ok(String::new())
}
fn unique_constraints_naming_namespace(
conn: &Connection,
table: &str,
) -> rusqlite::Result<Vec<NamespaceConstraint>> {
let list_sql = format!("PRAGMA index_list({})", quote_ident(table));
let mut list = conn.prepare(&list_sql)?;
let indexes: Vec<(String, i64, String, i64)> = list
.query_map([], |row| {
Ok((
row.get::<_, String>(1)?,
row.get::<_, i64>(2)?,
row.get::<_, String>(3)?,
row.get::<_, i64>(4)?,
))
})?
.collect::<rusqlite::Result<_>>()?;
let mut found = Vec::new();
for (index, unique, origin, partial) in indexes {
if unique == 0 {
continue;
}
let Some(origin) = ConstraintOrigin::parse(&origin) else {
continue;
};
let columns = index_key_columns(conn, &index)?;
let named = columns.iter().any(|c| {
c.as_deref()
.is_some_and(|c| c.eq_ignore_ascii_case("namespace"))
});
let named = named
|| (columns.iter().any(Option::is_none) && index_ddl_names_namespace(conn, &index)?);
if !named {
continue;
}
found.push(NamespaceConstraint {
table: table.to_string(),
index,
origin,
columns,
partial: partial != 0,
});
}
Ok(found)
}
fn index_ddl_names_namespace(conn: &Connection, index: &str) -> rusqlite::Result<bool> {
let ddl: Option<String> = conn
.query_row(
"SELECT sql FROM sqlite_master WHERE type = 'index' AND name = ?1",
[index],
|row| row.get(0),
)
.unwrap_or(None);
Ok(ddl.is_some_and(|ddl| mentions_namespace_as_a_word(&ddl)))
}
fn mentions_namespace_as_a_word(text: &str) -> bool {
const NEEDLE: &str = "namespace";
let lower = text.to_ascii_lowercase();
let bytes = lower.as_bytes();
let mut from = 0;
while let Some(offset) = lower[from..].find(NEEDLE) {
let start = from + offset;
let end = start + NEEDLE.len();
let before_ok = start == 0 || !is_ident_byte(bytes[start - 1]);
let after_ok = end == bytes.len() || !is_ident_byte(bytes[end]);
if before_ok && after_ok {
return true;
}
from = start + 1;
}
false
}
fn is_ident_byte(byte: u8) -> bool {
byte.is_ascii_alphanumeric() || byte == b'_'
}
fn index_key_columns(conn: &Connection, index: &str) -> rusqlite::Result<Vec<Option<String>>> {
let sql = format!("PRAGMA index_xinfo({})", quote_ident(index));
let mut stmt = conn.prepare(&sql)?;
let rows = stmt.query_map([], |row| {
Ok((row.get::<_, Option<String>>(2)?, row.get::<_, i64>(5)?))
})?;
let mut columns = Vec::new();
for row in rows {
let (name, key) = row?;
if key == 1 {
columns.push(name);
}
}
Ok(columns)
}
pub const TABLES_EXCLUDED_FROM_MOVE: &[&str] =
&["events", "ann_consumer_watermark", "ann_consumer_pending"];
pub const TABLE_REFUSED_BY_SCHEMA: &str = "note_streams";
pub fn reachable_constraints(census: &NamespaceCensus) -> Vec<&NamespaceConstraint> {
let excluded: BTreeSet<&str> = TABLES_EXCLUDED_FROM_MOVE
.iter()
.copied()
.chain(std::iter::once(TABLE_REFUSED_BY_SCHEMA))
.collect();
census
.constraints
.iter()
.filter(|c| !excluded.contains(c.table.as_str()))
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
use crate::migrations::run_migrations;
fn migrated() -> Connection {
let mut conn = Connection::open_in_memory().expect("in-memory connection");
run_migrations(&mut conn).expect("migrate to the current schema");
conn
}
fn constraint_names(census: &NamespaceCensus) -> BTreeSet<(String, String)> {
census
.constraints
.iter()
.map(|c| (c.table.clone(), c.index.clone()))
.collect()
}
#[test]
fn census_finds_every_namespace_bearing_uniqueness_constraint() {
let conn = migrated();
let report = super::census(&conn).expect("census");
let found = constraint_names(&report);
let by_index: BTreeSet<&str> = found.iter().map(|(_, i)| i.as_str()).collect();
for expected in [
"idx_notes_namespace_kind_key",
"idx_comm_message_external_id",
"idx_graph_edges_unique_triple",
"idx_knowledge_atoms_ns_slug",
"idx_knowledge_domains_ns_slug",
"idx_brain_serve_ledger_unique",
] {
assert!(
by_index.contains(expected),
"census missed the unique index {expected}; found {by_index:?}"
);
}
let by_table: BTreeSet<&str> = found.iter().map(|(t, _)| t.as_str()).collect();
for expected in [
"graph_edges",
"brain_implicit_mass",
"brain_profile_snapshots",
"ann_consumer_watermark",
"ann_consumer_pending",
"note_streams",
"fts_notes_rowids",
"fts_entities_rowids",
] {
assert!(
by_table.contains(expected),
"census missed a namespace-bearing primary key on {expected}; found {by_table:?}"
);
}
}
#[test]
fn an_expression_index_is_found_and_reports_its_expression_as_unnameable() {
let conn = migrated();
let report = super::census(&conn).expect("census");
let external_id = report
.constraints
.iter()
.find(|c| c.index == "idx_comm_message_external_id")
.expect("the comm external-id index is a namespace-bearing unique index");
assert_eq!(external_id.table, "notes");
assert_eq!(external_id.origin, ConstraintOrigin::UniqueIndex);
assert!(
external_id.partial,
"the index is filtered, and a collision check ignoring that refuses moves SQLite accepts"
);
assert!(
!external_id.columns_are_nameable(),
"the third key is json_extract(...), which has no column name: {:?}",
external_id.columns
);
assert!(
external_id
.columns
.iter()
.any(|c| c.as_deref() == Some("namespace")),
"namespace is still named inside an expression index: {:?}",
external_id.columns
);
}
#[test]
fn an_index_whose_namespace_is_inside_an_expression_is_found_from_its_own_ddl() {
let conn = migrated();
conn.execute_batch(
"CREATE UNIQUE INDEX idx_expr_ns ON notes(lower(namespace), kind, name)",
)
.expect("an index whose namespace sits inside an expression");
let columns = index_key_columns(&conn, "idx_expr_ns").expect("xinfo");
assert!(
!columns.iter().any(|c| c.as_deref() == Some("namespace")),
"control: index_xinfo must not name namespace here, got {columns:?}"
);
let report = super::census(&conn).expect("census");
assert!(
report.constraints.iter().any(|c| c.index == "idx_expr_ns"),
"an index carrying an expression is read from its own DDL"
);
}
#[test]
fn a_namespace_shaped_column_name_does_not_match_the_ddl_read() {
assert!(mentions_namespace_as_a_word(
"CREATE UNIQUE INDEX i ON t(lower(namespace), id)"
));
assert!(mentions_namespace_as_a_word("ON t(NAMESPACE)"));
assert!(!mentions_namespace_as_a_word(
"CREATE UNIQUE INDEX i ON t(lower(namespace_hash), id)"
));
assert!(!mentions_namespace_as_a_word("ON t(ns_namespace_key)"));
}
#[test]
fn a_composite_primary_key_reports_as_a_constraint_with_its_columns() {
let conn = migrated();
let report = super::census(&conn).expect("census");
let edges = report
.constraints_on("graph_edges")
.into_iter()
.find(|c| c.origin == ConstraintOrigin::PrimaryKey)
.expect("graph_edges is PRIMARY KEY (namespace, id)");
assert_eq!(
edges.columns,
vec![Some("namespace".to_string()), Some("id".to_string())]
);
}
#[test]
fn a_constraint_added_after_this_code_was_written_is_found_with_no_code_change() {
let conn = migrated();
let before = constraint_names(&super::census(&conn).expect("census"));
assert!(
!before.iter().any(|(_, i)| i == "idx_future_ns_status"),
"control: the index under test must not already exist"
);
conn.execute_batch(
"CREATE UNIQUE INDEX idx_future_ns_status ON notes(namespace, status, name)",
)
.expect("a later migration adds a namespace-bearing unique index");
let after = constraint_names(&super::census(&conn).expect("census"));
assert!(
after
.iter()
.any(|(t, i)| t == "notes" && i == "idx_future_ns_status"),
"the census has to find a constraint nobody told it about; found {after:?}"
);
assert_eq!(
after.len(),
before.len() + 1,
"and it must find exactly the one that was added"
);
}
#[test]
fn a_non_unique_index_naming_namespace_is_not_a_constraint() {
let conn = migrated();
let before = super::census(&conn).expect("census").constraints.len();
conn.execute_batch("CREATE INDEX idx_plain_ns_salience ON notes(namespace, salience)")
.expect("plain index");
let after = super::census(&conn).expect("census").constraints.len();
assert_eq!(
after, before,
"a non-unique index is not a uniqueness constraint"
);
}
#[test]
fn the_reachable_set_excludes_the_tables_a_move_never_writes() {
let conn = migrated();
let report = super::census(&conn).expect("census");
let reachable: BTreeSet<&str> = reachable_constraints(&report)
.into_iter()
.map(|c| c.table.as_str())
.collect();
for out_of_reach in [
"note_streams",
"ann_consumer_watermark",
"ann_consumer_pending",
] {
assert!(
report.constraints.iter().any(|c| c.table == out_of_reach),
"control: {out_of_reach} must be IN the census, or this arm proves nothing"
);
assert!(
!reachable.contains(out_of_reach),
"{out_of_reach} is out of reach for a move"
);
}
for in_reach in [
"notes",
"graph_edges",
"fts_notes_rowids",
"brain_serve_ledger",
] {
assert!(reachable.contains(in_reach), "{in_reach} is reachable");
}
}
#[test]
fn a_census_names_the_database_it_read() {
let conn = migrated();
let report = super::census(&conn).expect("census");
assert_eq!(
report.database, "",
"an in-memory database has no file, and the empty string is that answer"
);
let dir = tempfile::tempdir().expect("tempdir");
let path = dir.path().join("second-backend.db");
let mut file_conn = Connection::open(&path).expect("open a file-backed store");
run_migrations(&mut file_conn).expect("migrate the second backend");
let file_census = super::census(&file_conn).expect("census");
let resolved = std::fs::canonicalize(&path).expect("resolve the store path");
let reported = std::fs::canonicalize(&file_census.database)
.expect("the census names a path that exists");
assert_eq!(reported, resolved, "a file-backed census names its file");
}
#[test]
fn a_virtual_table_reporting_no_indexes_is_recorded_as_unread_not_as_clean() {
let conn = migrated();
let report = super::census(&conn).expect("census");
assert!(
report.unenumerable.iter().any(|t| t == "fts_notes"),
"fts_notes reports no index list, so its constraints are unread: {:?}",
report.unenumerable
);
assert!(
!report.unenumerable.iter().any(|t| t == "notes"),
"control: an ordinary table's constraints ARE readable, so it is not listed"
);
}
#[test]
fn the_table_census_finds_the_virtual_tables_and_marks_them() {
let conn = migrated();
let report = super::census(&conn).expect("census");
let names = report.table_names();
for expected in [
"notes",
"entities",
"graph_edges",
"knowledge_atoms",
"events",
] {
assert!(
names.contains(&expected),
"missing {expected} from {names:?}"
);
}
let virtual_tables: Vec<&str> = report
.tables
.iter()
.filter(|t| t.virtual_table)
.map(|t| t.name.as_str())
.collect();
println!("namespace-bearing virtual tables: {virtual_tables:?}");
assert_eq!(
virtual_tables,
["fts_entities", "fts_knowledge", "fts_notes", "fts_sections"],
"the namespace-bearing virtual tables of a freshly migrated store"
);
let notes = report
.tables
.iter()
.find(|t| t.name == "notes")
.expect("notes");
assert!(
!notes.virtual_table,
"control: an ordinary table is not marked virtual"
);
}
}