macrame/schema/ddl.rs
1//! DDL statements for the Macrame bitemporal schema as specified in §4.
2
3/// GLOB pattern matching the canonical timestamp form `YYYY-MM-DDTHH:MM:SS.ffffffZ`.
4///
5/// A macro rather than a `const` so it can be spliced into the DDL literals by
6/// `concat!`, which only accepts literals. Kept byte-identical to
7/// [`crate::util::timestamp::CANONICAL_TS_GLOB`] by the unit test at the bottom
8/// of this file — the storage-layer guard and the Rust-layer guard must agree
9/// or one of them is decorative.
10macro_rules! ts_glob {
11 () => {
12 "[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]T[0-9][0-9]:[0-9][0-9]:[0-9][0-9].[0-9][0-9][0-9][0-9][0-9][0-9]Z"
13 };
14}
15
16/// Table-level CHECK asserting every temporal column is canonical (§4.1, 0.5.4).
17///
18/// Timestamps are compared lexicographically everywhere — in SQL predicates, in
19/// `MAX(recorded_at)` when the clock recovers its floor, and in Rust `str`
20/// ordering. That is sound only if every value has the same width, so mixing
21/// `...T00:00:00Z` with `...T00:00:00.000000Z` makes `<=` disagree with
22/// chronology and traversals return empty sets with no error. The `Z` suffix
23/// alone does not achieve this; a fixed width does, and a CHECK is what makes
24/// it a property of the data rather than a convention.
25macro_rules! canonical_ts_check {
26 ($($col:literal),+ $(,)?) => {
27 concat!("CHECK (", $( $col, " GLOB '", ts_glob!(), "' AND ", )+ "1)")
28 };
29}
30
31/// A macro rather than a `const` for the same reason as [`ts_glob`]: `concat!`
32/// splices it into the table DDL and only accepts literals. [`WEIGHT_CHECK`] is
33/// the same text as a value, and carries the reasoning.
34macro_rules! weight_check {
35 () => {
36 "CHECK (weight >= 0.0 AND weight < 9e999 AND typeof(weight) = 'real')"
37 };
38}
39
40/// Table-level CHECK on `links.weight` (§4.7, T2.1, D-083).
41///
42/// Three clauses. Only the first is the one the item asked for; the other two
43/// were found by probing what the first still admits.
44///
45/// `weight >= 0.0` is the item as written: shortest-path analytics are unsound
46/// over negative weights, so Dijkstra and A\* refuse the graph at load time
47/// (D-039). Until now that refusal was the *only* place the property was
48/// enforced, which made it §4.7's one genuinely open gap — a database this crate
49/// wrote by itself could hold a row this crate would not read back.
50///
51/// `typeof(weight) = 'real'` closes a hole the item does not mention and which
52/// probing found. `REAL` in SQLite is an **affinity**, not a type: values that
53/// can be converted are, and values that cannot are stored as they came. `'abc'`
54/// cannot become a number, so it is stored as TEXT — and in SQLite's type
55/// ordering every text value sorts above every numeric one, so `'abc' >= 0.0`
56/// is *true* and the first clause passes it through.
57///
58/// That is not a wrong answer on the read side. It is a **panic**: reading a
59/// text `weight` as `f64` reaches `unreachable!("invalid value type")` inside
60/// libsql 0.9.30, in whatever unrelated query first touches the row. Measured,
61/// not reasoned about — see `examples/weight_check_probe.rs`.
62///
63/// The clause costs one `typeof` per insert and refuses nothing legitimate:
64/// `3`, `'5'` and `1.0` all arrive as REAL through affinity conversion and pass.
65/// It is taken **now** rather than in a later rung because SQLite has no
66/// `ADD CONSTRAINT` — every clause added later costs another full rebuild of the
67/// largest table in the schema.
68///
69/// `weight < 9e999` refuses `+∞`, and the reason is not the one anybody
70/// predicted. The plan expected the CHECK to admit infinity and argued the
71/// loader guard would catch it; the guard tests `< 0.0` and `is_nan()`, so it
72/// does not. The next guess — mine — was that this is harmless, since IEEE
73/// infinity propagates through addition and stays totally ordered, leaving
74/// Dijkstra terminating with "that edge is unusable": an odd answer, not a wrong
75/// one.
76///
77/// Both were wrong, and a test found it. **An infinite weight makes the
78/// transaction log unreplayable.** The log trigger serialises the row to JSON,
79/// and JSON has no representation for infinity, so the payload round-trips into
80/// `ReplayCorrupt { reason: "number out of range" }` — every later
81/// `reconstruct()` fails, including the one `close()` performs. The ledger is
82/// the source of truth under Doctrine III, so a value that cannot survive the
83/// log is not an eccentric weight, it is a corrupt one.
84///
85/// `9e999` is the idiom because SQLite has no `isinf`: the literal overflows to
86/// `+∞` on parse, and `inf < inf` is false. Finite values, including `1e308`,
87/// pass.
88///
89/// The loader guard still **stays**, for the reason the constraint cannot cover:
90/// `links_current` carries no CHECK, and neither do cold files created before
91/// this rung.
92pub const WEIGHT_CHECK: &str = weight_check!();
93
94/// The `RAISE(ABORT, …)` messages the schema's guards emit (§4.3).
95///
96/// Spliced into the trigger DDL *and* matched by [`crate::error::abort_kind`],
97/// so the guard and its classifier cannot drift. When they drift the failure is
98/// silent in the worst direction: the guard still fires, but the typed error
99/// (`SingleOpenViolation`, `RecordedAtRegression`, `ArchiveViolation`) degrades
100/// into an opaque `Engine` error that no caller can match on.
101macro_rules! abort_single_open {
102 () => {
103 "macrame: edge already has an open interval; retire it first"
104 };
105}
106macro_rules! abort_monotonic_ra {
107 () => {
108 "macrame: concept recorded_at must be strictly increasing"
109 };
110}
111macro_rules! abort_delete_guard {
112 () => {
113 "macrame: physical delete blocked outside archive session"
114 };
115}
116macro_rules! abort_cross_lineage {
117 () => {
118 "macrame: concept belongs to another lineage; a branch inherits concepts, it does not restate them"
119 };
120}
121macro_rules! abort_branch_immutable {
122 () => {
123 "macrame: branch_id is provenance and cannot be changed"
124 };
125}
126macro_rules! abort_branches_frozen {
127 () => {
128 "macrame: branch records are append-only"
129 };
130}
131
132pub const ABORT_SINGLE_OPEN: &str = abort_single_open!();
133pub const ABORT_MONOTONIC_RA: &str = abort_monotonic_ra!();
134pub const ABORT_DELETE_GUARD: &str = abort_delete_guard!();
135pub const ABORT_CROSS_LINEAGE: &str = abort_cross_lineage!();
136pub const ABORT_BRANCH_IMMUTABLE: &str = abort_branch_immutable!();
137pub const ABORT_BRANCHES_FROZEN: &str = abort_branches_frozen!();
138
139/// The root lineage every pre-v12 row is stamped with (§15.2, v12, D-214).
140///
141/// A macro as well as a `const` for [`ts_glob`]'s reason: it is spliced into
142/// the column defaults by `concat!`, which takes only literals. One spelling,
143/// so the default in the DDL, the seed row, and the Rust layer cannot drift
144/// into three databases that disagree about what the trunk is called.
145macro_rules! main_branch {
146 () => {
147 "main"
148 };
149}
150pub const MAIN_BRANCH: &str = main_branch!();
151
152/// The `branch_id` column, identical on all four ledger tables (§15.2, D-214).
153///
154/// The default is what makes the rung `ALTER TABLE` rather than a rewrite —
155/// SQLite records a constant default in the schema header and rewrites no row,
156/// measured at 83–139 µs over 20,000 rows in `examples/branch_identity_probe.rs`
157/// §1.
158///
159/// # The `REFERENCES` clause, and the condition it is actually gated on
160///
161/// SQLite specifies that a column added by `ALTER TABLE … ADD COLUMN` carrying
162/// a `REFERENCES` clause **must default to NULL** when foreign keys are
163/// enabled, because pre-existing rows cannot be validated against the new
164/// parent. That collides head-on with `NOT NULL DEFAULT 'main'`.
165///
166/// libSQL 0.9.30 applies that rule **dynamically rather than statically**, and
167/// probe §15 pins the four cases: the statement is refused only when the table
168/// **holds rows** *and* foreign keys are **on**. An empty table takes it with
169/// keys on; a populated table takes it with keys off. Being inside a
170/// transaction changes nothing either way.
171///
172/// This is the whole reason the v11 → v12 rung sets
173/// [`suspends_foreign_keys`]. It is worth being exact about what that buys,
174/// because "suspend the constraint to install the constraint" invites the
175/// suspicion that the result is decorative — §15 measures it and it is not.
176/// After an ALTER taken with keys suspended the clause is in `sqlite_master`,
177/// `PRAGMA foreign_key_list(concepts)` reports the key, an insert naming an
178/// unknown branch is refused with extended code 787, and deleting a referenced
179/// branch is refused **by the engine** rather than by a trigger. Enforcement is
180/// a per-connection pragma; the constraint is schema. Suspending the first
181/// never weakened the second.
182///
183/// Nor does the suspension launder a violation past the commit: `apply_step`
184/// runs `PRAGMA foreign_key_check` inside the transaction, and §15 confirms it
185/// reports the orphan when one is deliberately planted during the window.
186///
187/// Taken deliberately, with the dependency named in §19 rather than absorbed.
188/// The exposure is narrow and it is on the **upgrade** path only: fresh
189/// databases put the clause in a `CREATE TABLE`, where no engine has ever
190/// disputed it. If upstream ever tightens to SQLite's static reading, the rung
191/// fails **loudly** — at a named step, inside `BEGIN IMMEDIATE`, leaving the
192/// database honestly at v11 — and the fallback is one line: drop the clause
193/// from the ALTER and let the `branches` write guard carry lineage integrity
194/// alone, which is the weaker guarantee and the one D-030 has a name for.
195///
196/// [`suspends_foreign_keys`]: super::migrations
197macro_rules! branch_column {
198 () => {
199 concat!(
200 "branch_id TEXT NOT NULL DEFAULT '",
201 main_branch!(),
202 "' REFERENCES branches(branch_id)"
203 )
204 };
205}
206pub const BRANCH_COLUMN: &str = branch_column!();
207
208/// Marker table probed by the delete guards (D-008 revised).
209///
210/// The archive session creates this table and drops it again inside the single
211/// `BEGIN IMMEDIATE … COMMIT` archive transaction, so it never exists as
212/// committed state. Connection-locality — the property the original
213/// `temp.sqlite_master` probe was reaching for — is preserved by two
214/// independent mechanisms: uncommitted DDL is visible only to the writing
215/// connection, and the archive transaction holds the write lock for its
216/// duration, so no other connection can reach the guard at all.
217pub const ARCHIVE_SESSION_MARKER: &str = "macrame_archive_session";
218
219/// The concepts insert log trigger, **marker-gated since v10** (0.9.0, C3).
220///
221/// # Why an archive session must not log a concept insert
222///
223/// Rehydration is a physical move back and mints no transaction-time facts
224/// (§2.3): the concept returns to the hot table, the log entries describing it
225/// were never removed, and nothing about what was believed — or when — has
226/// changed. An unconditional `AFTER INSERT` makes that impossible to honour,
227/// because the move *is* an insert.
228///
229/// **And the damage is worse than a spurious row, which is what forced the
230/// rung.** The rehydrated row carries its **original** `recorded_at`, but the
231/// log row it would write gets a **new** `seq_id` at the end of the log. The
232/// fold partitions by `(table_name, entity_id)` and takes
233/// `ROW_NUMBER() OVER (… ORDER BY seq_id DESC) = 1` — last writer wins by
234/// *sequence*, not by timestamp. So the rehydration `'I'` would outrank the
235/// later `'U'` that retired the concept, and every `reconstruct` after the
236/// original creation time would return it **un-retired**. Rehydration would
237/// resurrect a belief the ledger had superseded, silently and retroactively,
238/// which is precisely what [Doctrine III] forbids.
239///
240/// Only the *insert* trigger is gated. `trg_concepts_log_update` stays
241/// unconditional because nothing inside a session updates a concept — archival
242/// deletes and rehydration inserts — so gating it would suppress nothing and
243/// widen the hole for no reason.
244pub const CREATE_CONCEPTS_LOG_INSERT: &str = concat!(
245 r#"
246 CREATE TRIGGER IF NOT EXISTS trg_concepts_log_insert
247 AFTER INSERT ON concepts
248 WHEN NOT EXISTS (
249 SELECT 1 FROM sqlite_master
250 WHERE type = 'table' AND name = '"#,
251 "macrame_archive_session",
252 r#"'
253 )
254 BEGIN
255 INSERT INTO transaction_log (table_name, entity_id, operation, payload, recorded_at, branch_id)
256 VALUES ('concepts', NEW.id, 'I',
257 json_object('v', 2, 'title', NEW.title, 'content', NEW.content,
258 'valid_from', NEW.valid_from, 'valid_to', NEW.valid_to,
259 'retired', NEW.retired,
260 'embedding_model', NEW.embedding_model),
261 NEW.recorded_at, NEW.branch_id);
262 END;
263 "#
264);
265
266/// The concepts delete guard, **marker-gated since v9** (0.9.0, C2, D-126).
267///
268/// A `pub const` rather than an anonymous entry in [`CREATE_TRIGGERS`] because
269/// two readers need exactly this text: the baseline, which installs it on a new
270/// database, and the `v8 → v9` rung, which replaces the v8 body on an existing
271/// one. A second copy is a copy that drifts, and this trigger is the one whose
272/// body carries a doctrine decision.
273///
274/// # What changed, and why re-issuing the baseline could not do it
275///
276/// Through v8 this guard was **unconditional**: `BEFORE DELETE ON concepts`
277/// aborting every time, on the reasoning that concepts are never physically
278/// archived ([D-022](../../docs/architecture/s13-decision-register.md)). C2
279/// makes that false — a declared archive session may now move a retired,
280/// unreferenced concept to the cold file — so the guard takes the same shape its
281/// two siblings have had since 0.5.3: it fires **unless** the archive-session
282/// marker is present.
283///
284/// It needs a rung of its own, and that was measured rather than assumed
285/// (D-126). `CREATE TRIGGER IF NOT EXISTS` on an existing name keeps the **old
286/// body** — re-issuing the baseline against a v8 database leaves the
287/// unconditional guard exactly where it was — and `verify` compared `type` and
288/// `name` and never bodies, so the stale guard passed verification in silence.
289/// Both halves are now closed: the rung drops and recreates, and `verify`
290/// checks that every delete guard's body probes the marker.
291pub const CREATE_CONCEPTS_GUARD_DELETE: &str = concat!(
292 r#"
293 CREATE TRIGGER IF NOT EXISTS trg_concepts_guard_delete
294 BEFORE DELETE ON concepts
295 WHEN NOT EXISTS (
296 SELECT 1 FROM sqlite_master
297 WHERE type = 'table' AND name = '"#,
298 "macrame_archive_session",
299 r#"'
300 )
301 BEGIN
302 SELECT RAISE(ABORT, '"#,
303 abort_delete_guard!(),
304 r#"');
305 END;
306 "#
307);
308
309/// The `concepts` ledger table (§4.1).
310///
311/// # `rowid_pk` is explicit, and that is the whole point (v8, D-119)
312///
313/// Through v7 this table declared `id TEXT PRIMARY KEY`, which left its rowid
314/// **implicit** — and `concepts_fts` is external-content keyed on that rowid.
315/// `VACUUM` renumbers implicit rowids, which would silently decouple the search
316/// index from the rows it indexes: no error, no integrity-check failure, just
317/// results that stop matching.
318///
319/// [D-071](../../docs/architecture/s13-decision-register.md) proved the hazard
320/// unreachable *by consequence rather than by design* — `trg_concepts_guard_delete`
321/// is unconditional, so rowids are dense `1..n` and `VACUUM`'s renumbering is
322/// the identity map. 0.9.0's archival makes them sparse and makes the hazard
323/// real, so v8 replaces the accident with a column: an `INTEGER PRIMARY KEY` is
324/// a stored value, and `VACUUM` preserves it whether the numbering is dense or
325/// not (measured in `examples/concepts_rebuild_probe.rs` §5).
326///
327/// SQLite permits one primary key per table, so `id` becomes `NOT NULL UNIQUE`.
328/// That keeps it a valid foreign-key parent for `links.source_id` /
329/// `links.target_id` and keeps `ON CONFLICT(id)` working, but it **is** a
330/// primary-key change — which [D-036](../../docs/architecture/s13-decision-register.md)
331/// forbids outright after 1.0. Taken pre-1.0 on purpose, or never.
332/// The lineage register (§15.2, v12, D-214).
333///
334/// Four columns and no more, because a branch is **not** a third temporal axis
335/// (Doctrine II): it carries no interval of its own, only the point in the
336/// second clock where it diverged.
337///
338/// `parent_id` is a self-referencing foreign key, declarable here because it
339/// sits in a `CREATE TABLE` where SQLite permits forward and self references
340/// freely. `NULL` marks the root, and the paired `CHECK` makes "root" a single
341/// state rather than two columns that can disagree: a row with a parent and no
342/// fork point is a lineage whose ancestry cannot be resolved, and a row with a
343/// fork point and no parent is a divergence from nothing.
344///
345/// `forked_at` is in the **`recorded_at` domain** — the transaction-time
346/// instant the lineage diverged, which is what §15.3's visibility cutoffs are
347/// computed over. Not a valid-time bound: a branch does not believe things
348/// about a period, it believes them from a moment onward.
349///
350/// The ordering `CHECK` is row-local on purpose. `forked_at <= created_at` is
351/// checkable from the row itself; an ordering against the *parent's* row is
352/// not, and a `CHECK` cannot see another row. The cross-row half is `fork()`'s
353/// to enforce at D-034's boundary, and saying so here is cheaper than a
354/// constraint that looks complete and is not.
355///
356/// **Which cross-row ordering, corrected in 0.14.7.** This said "the fork point
357/// is at or after the parent's *creation*" from v12 until `fork()` existed to
358/// enforce it, and that turned out to be uncheckable rather than merely
359/// unenforced: [`seed_root_branch`](crate::schema) stamps the trunk's
360/// `created_at` from `SystemTime::now()` during migration — before the
361/// database's injected clock is resolved, and it cannot simply run after,
362/// because the clock's floor is read from tables the migration creates. So
363/// `created_at` is not on the ledger's timeline and comparing a `forked_at` to
364/// it is comparing two clocks. What [`Database::fork`](crate::Database::fork) enforces instead is
365/// `forked_at >= parent.forked_at`, both issued by the same clock, which makes
366/// fork points non-decreasing down any root path.
367pub const CREATE_BRANCHES_TABLE: &str = concat!(
368 r#"
369CREATE TABLE IF NOT EXISTS branches (
370 branch_id TEXT NOT NULL PRIMARY KEY,
371 parent_id TEXT REFERENCES branches(branch_id),
372 forked_at TEXT,
373 created_at TEXT NOT NULL,
374 CHECK ((parent_id IS NULL) = (forked_at IS NULL)),
375 CHECK (forked_at IS NULL OR forked_at <= created_at),
376 CHECK (forked_at IS NULL OR forked_at GLOB '"#,
377 ts_glob!(),
378 r#"'),
379 "#,
380 canonical_ts_check!("created_at"),
381 r#"
382);
383"#
384);
385
386/// Seed the root lineage, idempotently.
387///
388/// One statement shared by the baseline and the v11 → v12 rung, taking
389/// `created_at` as a parameter. `OR IGNORE` rather than `IF NOT EXISTS`
390/// gymnastics because both callers may run against a database that already has
391/// the row — the rung on a retry, the baseline never, but a single statement
392/// that is safe for both is one fewer thing to reason about.
393///
394/// This must run **before** any row is stamped, on both paths: every
395/// `branch_id` default names `'main'`, and the foreign key means a database
396/// without this row cannot accept a single write.
397pub const SEED_MAIN_BRANCH: &str = concat!(
398 "INSERT OR IGNORE INTO branches (branch_id, parent_id, forked_at, created_at) \
399 VALUES ('",
400 main_branch!(),
401 "', NULL, NULL, ?1)"
402);
403
404/// `branches` is append-only outside an archive session (§15.2, §15.4).
405///
406/// The two guards no longer say the same thing, and 0.14.13 is where they
407/// parted. This one stays **unconditional**: no session of any kind may edit a
408/// lineage record in place. [`CREATE_BRANCHES_GUARD_DELETE`] is now gated on
409/// the archive-session marker like its three siblings, because
410/// [`crate::Database::archive_branch`] made removing a lineage record a legal
411/// operation — see that guard for what changed and why the change needed a
412/// rung of its own.
413///
414/// # Why `UPDATE` is refused whole-row
415///
416/// The foreign key already refuses renaming or deleting a lineage any row
417/// still points at, so this guard is not what keeps the ledger from being
418/// orphaned. What it keeps is narrower and harder to see: `parent_id` and
419/// `forked_at` are the inputs to ancestry, so editing either **re-derives the
420/// visibility of rows already written**, with no new assertion anywhere. That
421/// is the move [Doctrine III] forbids, reachable by one raw-SQL statement, and
422/// no foreign key has anything to say about it.
423///
424/// Whole-row rather than a named subset because nothing on the row legitimately
425/// changes, and a whole-row guard needs no maintenance the day a column is
426/// added.
427///
428/// [Doctrine III]: ../../docs/architecture/README.md
429pub const CREATE_BRANCHES_GUARD_UPDATE: &str = concat!(
430 r#"
431 CREATE TRIGGER IF NOT EXISTS trg_branches_frozen_update
432 BEFORE UPDATE ON branches
433 BEGIN
434 SELECT RAISE(ABORT, '"#,
435 abort_branches_frozen!(),
436 r#"');
437 END;
438 "#
439);
440
441/// The delete half of the rule, **marker-gated since v13** (0.14.13, §15.4,
442/// [D-230](../../docs/architecture/s13-decision-register.md#d-230)).
443///
444/// # What changed
445///
446/// Through v12 this guard was unconditional, and its own docstring said why:
447/// *"there is no session in which removing a lineage record is legal — branches
448/// are never archived"*. [`crate::Database::archive_branch`] makes that false.
449/// The sentence was a true description of the operations that existed, written
450/// as though it were a property of the table, which is the shape D-035 asks to
451/// be stated rather than assumed.
452///
453/// **The lineage row must move, and that is forced rather than chosen.** An
454/// abandonment arm that took the branch's `links` and left its `branches` row
455/// would leave `hot_log_reach` unsound: that probe's argument rests on *the
456/// newest row per entity is never archivable*, which holds for a predicate
457/// needing a later row to exist and fails for one that takes a whole lineage.
458/// Moving the `branches` row is what makes a hot fold that omits the lineage
459/// **correct rather than silently short** — every read and write naming the
460/// name now raises [`crate::DbError::UnknownBranch`], which is a refusal, not a
461/// wrong answer.
462///
463/// # Why it needed a rung
464///
465/// [`CREATE_CONCEPTS_GUARD_DELETE`]'s reason, measured once already (D-126):
466/// `CREATE TRIGGER IF NOT EXISTS` on an existing name keeps the **old body**,
467/// so re-issuing the baseline against a v12 database leaves the unconditional
468/// guard exactly where it is and `archive_branch` fails on every ledger that
469/// was not created by this build. The v12 → v13 rung drops and recreates, and
470/// `verify` now carries this name in `DELETE_GUARDS`, so a database whose
471/// guard predates the change is refused at open with a sentence rather than at
472/// archive time with a trigger abort.
473///
474/// The update guard is deliberately **not** gated — see
475/// [`CREATE_BRANCHES_GUARD_UPDATE`]. Archival is a move; there is still no
476/// session in which editing a lineage's parent or fork point is legal, and
477/// gating both would have suspended a rule the operation does not need
478/// suspended.
479pub const CREATE_BRANCHES_GUARD_DELETE: &str = concat!(
480 r#"
481 CREATE TRIGGER IF NOT EXISTS trg_branches_frozen_delete
482 BEFORE DELETE ON branches
483 WHEN NOT EXISTS (
484 SELECT 1 FROM sqlite_master
485 WHERE type = 'table' AND name = '"#,
486 "macrame_archive_session",
487 r#"'
488 )
489 BEGIN
490 SELECT RAISE(ABORT, '"#,
491 abort_branches_frozen!(),
492 r#"');
493 END;
494 "#
495);
496
497/// A branch inherits concepts; it does not restate them (§15.2, D-214).
498///
499/// `concepts` is a current-state projection keyed by identity — `id` is
500/// `NOT NULL UNIQUE` and the write path uses `ON CONFLICT(id) DO UPDATE` — so
501/// two lineages holding different beliefs about one concept is two rows with
502/// one `id`, which the unique index refuses on its own (probe §2). What it
503/// refuses it refuses as a *constraint failure*, naming nothing; this guard
504/// turns the same refusal into a sentence that says which rule was broken.
505///
506/// It fires **before** `ON CONFLICT` is considered, which is not obvious and
507/// was measured rather than assumed (probe §7): a cross-lineage upsert is
508/// refused, a same-lineage one is accepted, and a new id is accepted.
509///
510/// Exact-branch equality, not ancestry. A branch that may restate its parent's
511/// concepts is the overlay design, and the overlay is deferred with its reopen
512/// trigger named (D-214) — a guard that quietly permitted the ancestry case
513/// would ship half of it with none of the machinery that makes it correct.
514pub const CREATE_CONCEPTS_GUARD_LINEAGE: &str = concat!(
515 r#"
516 CREATE TRIGGER IF NOT EXISTS trg_concepts_cross_lineage
517 BEFORE INSERT ON concepts
518 WHEN EXISTS (
519 SELECT 1 FROM concepts
520 WHERE id = NEW.id AND branch_id <> NEW.branch_id
521 )
522 BEGIN
523 SELECT RAISE(ABORT, '"#,
524 abort_cross_lineage!(),
525 r#"');
526 END;
527 "#
528);
529
530/// `branch_id` records where a row was minted, and minting happened once.
531///
532/// The column is **provenance, not identity** (D-214), and the distinction is
533/// exactly what this guard keeps true. An `UPDATE` that moved a concept between
534/// lineages would rewrite where a belief came from without asserting anything
535/// new — the same shape as editing `branches.parent_id`, and forbidden for the
536/// same reason.
537pub const CREATE_CONCEPTS_GUARD_BRANCH: &str = concat!(
538 r#"
539 CREATE TRIGGER IF NOT EXISTS trg_concepts_branch_immutable
540 BEFORE UPDATE ON concepts
541 WHEN NEW.branch_id <> OLD.branch_id
542 BEGIN
543 SELECT RAISE(ABORT, '"#,
544 abort_branch_immutable!(),
545 r#"');
546 END;
547 "#
548);
549
550pub const CREATE_CONCEPTS_TABLE: &str = concat!(
551 r#"
552CREATE TABLE IF NOT EXISTS concepts (
553 rowid_pk INTEGER PRIMARY KEY,
554 id TEXT NOT NULL UNIQUE,
555 title TEXT NOT NULL,
556 content TEXT NOT NULL DEFAULT '',
557 embedding_model TEXT,
558 valid_from TEXT NOT NULL,
559 valid_to TEXT NOT NULL DEFAULT '9999-12-31T23:59:59.999999Z',
560 recorded_at TEXT NOT NULL,
561 retired INTEGER NOT NULL DEFAULT 0,
562 "#,
563 branch_column!(),
564 r#",
565 "#,
566 canonical_ts_check!("valid_from", "valid_to", "recorded_at"),
567 r#"
568);
569"#
570);
571
572/// The ledger. Append-only, one row per assertion, and **keyed by lineage
573/// since v15** (0.14.15, §15.4, [D-232]).
574///
575/// # `branch_id` is in the key, and the release it took to get there
576///
577/// v12 put `branch_id` in the key of `links_current` and left it out of this
578/// one, on an argument the v12 rung records: "`links` accepts both rows because
579/// `recorded_at` is already in its key". That is true of two assertions made at
580/// two instants, which is what a probe testing it by hand produces — and it is
581/// the reason the gap read as latent for seven releases.
582///
583/// It was not latent. **The batch write paths take one stamp for the whole
584/// batch** ([D-014]), and `reject_overlaps_within` groups candidates by
585/// `(source, target, edge_type, branch_id)` — so a pair differing *only* in
586/// lineage is not an overlap, is passed straight through, and collides on the
587/// key with a bare `UNIQUE constraint failed: links.…`. Both batch surfaces
588/// reach it, and `examples/links_key_reach_probe.rs` is the reproduction.
589///
590/// # Why `branch_id` goes last
591///
592/// The same reason the v12 rung gives for `links_current`, and it is stronger
593/// here because this table's autoindex is the *only* index over four of its
594/// columns: the leading `(source_id, target_id, edge_type, valid_from,
595/// recorded_at)` prefix is what `temporal::archive`'s predicates and
596/// `integrity::shadow` seek on, and appending leaves every one of those plans
597/// untouched. A branch-leading key would have re-planned the archive sweep to
598/// buy nothing — the probe's §5 has the plans.
599///
600/// # What this does not change
601///
602/// Not Doctrine III, and not what a row means. The key admits a row the old key
603/// refused; it removes none, alters none, and merges none. Every database that
604/// climbed the v14 → v15 rung holds exactly the rows it held before, which is
605/// what makes the rung a copy rather than a decision.
606///
607/// [D-014]: ../../docs/architecture/s13-decision-register.md#d-014
608/// [D-232]: ../../docs/architecture/s13-decision-register.md#d-232
609pub const CREATE_LINKS_TABLE: &str = concat!(
610 r#"
611CREATE TABLE IF NOT EXISTS links (
612 source_id TEXT NOT NULL REFERENCES concepts(id),
613 target_id TEXT NOT NULL REFERENCES concepts(id),
614 edge_type TEXT NOT NULL,
615 valid_from TEXT NOT NULL,
616 recorded_at TEXT NOT NULL,
617 valid_to TEXT NOT NULL DEFAULT '9999-12-31T23:59:59.999999Z',
618 weight REAL NOT NULL DEFAULT 1.0,
619 properties TEXT NOT NULL DEFAULT '{}',
620 "#,
621 branch_column!(),
622 r#",
623 PRIMARY KEY (source_id, target_id, edge_type, valid_from, recorded_at, branch_id),
624 "#,
625 weight_check!(),
626 r#",
627 "#,
628 canonical_ts_check!("valid_from", "valid_to", "recorded_at"),
629 r#"
630);
631"#
632);
633
634pub const CREATE_LINKS_CURRENT_TABLE: &str = concat!(
635 r#"
636CREATE TABLE IF NOT EXISTS links_current (
637 source_id TEXT NOT NULL,
638 target_id TEXT NOT NULL,
639 edge_type TEXT NOT NULL,
640 valid_from TEXT NOT NULL,
641 valid_to TEXT NOT NULL,
642 weight REAL NOT NULL,
643 properties TEXT NOT NULL,
644 recorded_at TEXT NOT NULL,
645 "#,
646 branch_column!(),
647 r#",
648 PRIMARY KEY (source_id, target_id, edge_type, valid_from, branch_id),
649 "#,
650 canonical_ts_check!("valid_from", "valid_to", "recorded_at"),
651 r#"
652);
653"#
654);
655
656pub const CREATE_TRANSACTION_LOG_TABLE: &str = concat!(
657 r#"
658CREATE TABLE IF NOT EXISTS transaction_log (
659 seq_id INTEGER PRIMARY KEY AUTOINCREMENT,
660 table_name TEXT NOT NULL,
661 entity_id TEXT NOT NULL,
662 operation TEXT NOT NULL,
663 payload TEXT NOT NULL,
664 recorded_at TEXT NOT NULL,
665 "#,
666 branch_column!(),
667 r#",
668 "#,
669 canonical_ts_check!("recorded_at"),
670 r#"
671);
672"#
673);
674
675/// The per-model embedding table (§4.1, D-005), for a validated model name.
676///
677/// A function rather than a `const` because the table's identity *and its
678/// column type* both depend on the model: `F32_BLOB(dim)` carries the declared
679/// dimension in the schema, which is what [`crate::vector::declared_dimension`]
680/// reads back so the crate never keeps a second copy of it.
681///
682/// Deliberately not part of the baseline migration. Which models exist is an
683/// application's choice made over time, not a property of the schema version,
684/// and D-036 classifies these tables as disposable periphery: a migration may
685/// drop one and re-embed. `IF NOT EXISTS` makes registration idempotent.
686///
687/// No temporal columns, on purpose. Doctrine VII makes an embedding a derived
688/// artifact of a model applied to content — it has no valid time of its own, and
689/// giving it a `recorded_at` would put a third clock next to the two §2 permits
690/// and invite queries that mix them.
691pub fn create_embeddings_table(model: &crate::vector::ModelName, dim: usize) -> String {
692 format!(
693 "CREATE TABLE IF NOT EXISTS {table} (
694 concept_id TEXT PRIMARY KEY REFERENCES concepts(id),
695 embedding F32_BLOB({dim}) NOT NULL
696);",
697 table = model.table(),
698 )
699}
700
701/// The DiskANN index over a model's vectors.
702///
703/// **Load-bearing for correctness, not only for speed.** Measured against
704/// libSQL 0.9.30: a blob of the wrong length inserted into an `F32_BLOB(4)`
705/// column is *accepted* while no vector index exists, and rejected — with the
706/// row not landing — once one does. §4.1 previously claimed the column type
707/// enforced its own dimension at insert time; it does not. So this index is
708/// created together with the table it indexes and is never optional, and
709/// dropping it to speed up a bulk load would silently disarm the only
710/// storage-layer check on dimension.
711pub fn create_embeddings_index(model: &crate::vector::ModelName) -> String {
712 format!(
713 "CREATE INDEX IF NOT EXISTS {index} ON {table} (libsql_vector_idx(embedding));",
714 index = model.index(),
715 table = model.table(),
716 )
717}
718
719/// Derived analytics output, keyed by concept and label (§5.4, D-041).
720///
721/// Deliberately outside the ledger. Three properties are load-bearing and each
722/// is the opposite of what the four normative tables above do.
723///
724/// **No log trigger.** Nothing in [`CREATE_TRIGGERS`] fires on this table, so an
725/// annotation never reaches `transaction_log`. That is Doctrine VII's reasoning
726/// about embeddings applied to the other derived artifact: a community label is
727/// a function of an algorithm, a version of that algorithm, and a graph — not a
728/// statement about the world, and a ledger that records it is recording the
729/// analytics schedule as though it were history. A reconstruction that wants
730/// labels recomputes them, which is the only honest way to ask what a past
731/// graph's communities *were*.
732///
733/// **No delete guard.** Doctrine V protects the hot ledger tables; this table is
734/// derivative state in Doctrine VI's second category, so wiping it must stay a
735/// legal, ordinary operation — a rerun replaces the previous pass, and dropping
736/// the whole table costs nothing but the recomputation.
737///
738/// **Upsert on `(concept_id, label)`.** One current value per label per concept.
739/// Storing a history of successive runs here would be the ledger again, by
740/// another name.
741///
742/// The foreign key is safe in a way `links_current`'s omitted ones are not:
743/// concepts are never physically deleted (D-022), and this table is rebuilt by
744/// re-running an algorithm that read `concepts` in the first place, so there is
745/// no insertion-order problem to solve.
746pub const CREATE_ANALYTICS_ANNOTATIONS_TABLE: &str = concat!(
747 r#"
748CREATE TABLE IF NOT EXISTS analytics_annotations (
749 concept_id TEXT NOT NULL REFERENCES concepts(id),
750 label TEXT NOT NULL,
751 value TEXT NOT NULL,
752 computed_at TEXT NOT NULL,
753 PRIMARY KEY (concept_id, label),
754 "#,
755 canonical_ts_check!("computed_at"),
756 r#"
757);
758"#
759);
760
761/// The keyword half of hybrid search: an FTS5 index over concept text (§5.9).
762///
763/// **External content.** The table declares `content='concepts'`, so the tokens
764/// are indexed but the text itself is not duplicated — FTS5 reads it back from
765/// `concepts` by rowid when it needs a column value. Two reasons beyond the
766/// storage saving, and the second is the one that decided it:
767///
768/// * There is exactly one copy of the text, so the index cannot disagree with
769/// the concept about what the concept says. A standalone FTS table would be a
770/// second description of data the ledger already holds, which is the failure
771/// class D-030 and D-035 exist to prevent.
772/// * `INSERT INTO concepts_fts(concepts_fts) VALUES('rebuild')` reconstructs the
773/// whole index from the content table in one statement. D-036 requires every
774/// derivative table to be rebuildable from the ledger, and here that is the
775/// engine's own operation rather than code of ours that has to be kept honest.
776///
777/// The cost is that external-content tables do not maintain themselves: an
778/// `UPDATE` must retract the *old* terms before adding the new ones, using the
779/// old column values. That is what `trg_concepts_fts_update` does, and getting
780/// it wrong leaves an index that still matches text no concept contains.
781///
782/// **`content_rowid` names `rowid_pk`, not `rowid` (v8, D-119).** They are the
783/// same value — an `INTEGER PRIMARY KEY` *is* the rowid — but naming the column
784/// is what makes the key a declared one rather than an implicit one `VACUUM` is
785/// free to renumber. See [`CREATE_CONCEPTS_TABLE`].
786pub const CREATE_CONCEPTS_FTS: &str = r#"
787CREATE VIRTUAL TABLE IF NOT EXISTS concepts_fts USING fts5(
788 title,
789 content,
790 content='concepts',
791 content_rowid='rowid_pk'
792);
793"#;
794
795/// FTS5's own consistency check — **and it cannot see the failure that matters**
796/// (§5.9, D-071).
797///
798/// Kept as a named constant so the finding has somewhere to live, and used by
799/// `an_emptied_fts_index_still_passes_integrity_check`, which is a tripwire
800/// rather than a guarantee.
801///
802/// On this libSQL build (0.9.30), `'integrity-check'` verifies the index's
803/// *internal* consistency and not its agreement with the content table. Measured:
804/// after `'delete-all'` the index answers zero matches where it answered ten, and
805/// both `'integrity-check'` and `'integrity-check', 0` still report success. So a
806/// `verify_fts()` built on this would report a healthy index for an empty one —
807/// which is why there is no `verify_fts()`. See D-071.
808pub const VERIFY_CONCEPTS_FTS: &str =
809 "INSERT INTO concepts_fts (concepts_fts) VALUES ('integrity-check');";
810
811/// Reconstruct the FTS index from `concepts` (§5.9, D-036).
812///
813/// The engine's own operation, so the rebuild path is not a second
814/// implementation of the triggers that could drift from them.
815pub const REBUILD_CONCEPTS_FTS: &str =
816 "INSERT INTO concepts_fts (concepts_fts) VALUES ('rebuild');";
817
818/// Refresh the query planner's statistics (0.12.4, D-149).
819///
820/// # Why this exists at all
821///
822/// Until 0.12.4 nothing in this crate ever ran `ANALYZE`, so `sqlite_stat1` did
823/// not exist in any database Macrame had created and **every plan was costed
824/// against SQLite's built-in defaults**: assume ~1M rows, assume each bound
825/// equality column divides the search by ten. That estimate is *structural* — a
826/// function of how many columns a query binds, not of what the table holds.
827///
828/// Which is a restatement of this schema's own worst recurring defect. From
829/// `tests/index_plan_tests.rs`: *"a covering index captures a query because it
830/// contains the columns, not because it discriminates."* D-042, D-059 and D-064
831/// are three instances of a planner doing the only thing available to it.
832/// [`CREATE_INDICES`] declares two indices that both lead on `source_id`, and
833/// with no statistics the planner separates them by column count alone.
834///
835/// # Bounded by construction
836///
837/// `ANALYZE` is a **write** — it writes `sqlite_stat1` and takes the write lock —
838/// so unbounded on a populated `links_current` it is exactly the kind of
839/// unbudgeted hold `CHUNK_BUDGET` exists to prevent. [`ANALYSIS_LIMIT`], set once
840/// per connection in `configure`, caps the rows examined per index and makes the
841/// cost a function of the index count rather than the table size. That is what
842/// lets this be scheduled as ordinary low-priority work.
843pub const ANALYZE: &str = "ANALYZE;";
844
845/// Re-analyse only what has gone stale (0.12.4, D-149).
846///
847/// SQLite tracks how much each table has changed since its last analysis and
848/// runs `ANALYZE` only where it believes the statistics no longer hold. A no-op
849/// when nothing has moved, which is what makes it safe to call on a schedule
850/// rather than only on demand.
851///
852/// Bounded by [`ANALYSIS_LIMIT`] like everything else on the connection.
853pub const OPTIMIZE: &str = "PRAGMA optimize;";
854
855/// The row cap that makes [`ANALYZE`] budgetable.
856///
857/// 400 is SQLite's own documented recommendation. It buys approximate statistics
858/// in roughly constant time instead of exact statistics in time proportional to
859/// the table — and approximate is emphatically enough here, because the decision
860/// being informed is *which of two indices discriminates*, not a cardinality
861/// estimate anyone reads.
862///
863/// Set on the connection rather than around each call, so it also bounds the
864/// analysis [`OPTIMIZE`] triggers internally. A limit that applied only to the
865/// explicit path would leave the scheduled one unbounded, which is the half that
866/// runs without anybody watching.
867///
868/// # Measured in 0.12.23: it is a constant factor, not a bound (D-166)
869///
870/// D-149 claimed this makes `ANALYZE`'s cost "a function of the index count
871/// rather than the table size". Measured on this schema — `examples/analyze_hold.rs`,
872/// which times the crate's own hold beside the same file analysed with the
873/// pragma off and on:
874///
875/// | edges | crate's hold | limit off | limit 400 |
876/// |---|---|---|---|
877/// | 10,000 | 5.26 ms | 18.4 ms | 6.01 ms |
878/// | 40,000 | 19.1 ms | 78.6 ms | 19.4 ms |
879///
880/// The pragma **is** in force — the crate's hold tracks the capped arm and not
881/// the uncapped one, which is how it is established at all, since the
882/// connection that runs `ANALYZE` is the actor's and no test can reach it. It
883/// is worth 3.1× at 10,000 edges and 4.1× at 40,000.
884///
885/// What it does not do is remove the table from the equation: over that 4×
886/// range the capped time grew 3.2×. So `analyze()` on a 40,000-edge ledger
887/// holds the write lock for ~19 ms, about 6× [`crate::CHUNK_BUDGET`], and
888/// [`crate::metrics::CommandKind::Analyze`] is **not** budget-exempt — it
889/// appears in `metrics().budget_violations()` and always had.
890///
891/// Since 0.13.24 that kind is `analyze()` alone; `optimize()` reports as
892/// [`crate::metrics::CommandKind::Optimize`] and is separately, deliberately
893/// not exempt (W10.5,
894/// [D-197](../../docs/architecture/s13-decision-register.md#d-197)).
895pub const ANALYSIS_LIMIT: &str = "PRAGMA analysis_limit = 400";
896
897/// Every index the schema declares.
898///
899/// # Two entries left in v8, and why the list is now allowed to be short
900///
901/// `idx_annotations_label` and `idx_lc_tgt_active` were dropped by the v7 → v8
902/// rung ([D-089](../../docs/architecture/s13-decision-register.md), completed by
903/// D-118). Neither had a reader anywhere in the crate — `analytics_annotations`
904/// is never selected from here at all, and no query seeks on
905/// `links_current.target_id` as a leading column — so each was an index write
906/// per insert, forever, buying nothing. One of them was on the crate's hottest
907/// write path.
908///
909/// `tests/index_plan_tests.rs` now requires the unread set to be **empty**,
910/// which turns "these two are known bad" into "an index with no reader is a red
911/// test". That is the guarantee this list is kept short by.
912///
913/// # `idx_links_target` is not `idx_lc_tgt_active` coming back
914///
915/// The two look like the same index and are not, which is worth stating because
916/// the resemblance is the trap. `idx_lc_tgt_active` was `(target_id, valid_to)`
917/// on **`links_current`**, the materialized projection, and it was dropped
918/// because *nothing in the crate seeks on it* — no reader, pure write cost.
919/// `idx_links_target` is `(target_id)` on **`links`**, the ledger, and it exists
920/// because `CONCEPTS_ARCHIVABLE` seeks on exactly that column and the plan is
921/// measured before and after.
922///
923/// D-089's rule was never "no index on a target column". It was "an index needs
924/// a named query that seeks on it", and the registry is what enforces the
925/// difference rather than this paragraph.
926/// The two indices on `links_current`, named because a rung has to restore
927/// them (§15.2, D-214).
928///
929/// `links_current` is derivative, so the v11 → v12 rung re-creates it rather
930/// than altering it — and `DROP TABLE` takes the table's indices with it.
931/// Neither [`CREATE_LINKS_CURRENT_TABLE`] nor `rebuild_within` puts them back:
932/// the first declares a table and the second fills one. The open-time schema
933/// verifier is what noticed, which is the argument for having it.
934///
935/// `pub(crate)` rather than `pub`: every other const this module publishes
936/// describes the schema a caller might want to read, and these two exist
937/// only so a rung can put back what its own `DROP TABLE` removed. The
938/// published form of an index is still [`CREATE_INDICES`], which contains
939/// both of these.
940///
941/// Named consts rather than a `CREATE_INDICES` scan for `ON links_current`,
942/// because a rung should state which indices it owes rather than derive the
943/// list from a definition that will keep changing after it. If a later release
944/// adds a third index here, that release's rung adds it — this one is a
945/// statement about v12 and stays one.
946pub(crate) const LC_TRAVERSAL_COVER: &str = "CREATE INDEX IF NOT EXISTS \
947 idx_lc_traversal_cover ON links_current \
948 (source_id, valid_from, valid_to, weight, edge_type, target_id);";
949
950/// See [`LC_TRAVERSAL_COVER`].
951pub(crate) const LC_OPEN_INTERVAL: &str = "CREATE INDEX IF NOT EXISTS \
952 idx_lc_open_interval ON links_current \
953 (source_id, target_id, edge_type, valid_to, valid_from);";
954
955/// The lineage read's own index (0.14.14, §15.4, D-231, shipped v13 -> v14).
956///
957/// See [`CREATE_INDICES`] for what seeks on it and why it is a **second** index
958/// rather than a column added to [`LC_TRAVERSAL_COVER`], which is what §15.4
959/// asked for.
960pub(crate) const LC_LINEAGE_CUT: &str = "CREATE INDEX IF NOT EXISTS \
961 idx_lc_lineage_cut ON links_current \
962 (branch_id, recorded_at, source_id, target_id, edge_type, valid_from, \
963 valid_to, weight);";
964
965pub const CREATE_INDICES: &[&str] = &[
966 // Covering index for the traversal CTE (§5.2, D-042).
967 //
968 // Column order is load-bearing and was measured with EXPLAIN QUERY PLAN.
969 // The seek column is `source_id`; everything after it is there so the
970 // recursive step never touches the base table. The two range columns come
971 // next and `edge_type` comes *after* them, because `edge_types` is empty
972 // unless a caller sets it: with `edge_type` in second position SQLite
973 // declines the index for the unfiltered traversal — the default one — and
974 // silently falls back to a non-covering plan.
975 //
976 // (source_id, edge_type, valid_from, ...) filtered: COVERING
977 // unfiltered: NOT covering
978 // (source_id, valid_from, valid_to, weight, edge_type, target_id)
979 // both: COVERING
980 //
981 // This subsumes the former idx_lc_src_active (source_id, valid_to): same
982 // prefix column, strictly more payload. Keeping both would pay two index
983 // writes per assertion on a table that already takes three writes.
984 LC_TRAVERSAL_COVER,
985 // The single-open-interval probe's own index (D-059, shipped v5 -> v6).
986 //
987 // `trg_links_single_open` runs an `EXISTS` on every edge insert, keyed on
988 // (source_id, target_id, edge_type, valid_to) with valid_from as an
989 // inequality. Before this index the planner served that probe from
990 // `idx_lc_traversal_cover` with only `source_id` bound — it wins as a
991 // covering index over the primary-key autoindex, which lacks `valid_to` —
992 // so **every insert scanned its source's entire out-degree**. Measured on a
993 // fixed 90-row chunk: 4.4 ms into an empty table, 18.4 ms into a
994 // 2,000-edge hub, 47.7 ms into an 8,000-edge one, and 1.06 s into 90,000.
995 // Growth in the table, not in the chunk.
996 //
997 // With this index the same 90 rows into the 8,000-edge hub take 8.0 ms and
998 // stay flat. It matters beyond bulk import: the probe is on the insert path,
999 // so an interactive `assert_edge` against a high-degree node paid the same
1000 // scan, and that is the path CHUNK_BUDGET's 3 ms exists to protect.
1001 //
1002 // Column order follows the trigger's WHERE exactly — the three equalities
1003 // first, then `valid_to` which is compared to the sentinel, then
1004 // `valid_from` which is the `<>` and cannot be a seek column. This does not
1005 // subsume `idx_lc_traversal_cover` and is not subsumed by it: that one leads
1006 // on `source_id` alone for the recursive walk, this one needs all three
1007 // equality columns bound. Both are kept, which is a fourth index write per
1008 // assertion buying a scan's removal from the same operation.
1009 LC_OPEN_INTERVAL,
1010 // The lineage read's base scans (0.14.14, W12.14, §15.4, D-231, shipped
1011 // v13 -> v14).
1012 //
1013 // `graph::lineage::churned_cte` and `links_cut_cte` are the only two
1014 // statements in the crate that read `links_current` **by lineage**, and
1015 // five call sites emit them: the traversal, `query_as_of_edges_on`,
1016 // `load_subgraph_with` and `diff`'s two tagged copies. Both drive from the
1017 // materialised `lineage` set — `JOIN lineage g ON g.branch_id =
1018 // lc.branch_id` — and both then compare `lc.recorded_at` to that lineage's
1019 // cutoff. So the seek is `(branch_id, recorded_at)` and the payload is
1020 // every other column the two arms project.
1021 //
1022 // Without it SQLite builds the index itself, three times per branched read:
1023 // `AUTOMATIC PARTIAL COVERING INDEX (branch_id=?)` twice over
1024 // `links_current` and once over the `links_cut` co-routine. The third is
1025 // not a table and no index can serve it; the first two are, and this is
1026 // them. Measured (`examples/branch_index_rung_probe.rs`, 1,110 edges,
1027 // chain of 10, best of 25):
1028 //
1029 // branched read, no post-fork churn 6.50 -> 5.43 ms 1.20x
1030 // branched read, 10% post-fork churn 16.94 -> 7.45 ms 2.28x
1031 // trunk traversal 1.64 -> 1.64 ms unchanged
1032 // 2,000 assertions 18.3 -> 20.6 ms +12.6%
1033 //
1034 // **Why a second index and not a column on `idx_lc_traversal_cover`,
1035 // against what §15.4 and D-219 both say.** D-219 measured three shapes and
1036 // preferred folding `branch_id` in after the range columns; it measured
1037 // them against `branch_id IN (ancestry)`, which that same probe run proved
1038 // is not a resolution and which 0.14.4 therefore did not ship. Under the
1039 // reader that did ship, the walk joins a CTE and never touches this table,
1040 // so the folded shape is never consulted and buys **nothing** — 6.18 vs
1041 // 6.27 ms, inside the run-to-run spread. And every single-index shape that
1042 // leads on `branch_id` — the one §15.3 proposed included — takes the trunk
1043 // walk off its covering index altogether:
1044 //
1045 // SEARCH l USING INDEX idx_lc_open_interval (source_id=?)
1046 //
1047 // one bound column and not covering, which is what
1048 // `the_shipped_traversal_cte_stays_on_the_covering_index` exists to refuse.
1049 // The two shapes stopped sharing an access path when the reader stopped
1050 // walking `links_current` directly, so they can no longer share an index.
1051 LC_LINEAGE_CUT,
1052 "CREATE INDEX IF NOT EXISTS idx_txlog_time ON transaction_log (recorded_at);",
1053 "CREATE INDEX IF NOT EXISTS idx_txlog_entity ON transaction_log (entity_id);",
1054 // The archive cutoff's seek column on the ledger table (0.12.6, W3.1,
1055 // D-151, review §2.1, shipped v10 -> v11).
1056 //
1057 // `links` carried a primary key and nothing else. `LINKS_ARCHIVABLE` opens
1058 // with `recorded_at < :cutoff`, and the primary key leads on `source_id`, so
1059 // there was nothing for that bound to seek on: both the archiving SELECT and
1060 // the archiving DELETE scanned the entire ledger. Measured on the
1061 // populated-and-analysed fixture in `tests/index_plan_tests.rs`:
1062 //
1063 // before SCAN links | CORRELATED SCALAR SUBQUERY 1 | SEARCH newer ...
1064 // after SEARCH links USING INDEX idx_links_recorded_at (recorded_at<?)
1065 //
1066 // The inner supersession probe was never the problem — it binds the whole
1067 // primary-key prefix and always did.
1068 //
1069 // **This index is justified on those two queries and not on the clock
1070 // floor.** Review §2.1 led with `recorded_at_floor`, the `MAX(recorded_at)`
1071 // read on every `open()`, and counted it among the scans this would close.
1072 // It is not one: SQLite already served the bare `MAX()` from the primary
1073 // key's covering index without traversing the table, and after this index it
1074 // does the same thing through a different covering index. The startup cost
1075 // the review predicted did not exist, so the justification rests entirely on
1076 // the archive path — see D-150 for how that was caught, and D-089 for why an
1077 // index bought on a believed benefit is the failure mode being avoided.
1078 "CREATE INDEX IF NOT EXISTS idx_links_recorded_at ON links (recorded_at);",
1079 // The other half of the concept-archival reachability check (0.12.6, W3.2,
1080 // D-151, review §2.2, shipped v10 -> v11).
1081 //
1082 // `CONCEPTS_ARCHIVABLE` asks whether any surviving link mentions a concept
1083 // *in either direction*: `links.source_id = concepts.id OR links.target_id =
1084 // concepts.id`. The primary key serves the left arm. Nothing served the
1085 // right one, and an `OR` is only as seekable as its worst arm, so the whole
1086 // correlated subquery degraded to a scan of `links` **once per candidate
1087 // concept** — O(concepts x links).
1088 //
1089 // before CORRELATED SCALAR SUBQUERY 1
1090 // | SCAN links USING COVERING INDEX sqlite_autoindex_links_1
1091 // after CORRELATED SCALAR SUBQUERY 1 | MULTI-INDEX OR
1092 // | INDEX 1 | SEARCH links USING COVERING INDEX
1093 // sqlite_autoindex_links_1 (source_id=?)
1094 // | INDEX 2 | SEARCH links USING INDEX
1095 // idx_links_target (target_id=?)
1096 //
1097 // `MULTI-INDEX OR` is SQLite deciding to run both arms as seeks and union
1098 // the rowids, which is exactly the plan the index was added to make
1099 // available. Both the SELECT and the DELETE form pick it up.
1100 //
1101 // A single-column index on the ledger's hottest write path needs the
1102 // strongest justification available, and it has one beyond the plan shape:
1103 // *before* this index the planner was building `AUTOMATIC COVERING INDEX
1104 // (target_id=?)` at query time to answer the same question. It had already
1105 // concluded the index was worth having and was paying to construct a
1106 // throwaway copy per statement.
1107 "CREATE INDEX IF NOT EXISTS idx_links_target ON links (target_id);",
1108];
1109
1110/// Every trigger the schema declares.
1111///
1112/// **`IF NOT EXISTS` means a changed body does not reach an existing file.**
1113/// `migrations::verify` checks trigger *presence by name*, which is deliberate
1114/// (a count refuses healthy databases) but does not and cannot notice that a
1115/// trigger present under the right name carries an older body. A database
1116/// stamped v5 by an earlier build therefore keeps whatever trigger text it was
1117/// created with until a rung drops and recreates it.
1118///
1119/// This is why the payload carries a version. Changing a log trigger's payload
1120/// splits the database population in two — files created after the change write
1121/// the new shape, files created before keep writing the old one — and the only
1122/// thing that makes that survivable is that every reader accepts both. A
1123/// payload change that did *not* bump `v` would be indistinguishable at read
1124/// time from corruption, which is the case `DbError::PayloadVersion` exists for.
1125///
1126/// The v1 → v2 concept payload (defect V) is deliberately left to ride along on
1127/// the next rung that has to move `user_version` anyway rather than claiming one
1128/// of its own: an old file loses `embedding_model` from its temporal reads, which
1129/// is exactly the behaviour it had before, and gains it the moment it is
1130/// migrated. Nothing regresses in the meantime.
1131/// `links_current` maintenance, one row per open belief **per lineage**.
1132///
1133/// A named `const` since v12 for [`CREATE_CONCEPTS_LOG_INSERT`]'s reason: the
1134/// rung has to re-issue this exact body, and a rung with its own copy is a copy
1135/// that drifts. The conflict target matches the table's primary key, which now
1136/// ends in `branch_id` — without that, a branch asserting an edge its parent
1137/// already holds would *overwrite* the parent's row instead of adding its own.
1138pub const CREATE_LINKS_CURRENT_SYNC: &str = r#"
1139 CREATE TRIGGER IF NOT EXISTS trg_links_current_sync
1140 AFTER INSERT ON links
1141 BEGIN
1142 INSERT INTO links_current
1143 (source_id, target_id, edge_type, valid_from, valid_to,
1144 weight, properties, recorded_at, branch_id)
1145 VALUES
1146 (NEW.source_id, NEW.target_id, NEW.edge_type, NEW.valid_from,
1147 NEW.valid_to, NEW.weight, NEW.properties, NEW.recorded_at,
1148 NEW.branch_id)
1149 ON CONFLICT(source_id, target_id, edge_type, valid_from, branch_id) DO UPDATE SET
1150 valid_to = excluded.valid_to,
1151 weight = excluded.weight,
1152 properties = excluded.properties,
1153 recorded_at = excluded.recorded_at
1154 WHERE excluded.recorded_at > links_current.recorded_at;
1155 END;
1156"#;
1157
1158/// One open interval per edge **per lineage** (§4.3, branch-scoped at v12).
1159///
1160/// The `branch_id` clause is row-level and deliberately not ancestry-aware. A
1161/// branch that inherits an open interval from its parent and asserts its own is
1162/// not violating this rule — it is superseding a belief, which is the thing a
1163/// branch is for.
1164///
1165/// # The question this comment parked, answered at 0.14.8 (D-225)
1166///
1167/// *Whether the inherited interval should also close.* It should not, and
1168/// cannot: closing the ancestor's row is the parent corruption Doctrine III
1169/// forbids, and `links` is append-only so no statement in the crate could do
1170/// it. What a branch writes instead is its **own** row at the ancestor's key,
1171/// which the read prefers by `dist` — shadow retirement.
1172///
1173/// The half a trigger genuinely cannot answer went to the Rust layer, where
1174/// the ancestry is reachable: `lineage::overlap_candidates_resolved` refuses an
1175/// assertion whose interval overlaps **what the writing lineage can see**,
1176/// which is the read's definition applied to the write. That is a guard against
1177/// callers going through the actor and not against raw SQL, which is the same
1178/// honest cost `reject_overlapping_interval` has carried since D-060 — a
1179/// trigger able to make it would need a recursive ancestry walk on every
1180/// insert, on the path D-059 exists to keep fast.
1181pub const CREATE_LINKS_SINGLE_OPEN: &str = concat!(
1182 r#"
1183 CREATE TRIGGER IF NOT EXISTS trg_links_single_open
1184 BEFORE INSERT ON links
1185 WHEN NEW.valid_to = '9999-12-31T23:59:59.999999Z'
1186 AND EXISTS (
1187 SELECT 1 FROM links_current
1188 WHERE source_id = NEW.source_id
1189 AND target_id = NEW.target_id
1190 AND edge_type = NEW.edge_type
1191 AND branch_id = NEW.branch_id
1192 AND valid_from <> NEW.valid_from
1193 AND valid_to = '9999-12-31T23:59:59.999999Z'
1194 )
1195 BEGIN
1196 SELECT RAISE(ABORT, '"#,
1197 abort_single_open!(),
1198 r#"');
1199 END;
1200 "#
1201);
1202
1203/// The update half of the concepts log. See [`CREATE_CONCEPTS_LOG_INSERT`].
1204///
1205/// Unconditional where its insert sibling is marker-gated, and the asymmetry is
1206/// deliberate: nothing inside an archive session updates a concept, so gating
1207/// this would suppress nothing.
1208///
1209/// `branch_id` is in the column list since v12 and the omission would have been
1210/// expensive. `concepts` permits a **same-lineage** update — the guards refuse
1211/// cross-lineage inserts and `branch_id` changes, not this — so a branch
1212/// correcting a concept it minted would have logged the change against `'main'`,
1213/// putting a branch's own history in the trunk's fold and leaving the row
1214/// invisible to the abandonment sweep that §15.5's `archive` arm performs.
1215pub const CREATE_CONCEPTS_LOG_UPDATE: &str = r#"
1216 CREATE TRIGGER IF NOT EXISTS trg_concepts_log_update
1217 AFTER UPDATE ON concepts
1218 BEGIN
1219 INSERT INTO transaction_log (table_name, entity_id, operation, payload, recorded_at, branch_id)
1220 VALUES ('concepts', NEW.id, 'U',
1221 json_object('v', 2, 'title', NEW.title, 'content', NEW.content,
1222 'valid_from', NEW.valid_from, 'valid_to', NEW.valid_to,
1223 'retired', NEW.retired,
1224 'embedding_model', NEW.embedding_model),
1225 NEW.recorded_at, NEW.branch_id);
1226 END;
1227"#;
1228
1229/// The links log, and the entry whose `entity_id` is composed rather than copied.
1230///
1231/// `source|target|type|valid_from` identifies an edge assertion and carries **no
1232/// lineage**, which is why `branch_id` had to become a column of its own rather
1233/// than a fifth field in that string. Re-keying `entity_id` was the other
1234/// option and was rejected: it changes what a log entry identifies, so rows
1235/// written before the rung would no longer match rows written after it, and the
1236/// fold would silently split one edge's history in two.
1237///
1238/// With the column present, the four folds in `temporal::replay` — a private
1239/// module, so the name is plain text rather than a link that would not resolve —
1240/// partition by `(table_name, entity_id, branch_id)` and two lineages'
1241/// assertions about one edge stay two beliefs. Without it they collapse to
1242/// whichever has the higher `seq_id` — no error, no drift report, just one
1243/// lineage's belief gone.
1244pub const CREATE_LINKS_LOG_INSERT: &str = r#"
1245 CREATE TRIGGER IF NOT EXISTS trg_links_log_insert
1246 AFTER INSERT ON links
1247 BEGIN
1248 INSERT INTO transaction_log (table_name, entity_id, operation, payload, recorded_at, branch_id)
1249 VALUES ('links',
1250 NEW.source_id || '|' || NEW.target_id || '|' || NEW.edge_type || '|' || NEW.valid_from,
1251 'I',
1252 json_object('v', 1, 'source_id', NEW.source_id, 'target_id', NEW.target_id,
1253 'edge_type', NEW.edge_type, 'valid_from', NEW.valid_from,
1254 'valid_to', NEW.valid_to, 'weight', NEW.weight,
1255 'properties', json(NEW.properties)),
1256 NEW.recorded_at, NEW.branch_id);
1257 END;
1258"#;
1259
1260/// The ledger's delete guard, named since v15 (0.14.15, [D-232]).
1261///
1262/// An anonymous entry in [`CREATE_TRIGGERS`] until a rung needed to put it
1263/// back: the v14 → v15 rung rebuilds `links`, `DROP TABLE` takes its four
1264/// triggers with it, and a rung cannot re-issue a body it has no name for.
1265/// Promoted rather than copied, which is the rule
1266/// [`CREATE_LINKS_CURRENT_SYNC`] states — a rung with its own copy of a trigger
1267/// is a copy that drifts. The v12 rung promoted four bodies for exactly this
1268/// reason; this is the fifth.
1269///
1270/// D-008 (revised): probe `main.sqlite_master` for the archive-session marker.
1271/// SQLite forbids a trigger in `main` from referencing objects in another
1272/// database, temp included, so the original `temp.sqlite_master` probe fails at
1273/// `CREATE TRIGGER` time and is unimplementable.
1274pub const CREATE_LINKS_GUARD_DELETE: &str = concat!(
1275 r#"
1276 CREATE TRIGGER IF NOT EXISTS trg_links_guard_delete
1277 BEFORE DELETE ON links
1278 WHEN NOT EXISTS (
1279 SELECT 1 FROM sqlite_master
1280 WHERE type = 'table' AND name = 'macrame_archive_session'
1281 )
1282 BEGIN
1283 SELECT RAISE(ABORT, '"#,
1284 abort_delete_guard!(),
1285 r#"');
1286 END;
1287 "#
1288);
1289
1290pub const CREATE_TRIGGERS: &[&str] = &[
1291 CREATE_LINKS_CURRENT_SYNC,
1292 CREATE_LINKS_SINGLE_OPEN,
1293 concat!(
1294 r#"
1295 CREATE TRIGGER IF NOT EXISTS trg_concepts_monotonic_ra
1296 BEFORE UPDATE ON concepts
1297 WHEN NEW.recorded_at <= OLD.recorded_at
1298 BEGIN
1299 SELECT RAISE(ABORT, '"#,
1300 abort_monotonic_ra!(),
1301 r#"');
1302 END;
1303 "#
1304 ),
1305 // Payload v2 adds `embedding_model` (defect V). Before it, the field was
1306 // written by nobody and read by two — `replay::fold_delta` and
1307 // `as_of::hydrate_attributes` both asked the payload for it and both always
1308 // saw null, so `AttributeMode::AtTime`, the faithful mode Doctrine VIII
1309 // exists to offer, returned a *less* complete record than `Current`.
1310 //
1311 // The version number moves because the shape is a compat surface: readers
1312 // must be able to tell "this build wrote no model" from "this payload
1313 // predates the field". v1 is still accepted and folds with the field absent,
1314 // which is what makes this safe without a migration rung — see the note on
1315 // [`CREATE_TRIGGERS`].
1316 CREATE_CONCEPTS_LOG_INSERT,
1317 CREATE_CONCEPTS_LOG_UPDATE,
1318 CREATE_LINKS_LOG_INSERT,
1319 CREATE_CONCEPTS_GUARD_DELETE,
1320 // v12 (§15.2, D-214). Order matters only in that every one of these names a
1321 // table the baseline has already created; `verify` recovers the names from
1322 // this array, so a trigger added here is a trigger the ladder must produce.
1323 CREATE_CONCEPTS_GUARD_LINEAGE,
1324 CREATE_CONCEPTS_GUARD_BRANCH,
1325 CREATE_BRANCHES_GUARD_UPDATE,
1326 CREATE_BRANCHES_GUARD_DELETE,
1327 CREATE_LINKS_GUARD_DELETE,
1328 concat!(
1329 r#"
1330 CREATE TRIGGER IF NOT EXISTS trg_txlog_guard_delete
1331 BEFORE DELETE ON transaction_log
1332 WHEN NOT EXISTS (
1333 SELECT 1 FROM sqlite_master
1334 WHERE type = 'table' AND name = 'macrame_archive_session'
1335 )
1336 BEGIN
1337 SELECT RAISE(ABORT, '"#,
1338 abort_delete_guard!(),
1339 r#"');
1340 END;
1341 "#
1342 ),
1343 // --- FTS sync (§5.9) ------------------------------------------------
1344 //
1345 // These write to `concepts_fts` and to nothing else. In particular they do
1346 // not touch `transaction_log`: an FTS index is derived from concept text
1347 // the ledger already records, so logging it would record the same fact
1348 // twice — the reasoning Doctrine VII applies to embeddings, and the reason
1349 // `doctrine_static_tests` scans this array.
1350 r#"
1351 CREATE TRIGGER IF NOT EXISTS trg_concepts_fts_insert
1352 AFTER INSERT ON concepts
1353 BEGIN
1354 INSERT INTO concepts_fts (rowid, title, content)
1355 VALUES (NEW.rowid_pk, NEW.title, NEW.content);
1356 END;
1357 "#,
1358 // The retraction is not optional and not symmetric with the insert. An
1359 // external-content FTS5 index stores terms, not text, so replacing a row
1360 // means telling it which terms to *remove* — and it needs the old column
1361 // values to work that out. Omit this and the index keeps matching words the
1362 // concept no longer contains, with no error and no way to notice except by
1363 // searching for something that is no longer there.
1364 r#"
1365 CREATE TRIGGER IF NOT EXISTS trg_concepts_fts_update
1366 AFTER UPDATE ON concepts
1367 BEGIN
1368 INSERT INTO concepts_fts (concepts_fts, rowid, title, content)
1369 VALUES ('delete', OLD.rowid_pk, OLD.title, OLD.content);
1370 INSERT INTO concepts_fts (rowid, title, content)
1371 VALUES (NEW.rowid_pk, NEW.title, NEW.content);
1372 END;
1373 "#,
1374 // The third trigger, installed **inert** by v8 (§4.6, D-119).
1375 //
1376 // Through v7 this array had no delete trigger, and the stated reason was
1377 // that `trg_concepts_guard_delete` is unconditional (D-022) so no delete
1378 // path exists to keep in sync. That was true and it was the wrong shape:
1379 // the index's correctness depended on a *different* trigger staying
1380 // unconditional, and nothing connected the two except a comment.
1381 //
1382 // It cannot fire today — the guard is a `BEFORE DELETE` that always aborts,
1383 // so the statement never reaches `AFTER DELETE`. It is here because 0.9.0's
1384 // archive session is what makes the guard conditional, and the moment that
1385 // lands the index would go silently stale without this. Installing the
1386 // capability in the rung that is already rebuilding the table costs nothing.
1387 //
1388 // It does **not** mean 0.9.0 needs no migration of its own — that claim was
1389 // written here and it is wrong (D-126, corrected 0.8.0 pre-tag). This trigger
1390 // is C2's step 3; step 2 is making `trg_concepts_guard_delete` conditional,
1391 // and that is a `v8 → v9` rung, because `CREATE TRIGGER IF NOT EXISTS` on an
1392 // existing name keeps the **old body** and `verify()` compares names only, so
1393 // a re-issued baseline would leave the unconditional guard in place and pass.
1394 // Deliberately not fixed here: the archive-session marker exists during
1395 // *links* archival too, so a conditional concepts guard shipped in 0.8.0
1396 // would leave concepts deletable during those sessions.
1397 //
1398 // `the_fts_delete_trigger_is_installed_and_inert` (wave1_regression_tests)
1399 // pins both halves rather than assuming either.
1400 r#"
1401 CREATE TRIGGER IF NOT EXISTS trg_concepts_fts_delete
1402 AFTER DELETE ON concepts
1403 BEGIN
1404 INSERT INTO concepts_fts (concepts_fts, rowid, title, content)
1405 VALUES ('delete', OLD.rowid_pk, OLD.title, OLD.content);
1406 END;
1407 "#,
1408];
1409
1410#[cfg(test)]
1411mod tests {
1412 use crate::util::timestamp::{CANONICAL_TS_GLOB, OPEN_SENTINEL};
1413
1414 /// The DDL's CHECK pattern and the Rust-side pattern must be the same
1415 /// pattern. If they drift, one layer accepts what the other rejects and the
1416 /// canonical-form invariant is enforced in name only.
1417 #[test]
1418 fn ddl_glob_matches_the_rust_canonical_pattern() {
1419 assert_eq!(format!("'{}'", ts_glob!()), CANONICAL_TS_GLOB);
1420 }
1421
1422 /// Every DDL statement that declares a temporal default must use the
1423 /// canonical sentinel; a second-precision default would be rejected by the
1424 /// very CHECK sitting next to it.
1425 #[test]
1426 fn ddl_defaults_use_the_canonical_sentinel() {
1427 for ddl in [
1428 super::CREATE_CONCEPTS_TABLE,
1429 super::CREATE_LINKS_TABLE,
1430 super::CREATE_LINKS_CURRENT_TABLE,
1431 super::CREATE_TRANSACTION_LOG_TABLE,
1432 ] {
1433 assert!(
1434 !ddl.contains("9999-12-31T23:59:59Z"),
1435 "DDL still carries the pre-0.5.4 second-precision sentinel: {ddl}"
1436 );
1437 }
1438 for trigger in super::CREATE_TRIGGERS {
1439 assert!(
1440 !trigger.contains("9999-12-31T23:59:59Z"),
1441 "trigger still carries the pre-0.5.4 sentinel: {trigger}"
1442 );
1443 }
1444 assert!(super::CREATE_LINKS_TABLE.contains(OPEN_SENTINEL));
1445 }
1446
1447 /// Every abort message the classifier matches on must actually appear in the
1448 /// DDL that emits it. `concat!` makes this true by construction today; the
1449 /// test is what keeps it true if someone re-inlines a literal.
1450 #[test]
1451 fn every_abort_message_appears_in_a_trigger() {
1452 for msg in [
1453 super::ABORT_SINGLE_OPEN,
1454 super::ABORT_MONOTONIC_RA,
1455 super::ABORT_DELETE_GUARD,
1456 ] {
1457 assert!(
1458 super::CREATE_TRIGGERS.iter().any(|t| t.contains(msg)),
1459 "no trigger emits {msg:?}, so its typed error is unreachable"
1460 );
1461 }
1462 }
1463}