macrame/schema/ddl.rs
1//! DDL statements for the Macrame bitemporal schema as specified in §4.
2
3/// GLOB pattern matching the canonical timestamp form `YYYY-MM-DDTHH:MM:SS.ffffffZ`.
4///
5/// A macro rather than a `const` so it can be spliced into the DDL literals by
6/// `concat!`, which only accepts literals. Kept byte-identical to
7/// [`crate::util::timestamp::CANONICAL_TS_GLOB`] by the unit test at the bottom
8/// of this file — the storage-layer guard and the Rust-layer guard must agree
9/// or one of them is decorative.
10macro_rules! ts_glob {
11 () => {
12 "[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]T[0-9][0-9]:[0-9][0-9]:[0-9][0-9].[0-9][0-9][0-9][0-9][0-9][0-9]Z"
13 };
14}
15
16/// Table-level CHECK asserting every temporal column is canonical (§4.1, 0.5.4).
17///
18/// Timestamps are compared lexicographically everywhere — in SQL predicates, in
19/// `MAX(recorded_at)` when the clock recovers its floor, and in Rust `str`
20/// ordering. That is sound only if every value has the same width, so mixing
21/// `...T00:00:00Z` with `...T00:00:00.000000Z` makes `<=` disagree with
22/// chronology and traversals return empty sets with no error. The `Z` suffix
23/// alone does not achieve this; a fixed width does, and a CHECK is what makes
24/// it a property of the data rather than a convention.
25macro_rules! canonical_ts_check {
26 ($($col:literal),+ $(,)?) => {
27 concat!("CHECK (", $( $col, " GLOB '", ts_glob!(), "' AND ", )+ "1)")
28 };
29}
30
31/// A macro rather than a `const` for the same reason as [`ts_glob`]: `concat!`
32/// splices it into the table DDL and only accepts literals. [`WEIGHT_CHECK`] is
33/// the same text as a value, and carries the reasoning.
34macro_rules! weight_check {
35 () => {
36 "CHECK (weight >= 0.0 AND weight < 9e999 AND typeof(weight) = 'real')"
37 };
38}
39
40/// Table-level CHECK on `links.weight` (§4.7, T2.1, D-083).
41///
42/// Three clauses. Only the first is the one the item asked for; the other two
43/// were found by probing what the first still admits.
44///
45/// `weight >= 0.0` is the item as written: shortest-path analytics are unsound
46/// over negative weights, so Dijkstra and A\* refuse the graph at load time
47/// (D-039). Until now that refusal was the *only* place the property was
48/// enforced, which made it §4.7's one genuinely open gap — a database this crate
49/// wrote by itself could hold a row this crate would not read back.
50///
51/// `typeof(weight) = 'real'` closes a hole the item does not mention and which
52/// probing found. `REAL` in SQLite is an **affinity**, not a type: values that
53/// can be converted are, and values that cannot are stored as they came. `'abc'`
54/// cannot become a number, so it is stored as TEXT — and in SQLite's type
55/// ordering every text value sorts above every numeric one, so `'abc' >= 0.0`
56/// is *true* and the first clause passes it through.
57///
58/// That is not a wrong answer on the read side. It is a **panic**: reading a
59/// text `weight` as `f64` reaches `unreachable!("invalid value type")` inside
60/// libsql 0.9.30, in whatever unrelated query first touches the row. Measured,
61/// not reasoned about — see `examples/weight_check_probe.rs`.
62///
63/// The clause costs one `typeof` per insert and refuses nothing legitimate:
64/// `3`, `'5'` and `1.0` all arrive as REAL through affinity conversion and pass.
65/// It is taken **now** rather than in a later rung because SQLite has no
66/// `ADD CONSTRAINT` — every clause added later costs another full rebuild of the
67/// largest table in the schema.
68///
69/// `weight < 9e999` refuses `+∞`, and the reason is not the one anybody
70/// predicted. The plan expected the CHECK to admit infinity and argued the
71/// loader guard would catch it; the guard tests `< 0.0` and `is_nan()`, so it
72/// does not. The next guess — mine — was that this is harmless, since IEEE
73/// infinity propagates through addition and stays totally ordered, leaving
74/// Dijkstra terminating with "that edge is unusable": an odd answer, not a wrong
75/// one.
76///
77/// Both were wrong, and a test found it. **An infinite weight makes the
78/// transaction log unreplayable.** The log trigger serialises the row to JSON,
79/// and JSON has no representation for infinity, so the payload round-trips into
80/// `ReplayCorrupt { reason: "number out of range" }` — every later
81/// `reconstruct()` fails, including the one `close()` performs. The ledger is
82/// the source of truth under Doctrine III, so a value that cannot survive the
83/// log is not an eccentric weight, it is a corrupt one.
84///
85/// `9e999` is the idiom because SQLite has no `isinf`: the literal overflows to
86/// `+∞` on parse, and `inf < inf` is false. Finite values, including `1e308`,
87/// pass.
88///
89/// The loader guard still **stays**, for the reason the constraint cannot cover:
90/// `links_current` carries no CHECK, and neither do cold files created before
91/// this rung.
92pub const WEIGHT_CHECK: &str = weight_check!();
93
94/// The `RAISE(ABORT, …)` messages the schema's guards emit (§4.3).
95///
96/// Spliced into the trigger DDL *and* matched by [`crate::error::abort_kind`],
97/// so the guard and its classifier cannot drift. When they drift the failure is
98/// silent in the worst direction: the guard still fires, but the typed error
99/// (`SingleOpenViolation`, `RecordedAtRegression`, `ArchiveViolation`) degrades
100/// into an opaque `Engine` error that no caller can match on.
101macro_rules! abort_single_open {
102 () => {
103 "macrame: edge already has an open interval; retire it first"
104 };
105}
106macro_rules! abort_monotonic_ra {
107 () => {
108 "macrame: concept recorded_at must be strictly increasing"
109 };
110}
111macro_rules! abort_delete_guard {
112 () => {
113 "macrame: physical delete blocked outside archive session"
114 };
115}
116macro_rules! abort_cross_lineage {
117 () => {
118 "macrame: concept belongs to another lineage; a branch inherits concepts, it does not restate them"
119 };
120}
121macro_rules! abort_branch_immutable {
122 () => {
123 "macrame: branch_id is provenance and cannot be changed"
124 };
125}
126macro_rules! abort_branches_frozen {
127 () => {
128 "macrame: branch records are append-only"
129 };
130}
131
132pub const ABORT_SINGLE_OPEN: &str = abort_single_open!();
133pub const ABORT_MONOTONIC_RA: &str = abort_monotonic_ra!();
134pub const ABORT_DELETE_GUARD: &str = abort_delete_guard!();
135pub const ABORT_CROSS_LINEAGE: &str = abort_cross_lineage!();
136pub const ABORT_BRANCH_IMMUTABLE: &str = abort_branch_immutable!();
137pub const ABORT_BRANCHES_FROZEN: &str = abort_branches_frozen!();
138
139/// The root lineage every pre-v12 row is stamped with (§15.2, v12, D-214).
140///
141/// A macro as well as a `const` for [`ts_glob`]'s reason: it is spliced into
142/// the column defaults by `concat!`, which takes only literals. One spelling,
143/// so the default in the DDL, the seed row, and the Rust layer cannot drift
144/// into three databases that disagree about what the trunk is called.
145macro_rules! main_branch {
146 () => {
147 "main"
148 };
149}
150pub const MAIN_BRANCH: &str = main_branch!();
151
152/// The `branch_id` column, identical on all four ledger tables (§15.2, D-214).
153///
154/// The default is what makes the rung `ALTER TABLE` rather than a rewrite —
155/// SQLite records a constant default in the schema header and rewrites no row,
156/// measured at 83–139 µs over 20,000 rows in `examples/branch_identity_probe.rs`
157/// §1.
158///
159/// # The `REFERENCES` clause, and the condition it is actually gated on
160///
161/// SQLite specifies that a column added by `ALTER TABLE … ADD COLUMN` carrying
162/// a `REFERENCES` clause **must default to NULL** when foreign keys are
163/// enabled, because pre-existing rows cannot be validated against the new
164/// parent. That collides head-on with `NOT NULL DEFAULT 'main'`.
165///
166/// libSQL 0.9.30 applies that rule **dynamically rather than statically**, and
167/// probe §15 pins the four cases: the statement is refused only when the table
168/// **holds rows** *and* foreign keys are **on**. An empty table takes it with
169/// keys on; a populated table takes it with keys off. Being inside a
170/// transaction changes nothing either way.
171///
172/// This is the whole reason the v11 → v12 rung sets
173/// [`suspends_foreign_keys`]. It is worth being exact about what that buys,
174/// because "suspend the constraint to install the constraint" invites the
175/// suspicion that the result is decorative — §15 measures it and it is not.
176/// After an ALTER taken with keys suspended the clause is in `sqlite_master`,
177/// `PRAGMA foreign_key_list(concepts)` reports the key, an insert naming an
178/// unknown branch is refused with extended code 787, and deleting a referenced
179/// branch is refused **by the engine** rather than by a trigger. Enforcement is
180/// a per-connection pragma; the constraint is schema. Suspending the first
181/// never weakened the second.
182///
183/// Nor does the suspension launder a violation past the commit: `apply_step`
184/// runs `PRAGMA foreign_key_check` inside the transaction, and §15 confirms it
185/// reports the orphan when one is deliberately planted during the window.
186///
187/// Taken deliberately, with the dependency named in §19 rather than absorbed.
188/// The exposure is narrow and it is on the **upgrade** path only: fresh
189/// databases put the clause in a `CREATE TABLE`, where no engine has ever
190/// disputed it. If upstream ever tightens to SQLite's static reading, the rung
191/// fails **loudly** — at a named step, inside `BEGIN IMMEDIATE`, leaving the
192/// database honestly at v11 — and the fallback is one line: drop the clause
193/// from the ALTER and let the `branches` write guard carry lineage integrity
194/// alone, which is the weaker guarantee and the one D-030 has a name for.
195///
196/// [`suspends_foreign_keys`]: super::migrations
197macro_rules! branch_column {
198 () => {
199 concat!(
200 "branch_id TEXT NOT NULL DEFAULT '",
201 main_branch!(),
202 "' REFERENCES branches(branch_id)"
203 )
204 };
205}
206pub const BRANCH_COLUMN: &str = branch_column!();
207
208/// Marker table probed by the delete guards (D-008 revised).
209///
210/// The archive session creates this table and drops it again inside the single
211/// `BEGIN IMMEDIATE … COMMIT` archive transaction, so it never exists as
212/// committed state. Connection-locality — the property the original
213/// `temp.sqlite_master` probe was reaching for — is preserved by two
214/// independent mechanisms: uncommitted DDL is visible only to the writing
215/// connection, and the archive transaction holds the write lock for its
216/// duration, so no other connection can reach the guard at all.
217pub const ARCHIVE_SESSION_MARKER: &str = "macrame_archive_session";
218
219/// The concepts insert log trigger, **marker-gated since v10** (0.9.0, C3).
220///
221/// # Why an archive session must not log a concept insert
222///
223/// Rehydration is a physical move back and mints no transaction-time facts
224/// (§2.3): the concept returns to the hot table, the log entries describing it
225/// were never removed, and nothing about what was believed — or when — has
226/// changed. An unconditional `AFTER INSERT` makes that impossible to honour,
227/// because the move *is* an insert.
228///
229/// **And the damage is worse than a spurious row, which is what forced the
230/// rung.** The rehydrated row carries its **original** `recorded_at`, but the
231/// log row it would write gets a **new** `seq_id` at the end of the log. The
232/// fold partitions by `(table_name, entity_id)` and takes
233/// `ROW_NUMBER() OVER (… ORDER BY seq_id DESC) = 1` — last writer wins by
234/// *sequence*, not by timestamp. So the rehydration `'I'` would outrank the
235/// later `'U'` that retired the concept, and every `reconstruct` after the
236/// original creation time would return it **un-retired**. Rehydration would
237/// resurrect a belief the ledger had superseded, silently and retroactively,
238/// which is precisely what [Doctrine III] forbids.
239///
240/// Only the *insert* trigger is gated. `trg_concepts_log_update` stays
241/// unconditional because nothing inside a session updates a concept — archival
242/// deletes and rehydration inserts — so gating it would suppress nothing and
243/// widen the hole for no reason.
244pub const CREATE_CONCEPTS_LOG_INSERT: &str = concat!(
245 r#"
246 CREATE TRIGGER IF NOT EXISTS trg_concepts_log_insert
247 AFTER INSERT ON concepts
248 WHEN NOT EXISTS (
249 SELECT 1 FROM sqlite_master
250 WHERE type = 'table' AND name = '"#,
251 "macrame_archive_session",
252 r#"'
253 )
254 BEGIN
255 INSERT INTO transaction_log (table_name, entity_id, operation, payload, recorded_at, branch_id)
256 VALUES ('concepts', NEW.id, 'I',
257 json_object('v', 2, 'title', NEW.title, 'content', NEW.content,
258 'valid_from', NEW.valid_from, 'valid_to', NEW.valid_to,
259 'retired', NEW.retired,
260 'embedding_model', NEW.embedding_model),
261 NEW.recorded_at, NEW.branch_id);
262 END;
263 "#
264);
265
266/// The concepts delete guard, **marker-gated since v9** (0.9.0, C2, D-126).
267///
268/// A `pub const` rather than an anonymous entry in [`CREATE_TRIGGERS`] because
269/// two readers need exactly this text: the baseline, which installs it on a new
270/// database, and the `v8 → v9` rung, which replaces the v8 body on an existing
271/// one. A second copy is a copy that drifts, and this trigger is the one whose
272/// body carries a doctrine decision.
273///
274/// # What changed, and why re-issuing the baseline could not do it
275///
276/// Through v8 this guard was **unconditional**: `BEFORE DELETE ON concepts`
277/// aborting every time, on the reasoning that concepts are never physically
278/// archived ([D-022](../../docs/architecture/s13-decision-register.md)). C2
279/// makes that false — a declared archive session may now move a retired,
280/// unreferenced concept to the cold file — so the guard takes the same shape its
281/// two siblings have had since 0.5.3: it fires **unless** the archive-session
282/// marker is present.
283///
284/// It needs a rung of its own, and that was measured rather than assumed
285/// (D-126). `CREATE TRIGGER IF NOT EXISTS` on an existing name keeps the **old
286/// body** — re-issuing the baseline against a v8 database leaves the
287/// unconditional guard exactly where it was — and `verify` compared `type` and
288/// `name` and never bodies, so the stale guard passed verification in silence.
289/// Both halves are now closed: the rung drops and recreates, and `verify`
290/// checks that every delete guard's body probes the marker.
291pub const CREATE_CONCEPTS_GUARD_DELETE: &str = concat!(
292 r#"
293 CREATE TRIGGER IF NOT EXISTS trg_concepts_guard_delete
294 BEFORE DELETE ON concepts
295 WHEN NOT EXISTS (
296 SELECT 1 FROM sqlite_master
297 WHERE type = 'table' AND name = '"#,
298 "macrame_archive_session",
299 r#"'
300 )
301 BEGIN
302 SELECT RAISE(ABORT, '"#,
303 abort_delete_guard!(),
304 r#"');
305 END;
306 "#
307);
308
309/// The `concepts` ledger table (§4.1).
310///
311/// # `rowid_pk` is explicit, and that is the whole point (v8, D-119)
312///
313/// Through v7 this table declared `id TEXT PRIMARY KEY`, which left its rowid
314/// **implicit** — and `concepts_fts` is external-content keyed on that rowid.
315/// `VACUUM` renumbers implicit rowids, which would silently decouple the search
316/// index from the rows it indexes: no error, no integrity-check failure, just
317/// results that stop matching.
318///
319/// [D-071](../../docs/architecture/s13-decision-register.md) proved the hazard
320/// unreachable *by consequence rather than by design* — `trg_concepts_guard_delete`
321/// is unconditional, so rowids are dense `1..n` and `VACUUM`'s renumbering is
322/// the identity map. 0.9.0's archival makes them sparse and makes the hazard
323/// real, so v8 replaces the accident with a column: an `INTEGER PRIMARY KEY` is
324/// a stored value, and `VACUUM` preserves it whether the numbering is dense or
325/// not (measured in `examples/concepts_rebuild_probe.rs` §5).
326///
327/// SQLite permits one primary key per table, so `id` becomes `NOT NULL UNIQUE`.
328/// That keeps it a valid foreign-key parent for `links.source_id` /
329/// `links.target_id` and keeps `ON CONFLICT(id)` working, but it **is** a
330/// primary-key change — which [D-036](../../docs/architecture/s13-decision-register.md)
331/// forbids outright after 1.0. Taken pre-1.0 on purpose, or never.
332/// The lineage register (§15.2, v12, D-214).
333///
334/// Four columns and no more, because a branch is **not** a third temporal axis
335/// (Doctrine II): it carries no interval of its own, only the point in the
336/// second clock where it diverged.
337///
338/// `parent_id` is a self-referencing foreign key, declarable here because it
339/// sits in a `CREATE TABLE` where SQLite permits forward and self references
340/// freely. `NULL` marks the root, and the paired `CHECK` makes "root" a single
341/// state rather than two columns that can disagree: a row with a parent and no
342/// fork point is a lineage whose ancestry cannot be resolved, and a row with a
343/// fork point and no parent is a divergence from nothing.
344///
345/// `forked_at` is in the **`recorded_at` domain** — the transaction-time
346/// instant the lineage diverged, which is what §15.3's visibility cutoffs are
347/// computed over. Not a valid-time bound: a branch does not believe things
348/// about a period, it believes them from a moment onward.
349///
350/// The ordering `CHECK` is row-local on purpose. `forked_at <= created_at` is
351/// checkable from the row itself; an ordering against the *parent's* row is
352/// not, and a `CHECK` cannot see another row. The cross-row half is `fork()`'s
353/// to enforce at D-034's boundary, and saying so here is cheaper than a
354/// constraint that looks complete and is not.
355///
356/// **Which cross-row ordering, corrected in 0.14.7.** This said "the fork point
357/// is at or after the parent's *creation*" from v12 until `fork()` existed to
358/// enforce it, and that turned out to be uncheckable rather than merely
359/// unenforced: [`seed_root_branch`](crate::schema) stamps the trunk's
360/// `created_at` from `SystemTime::now()` during migration — before the
361/// database's injected clock is resolved, and it cannot simply run after,
362/// because the clock's floor is read from tables the migration creates. So
363/// `created_at` is not on the ledger's timeline and comparing a `forked_at` to
364/// it is comparing two clocks. What [`Database::fork`](crate::Database::fork) enforces instead is
365/// `forked_at >= parent.forked_at`, both issued by the same clock, which makes
366/// fork points non-decreasing down any root path.
367pub const CREATE_BRANCHES_TABLE: &str = concat!(
368 r#"
369CREATE TABLE IF NOT EXISTS branches (
370 branch_id TEXT NOT NULL PRIMARY KEY,
371 parent_id TEXT REFERENCES branches(branch_id),
372 forked_at TEXT,
373 created_at TEXT NOT NULL,
374 CHECK ((parent_id IS NULL) = (forked_at IS NULL)),
375 CHECK (forked_at IS NULL OR forked_at <= created_at),
376 CHECK (forked_at IS NULL OR forked_at GLOB '"#,
377 ts_glob!(),
378 r#"'),
379 "#,
380 canonical_ts_check!("created_at"),
381 r#"
382);
383"#
384);
385
386/// Seed the root lineage, idempotently.
387///
388/// One statement shared by the baseline and the v11 → v12 rung, taking
389/// `created_at` as a parameter. `OR IGNORE` rather than `IF NOT EXISTS`
390/// gymnastics because both callers may run against a database that already has
391/// the row — the rung on a retry, the baseline never, but a single statement
392/// that is safe for both is one fewer thing to reason about.
393///
394/// This must run **before** any row is stamped, on both paths: every
395/// `branch_id` default names `'main'`, and the foreign key means a database
396/// without this row cannot accept a single write.
397pub const SEED_MAIN_BRANCH: &str = concat!(
398 "INSERT OR IGNORE INTO branches (branch_id, parent_id, forked_at, created_at) \
399 VALUES ('",
400 main_branch!(),
401 "', NULL, NULL, ?1)"
402);
403
404/// `branches` is append-only outside an archive session (§15.2, §15.4).
405///
406/// The two guards no longer say the same thing, and 0.14.13 is where they
407/// parted. This one stays **unconditional**: no session of any kind may edit a
408/// lineage record in place. [`CREATE_BRANCHES_GUARD_DELETE`] is now gated on
409/// the archive-session marker like its three siblings, because
410/// [`crate::Database::archive_branch`] made removing a lineage record a legal
411/// operation — see that guard for what changed and why the change needed a
412/// rung of its own.
413///
414/// # Why `UPDATE` is refused whole-row
415///
416/// The foreign key already refuses renaming or deleting a lineage any row
417/// still points at, so this guard is not what keeps the ledger from being
418/// orphaned. What it keeps is narrower and harder to see: `parent_id` and
419/// `forked_at` are the inputs to ancestry, so editing either **re-derives the
420/// visibility of rows already written**, with no new assertion anywhere. That
421/// is the move [Doctrine III] forbids, reachable by one raw-SQL statement, and
422/// no foreign key has anything to say about it.
423///
424/// Whole-row rather than a named subset because nothing on the row legitimately
425/// changes, and a whole-row guard needs no maintenance the day a column is
426/// added.
427///
428/// [Doctrine III]: ../../docs/architecture/README.md
429pub const CREATE_BRANCHES_GUARD_UPDATE: &str = concat!(
430 r#"
431 CREATE TRIGGER IF NOT EXISTS trg_branches_frozen_update
432 BEFORE UPDATE ON branches
433 BEGIN
434 SELECT RAISE(ABORT, '"#,
435 abort_branches_frozen!(),
436 r#"');
437 END;
438 "#
439);
440
441/// The delete half of the rule, **marker-gated since v13** (0.14.13, §15.4,
442/// [D-230](../../docs/architecture/s13-decision-register.md#d-230)).
443///
444/// # What changed
445///
446/// Through v12 this guard was unconditional, and its own docstring said why:
447/// *"there is no session in which removing a lineage record is legal — branches
448/// are never archived"*. [`crate::Database::archive_branch`] makes that false.
449/// The sentence was a true description of the operations that existed, written
450/// as though it were a property of the table, which is the shape D-035 asks to
451/// be stated rather than assumed.
452///
453/// **The lineage row must move, and that is forced rather than chosen.** An
454/// abandonment arm that took the branch's `links` and left its `branches` row
455/// would leave `hot_log_reach` unsound: that probe's argument rests on *the
456/// newest row per entity is never archivable*, which holds for a predicate
457/// needing a later row to exist and fails for one that takes a whole lineage.
458/// Moving the `branches` row is what makes a hot fold that omits the lineage
459/// **correct rather than silently short** — every read and write naming the
460/// name now raises [`crate::DbError::UnknownBranch`], which is a refusal, not a
461/// wrong answer.
462///
463/// # Why it needed a rung
464///
465/// [`CREATE_CONCEPTS_GUARD_DELETE`]'s reason, measured once already (D-126):
466/// `CREATE TRIGGER IF NOT EXISTS` on an existing name keeps the **old body**,
467/// so re-issuing the baseline against a v12 database leaves the unconditional
468/// guard exactly where it is and `archive_branch` fails on every ledger that
469/// was not created by this build. The v12 → v13 rung drops and recreates, and
470/// `verify` now carries this name in `DELETE_GUARDS`, so a database whose
471/// guard predates the change is refused at open with a sentence rather than at
472/// archive time with a trigger abort.
473///
474/// The update guard is deliberately **not** gated — see
475/// [`CREATE_BRANCHES_GUARD_UPDATE`]. Archival is a move; there is still no
476/// session in which editing a lineage's parent or fork point is legal, and
477/// gating both would have suspended a rule the operation does not need
478/// suspended.
479pub const CREATE_BRANCHES_GUARD_DELETE: &str = concat!(
480 r#"
481 CREATE TRIGGER IF NOT EXISTS trg_branches_frozen_delete
482 BEFORE DELETE ON branches
483 WHEN NOT EXISTS (
484 SELECT 1 FROM sqlite_master
485 WHERE type = 'table' AND name = '"#,
486 "macrame_archive_session",
487 r#"'
488 )
489 BEGIN
490 SELECT RAISE(ABORT, '"#,
491 abort_branches_frozen!(),
492 r#"');
493 END;
494 "#
495);
496
497/// A branch inherits concepts; it does not restate them (§15.2, D-214).
498///
499/// `concepts` is a current-state projection keyed by identity — `id` is
500/// `NOT NULL UNIQUE` and the write path uses `ON CONFLICT(id) DO UPDATE` — so
501/// two lineages holding different beliefs about one concept is two rows with
502/// one `id`, which the unique index refuses on its own (probe §2). What it
503/// refuses it refuses as a *constraint failure*, naming nothing; this guard
504/// turns the same refusal into a sentence that says which rule was broken.
505///
506/// It fires **before** `ON CONFLICT` is considered, which is not obvious and
507/// was measured rather than assumed (probe §7): a cross-lineage upsert is
508/// refused, a same-lineage one is accepted, and a new id is accepted.
509///
510/// Exact-branch equality, not ancestry. A branch that may restate its parent's
511/// concepts is the overlay design, and the overlay is deferred with its reopen
512/// trigger named (D-214) — a guard that quietly permitted the ancestry case
513/// would ship half of it with none of the machinery that makes it correct.
514pub const CREATE_CONCEPTS_GUARD_LINEAGE: &str = concat!(
515 r#"
516 CREATE TRIGGER IF NOT EXISTS trg_concepts_cross_lineage
517 BEFORE INSERT ON concepts
518 WHEN EXISTS (
519 SELECT 1 FROM concepts
520 WHERE id = NEW.id AND branch_id <> NEW.branch_id
521 )
522 BEGIN
523 SELECT RAISE(ABORT, '"#,
524 abort_cross_lineage!(),
525 r#"');
526 END;
527 "#
528);
529
530/// `branch_id` records where a row was minted, and minting happened once.
531///
532/// The column is **provenance, not identity** (D-214), and the distinction is
533/// exactly what this guard keeps true. An `UPDATE` that moved a concept between
534/// lineages would rewrite where a belief came from without asserting anything
535/// new — the same shape as editing `branches.parent_id`, and forbidden for the
536/// same reason.
537pub const CREATE_CONCEPTS_GUARD_BRANCH: &str = concat!(
538 r#"
539 CREATE TRIGGER IF NOT EXISTS trg_concepts_branch_immutable
540 BEFORE UPDATE ON concepts
541 WHEN NEW.branch_id <> OLD.branch_id
542 BEGIN
543 SELECT RAISE(ABORT, '"#,
544 abort_branch_immutable!(),
545 r#"');
546 END;
547 "#
548);
549
550pub const CREATE_CONCEPTS_TABLE: &str = concat!(
551 r#"
552CREATE TABLE IF NOT EXISTS concepts (
553 rowid_pk INTEGER PRIMARY KEY,
554 id TEXT NOT NULL UNIQUE,
555 title TEXT NOT NULL,
556 content TEXT NOT NULL DEFAULT '',
557 embedding_model TEXT,
558 valid_from TEXT NOT NULL,
559 valid_to TEXT NOT NULL DEFAULT '9999-12-31T23:59:59.999999Z',
560 recorded_at TEXT NOT NULL,
561 retired INTEGER NOT NULL DEFAULT 0,
562 "#,
563 branch_column!(),
564 r#",
565 "#,
566 canonical_ts_check!("valid_from", "valid_to", "recorded_at"),
567 r#"
568);
569"#
570);
571
572/// The ledger. Append-only, one row per assertion, and **keyed by lineage
573/// since v15** (0.14.15, §15.4, [D-232]).
574///
575/// # `branch_id` is in the key, and the release it took to get there
576///
577/// v12 put `branch_id` in the key of `links_current` and left it out of this
578/// one, on an argument the v12 rung records: "`links` accepts both rows because
579/// `recorded_at` is already in its key". That is true of two assertions made at
580/// two instants, which is what a probe testing it by hand produces — and it is
581/// the reason the gap read as latent for seven releases.
582///
583/// It was not latent. **The batch write paths take one stamp for the whole
584/// batch** ([D-014]), and `reject_overlaps_within` groups candidates by
585/// `(source, target, edge_type, branch_id)` — so a pair differing *only* in
586/// lineage is not an overlap, is passed straight through, and collides on the
587/// key with a bare `UNIQUE constraint failed: links.…`. Both batch surfaces
588/// reach it, and `examples/links_key_reach_probe.rs` is the reproduction.
589///
590/// # Why `branch_id` goes last
591///
592/// The same reason the v12 rung gives for `links_current`, and it is stronger
593/// here because this table's autoindex is the *only* index over four of its
594/// columns: the leading `(source_id, target_id, edge_type, valid_from,
595/// recorded_at)` prefix is what `temporal::archive`'s predicates and
596/// `integrity::shadow` seek on, and appending leaves every one of those plans
597/// untouched. A branch-leading key would have re-planned the archive sweep to
598/// buy nothing — the probe's §5 has the plans.
599///
600/// # What this does not change
601///
602/// Not Doctrine III, and not what a row means. The key admits a row the old key
603/// refused; it removes none, alters none, and merges none. Every database that
604/// climbed the v14 → v15 rung holds exactly the rows it held before, which is
605/// what makes the rung a copy rather than a decision.
606///
607/// [D-014]: ../../docs/architecture/s13-decision-register.md#d-014
608/// [D-232]: ../../docs/architecture/s13-decision-register.md#d-232
609pub const CREATE_LINKS_TABLE: &str = concat!(
610 r#"
611CREATE TABLE IF NOT EXISTS links (
612 source_id TEXT NOT NULL REFERENCES concepts(id),
613 target_id TEXT NOT NULL REFERENCES concepts(id),
614 edge_type TEXT NOT NULL,
615 valid_from TEXT NOT NULL,
616 recorded_at TEXT NOT NULL,
617 valid_to TEXT NOT NULL DEFAULT '9999-12-31T23:59:59.999999Z',
618 weight REAL NOT NULL DEFAULT 1.0,
619 properties TEXT NOT NULL DEFAULT '{}',
620 "#,
621 branch_column!(),
622 r#",
623 PRIMARY KEY (source_id, target_id, edge_type, valid_from, recorded_at, branch_id),
624 "#,
625 weight_check!(),
626 r#",
627 "#,
628 canonical_ts_check!("valid_from", "valid_to", "recorded_at"),
629 r#"
630);
631"#
632);
633
634pub const CREATE_LINKS_CURRENT_TABLE: &str = concat!(
635 r#"
636CREATE TABLE IF NOT EXISTS links_current (
637 source_id TEXT NOT NULL,
638 target_id TEXT NOT NULL,
639 edge_type TEXT NOT NULL,
640 valid_from TEXT NOT NULL,
641 valid_to TEXT NOT NULL,
642 weight REAL NOT NULL,
643 properties TEXT NOT NULL,
644 recorded_at TEXT NOT NULL,
645 "#,
646 branch_column!(),
647 r#",
648 PRIMARY KEY (source_id, target_id, edge_type, valid_from, branch_id),
649 "#,
650 canonical_ts_check!("valid_from", "valid_to", "recorded_at"),
651 r#"
652);
653"#
654);
655
656pub const CREATE_TRANSACTION_LOG_TABLE: &str = concat!(
657 r#"
658CREATE TABLE IF NOT EXISTS transaction_log (
659 seq_id INTEGER PRIMARY KEY AUTOINCREMENT,
660 table_name TEXT NOT NULL,
661 entity_id TEXT NOT NULL,
662 operation TEXT NOT NULL,
663 payload TEXT NOT NULL,
664 recorded_at TEXT NOT NULL,
665 "#,
666 branch_column!(),
667 r#",
668 "#,
669 canonical_ts_check!("recorded_at"),
670 r#"
671);
672"#
673);
674
675/// The per-model embedding table (§4.1, D-005), for a validated model name.
676///
677/// A function rather than a `const` because the table's identity *and its
678/// column type* both depend on the model: `F32_BLOB(dim)` carries the declared
679/// dimension in the schema, which is what [`crate::vector::declared_dimension`]
680/// reads back so the crate never keeps a second copy of it.
681///
682/// Deliberately not part of the baseline migration. Which models exist is an
683/// application's choice made over time, not a property of the schema version,
684/// and D-036 classifies these tables as disposable periphery: a migration may
685/// drop one and re-embed. `IF NOT EXISTS` makes registration idempotent.
686///
687/// No temporal columns, on purpose. Doctrine VII makes an embedding a derived
688/// artifact of a model applied to content — it has no valid time of its own, and
689/// giving it a `recorded_at` would put a third clock next to the two §2 permits
690/// and invite queries that mix them.
691pub fn create_embeddings_table(model: &crate::vector::ModelName, dim: usize) -> String {
692 format!(
693 "CREATE TABLE IF NOT EXISTS {table} (
694 concept_id TEXT PRIMARY KEY REFERENCES concepts(id),
695 embedding F32_BLOB({dim}) NOT NULL
696);",
697 table = model.table(),
698 )
699}
700
701/// The DiskANN index over a model's vectors.
702///
703/// **Load-bearing for correctness, not only for speed.** Measured against
704/// libSQL 0.9.30: a blob of the wrong length inserted into an `F32_BLOB(4)`
705/// column is *accepted* while no vector index exists, and rejected — with the
706/// row not landing — once one does. §4.1 previously claimed the column type
707/// enforced its own dimension at insert time; it does not. So this index is
708/// created together with the table it indexes and is never optional, and
709/// dropping it to speed up a bulk load would silently disarm the only
710/// storage-layer check on dimension.
711pub fn create_embeddings_index(model: &crate::vector::ModelName) -> String {
712 format!(
713 "CREATE INDEX IF NOT EXISTS {index} ON {table} (libsql_vector_idx(embedding));",
714 index = model.index(),
715 table = model.table(),
716 )
717}
718
719/// Derived analytics output, keyed by concept and label (§5.4, D-041).
720///
721/// Deliberately outside the ledger. Three properties are load-bearing and each
722/// is the opposite of what the four normative tables above do.
723///
724/// **No log trigger.** Nothing in [`CREATE_TRIGGERS`] fires on this table, so an
725/// annotation never reaches `transaction_log`. That is Doctrine VII's reasoning
726/// about embeddings applied to the other derived artifact: a community label is
727/// a function of an algorithm, a version of that algorithm, and a graph — not a
728/// statement about the world, and a ledger that records it is recording the
729/// analytics schedule as though it were history. A reconstruction that wants
730/// labels recomputes them, which is the only honest way to ask what a past
731/// graph's communities *were*.
732///
733/// **No delete guard.** Doctrine V protects the hot ledger tables; this table is
734/// derivative state in Doctrine VI's second category, so wiping it must stay a
735/// legal, ordinary operation — a rerun replaces the previous pass, and dropping
736/// the whole table costs nothing but the recomputation.
737///
738/// **Upsert on `(concept_id, label)`.** One current value per label per concept.
739/// Storing a history of successive runs here would be the ledger again, by
740/// another name.
741///
742/// The foreign key is safe in a way `links_current`'s omitted ones are not:
743/// concepts are never physically deleted (D-022), and this table is rebuilt by
744/// re-running an algorithm that read `concepts` in the first place, so there is
745/// no insertion-order problem to solve.
746/// One row, one bit: has anything ever been deleted from `transaction_log`?
747/// (v16, 0.15.7, W14.5, [D-249], review C-5.)
748///
749/// # Why this is a table and not a query
750///
751/// The bit is `temporal::replay`'s reach guard, and until v16 the
752/// guard computed it: `MIN(seq_id) = 1 AND COUNT(*) = MAX(seq_id)`, exact
753/// because `seq_id` is `INTEGER PRIMARY KEY AUTOINCREMENT` and never reused.
754/// The `MIN` and `MAX` are index seeks; the `COUNT(*)` is a scan of the whole
755/// log, and it ran on every recorded-time read below the newest surviving
756/// stamp. Measured (`examples/log_integrity_probe.rs`): 0.134 ms at 2,000 rows
757/// and **32.6 ms at 500,000**, against an id-bounded hydration that is flat at
758/// 0.14 ms whatever the log holds. Reading this row instead is 0.033 ms at
759/// every size.
760///
761/// There is no cheaper exact query. `LOG_ARCHIVABLE` removes superseded rows
762/// wherever they sit, so a gap can be anywhere in the sequence and only
763/// counting finds it. What there is instead is a fact the storage already
764/// knows at the moment it becomes true, and did not write down.
765///
766/// # Why a trigger and not the archive code
767///
768/// [`CREATE_TXLOG_MARK_GAP`] maintains it, so the bit is a property of the
769/// **table** rather than of the crate's archive path. §4.2 admits that raw SQL
770/// against the same file can do what this API refuses; a bit maintained in Rust
771/// would be wrong after exactly that, and wrong in the direction that folds a
772/// gap silently. A trigger is wrong in neither direction, because there is no
773/// route to deleting a log row that does not pass through it.
774///
775/// # The seed is computed, not assumed
776///
777/// A database arriving at v16 may already have gaps, so
778/// [`SEED_LOG_INTEGRITY`] derives the initial value from the log's own
779/// `sqlite_sequence` high-water mark. That test is exact where the guard's old
780/// `COUNT(*) = MAX(seq_id)` was exact **and in one state where it was not** —
781/// a hot log archived down to nothing, which the old form called intact. See
782/// `the_bit_agrees_with_the_count_it_replaced`.
783///
784/// [D-249]: ../../docs/architecture/s13-decision-register.md#d-249
785pub const CREATE_LOG_INTEGRITY_TABLE: &str = r#"
786 CREATE TABLE IF NOT EXISTS log_integrity (
787 id INTEGER PRIMARY KEY CHECK (id = 1),
788 rows_removed INTEGER NOT NULL DEFAULT 0 CHECK (rows_removed IN (0, 1))
789 )
790"#;
791
792/// Compute [`CREATE_LOG_INTEGRITY_TABLE`]'s bit from the log itself (v16, [D-249]).
793///
794/// One statement, run by `baseline` and by the v15 -> v16 rung alike, because a
795/// rule stated twice is a rule that can disagree with itself ([D-035]) — and
796/// these two would have: a baseline log is empty, an upgraded one may have been
797/// archived for years, and "empty" is exactly where the obvious test is wrong.
798///
799/// # The witness is `sqlite_sequence`, not `MAX(seq_id)`
800///
801/// `transaction_log.seq_id` is `INTEGER PRIMARY KEY AUTOINCREMENT`, so SQLite
802/// keeps the high-water mark of every id it has ever allocated in
803/// `sqlite_sequence`, and **deleting rows does not lower it**. A rolled-back
804/// transaction rolls the counter back with it ([D-049]), so the mark is exactly
805/// the number of rows the log has ever held. Therefore `COUNT(*) = seq` holds
806/// if and only if nothing has left, whatever the shape of what left: interior
807/// gaps, a raised floor, or every row at once.
808///
809/// That last one is why this is not the test `temporal::replay` used
810/// before v16. `MIN(seq_id) = 1 AND COUNT(*) = MAX(seq_id)` is exact on a
811/// non-empty log and says *intact* on an empty one, which is right for a
812/// database that has never been written and wrong for one that has been fully
813/// archived — the two states it cannot see apart. `sqlite_sequence` sees them
814/// apart: no row for a young log, a positive mark for an emptied one.
815///
816/// `OR REPLACE` because the ladder re-runs rungs over a stamped-back database
817/// and this one has to be idempotent. It is: the value is a function of the
818/// log, not of what is already in the row.
819///
820/// [D-035]: ../../docs/architecture/s13-decision-register.md#d-035
821/// [D-049]: ../../docs/architecture/s13-decision-register.md#d-049
822/// [D-249]: ../../docs/architecture/s13-decision-register.md#d-249
823pub const SEED_LOG_INTEGRITY: &str = r#"
824 INSERT OR REPLACE INTO log_integrity (id, rows_removed)
825 SELECT 1, CASE
826 WHEN (SELECT COUNT(*) FROM transaction_log)
827 = COALESCE(
828 (SELECT seq FROM sqlite_sequence WHERE name = 'transaction_log'),
829 0)
830 THEN 0
831 ELSE 1
832 END
833"#;
834
835/// Set the bit when a log row is physically deleted (v16, [D-249]).
836///
837/// `AFTER DELETE`, so it fires only on a delete that happened —
838/// `trg_txlog_guard_delete`'s `BEFORE DELETE` ([`CREATE_TRIGGERS`]) aborts
839/// first when there is
840/// no archive session, and an aborted delete must not mark the log.
841///
842/// It is `FOR EACH ROW` and it writes the same value every time, which looks
843/// wasteful and is the cheapest correct shape available: SQLite has no
844/// statement-level triggers, and a `WHEN` clause reading `log_integrity` to
845/// skip the write would cost a lookup per row to save a one-page update per
846/// row. Measured on a 333,000-row archive session: 2,520 ms without the
847/// trigger, 2,663 ms with it — **0.43 us per row deleted, 5.6% of a delete
848/// that was already the expensive half of archiving**. The read it pays for
849/// runs on every recorded-time read; this runs once per archive.
850pub const CREATE_TXLOG_MARK_GAP: &str = r#"
851 CREATE TRIGGER IF NOT EXISTS trg_txlog_mark_gap
852 AFTER DELETE ON transaction_log
853 BEGIN
854 UPDATE log_integrity SET rows_removed = 1 WHERE id = 1;
855 END;
856"#;
857
858pub const CREATE_ANALYTICS_ANNOTATIONS_TABLE: &str = concat!(
859 r#"
860CREATE TABLE IF NOT EXISTS analytics_annotations (
861 concept_id TEXT NOT NULL REFERENCES concepts(id),
862 label TEXT NOT NULL,
863 value TEXT NOT NULL,
864 computed_at TEXT NOT NULL,
865 PRIMARY KEY (concept_id, label),
866 "#,
867 canonical_ts_check!("computed_at"),
868 r#"
869);
870"#
871);
872
873/// The keyword half of hybrid search: an FTS5 index over concept text (§5.9).
874///
875/// **External content.** The table declares `content='concepts'`, so the tokens
876/// are indexed but the text itself is not duplicated — FTS5 reads it back from
877/// `concepts` by rowid when it needs a column value. Two reasons beyond the
878/// storage saving, and the second is the one that decided it:
879///
880/// * There is exactly one copy of the text, so the index cannot disagree with
881/// the concept about what the concept says. A standalone FTS table would be a
882/// second description of data the ledger already holds, which is the failure
883/// class D-030 and D-035 exist to prevent.
884/// * `INSERT INTO concepts_fts(concepts_fts) VALUES('rebuild')` reconstructs the
885/// whole index from the content table in one statement. D-036 requires every
886/// derivative table to be rebuildable from the ledger, and here that is the
887/// engine's own operation rather than code of ours that has to be kept honest.
888///
889/// The cost is that external-content tables do not maintain themselves: an
890/// `UPDATE` must retract the *old* terms before adding the new ones, using the
891/// old column values. That is what `trg_concepts_fts_update` does, and getting
892/// it wrong leaves an index that still matches text no concept contains.
893///
894/// **`content_rowid` names `rowid_pk`, not `rowid` (v8, D-119).** They are the
895/// same value — an `INTEGER PRIMARY KEY` *is* the rowid — but naming the column
896/// is what makes the key a declared one rather than an implicit one `VACUUM` is
897/// free to renumber. See [`CREATE_CONCEPTS_TABLE`].
898pub const CREATE_CONCEPTS_FTS: &str = r#"
899CREATE VIRTUAL TABLE IF NOT EXISTS concepts_fts USING fts5(
900 title,
901 content,
902 content='concepts',
903 content_rowid='rowid_pk'
904);
905"#;
906
907/// FTS5's own consistency check — **and it cannot see the failure that matters**
908/// (§5.9, D-071).
909///
910/// Kept as a named constant so the finding has somewhere to live, and used by
911/// `an_emptied_fts_index_still_passes_integrity_check`, which is a tripwire
912/// rather than a guarantee.
913///
914/// On this libSQL build (0.9.30), `'integrity-check'` verifies the index's
915/// *internal* consistency and not its agreement with the content table. Measured:
916/// after `'delete-all'` the index answers zero matches where it answered ten, and
917/// both `'integrity-check'` and `'integrity-check', 0` still report success. So a
918/// `verify_fts()` built on this would report a healthy index for an empty one —
919/// which is why there is no `verify_fts()`. See D-071.
920pub const VERIFY_CONCEPTS_FTS: &str =
921 "INSERT INTO concepts_fts (concepts_fts) VALUES ('integrity-check');";
922
923/// Reconstruct the FTS index from `concepts` (§5.9, D-036).
924///
925/// The engine's own operation, so the rebuild path is not a second
926/// implementation of the triggers that could drift from them.
927pub const REBUILD_CONCEPTS_FTS: &str =
928 "INSERT INTO concepts_fts (concepts_fts) VALUES ('rebuild');";
929
930/// Refresh the query planner's statistics (0.12.4, D-149).
931///
932/// # Why this exists at all
933///
934/// Until 0.12.4 nothing in this crate ever ran `ANALYZE`, so `sqlite_stat1` did
935/// not exist in any database Macrame had created and **every plan was costed
936/// against SQLite's built-in defaults**: assume ~1M rows, assume each bound
937/// equality column divides the search by ten. That estimate is *structural* — a
938/// function of how many columns a query binds, not of what the table holds.
939///
940/// Which is a restatement of this schema's own worst recurring defect. From
941/// `tests/index_plan_tests.rs`: *"a covering index captures a query because it
942/// contains the columns, not because it discriminates."* D-042, D-059 and D-064
943/// are three instances of a planner doing the only thing available to it.
944/// [`CREATE_INDICES`] declares two indices that both lead on `source_id`, and
945/// with no statistics the planner separates them by column count alone.
946///
947/// # Bounded by construction
948///
949/// `ANALYZE` is a **write** — it writes `sqlite_stat1` and takes the write lock —
950/// so unbounded on a populated `links_current` it is exactly the kind of
951/// unbudgeted hold `CHUNK_BUDGET` exists to prevent. [`ANALYSIS_LIMIT`], set once
952/// per connection in `configure`, caps the rows examined per index and makes the
953/// cost a function of the index count rather than the table size. That is what
954/// lets this be scheduled as ordinary low-priority work.
955pub const ANALYZE: &str = "ANALYZE;";
956
957/// Re-analyse only what has gone stale (0.12.4, D-149).
958///
959/// SQLite tracks how much each table has changed since its last analysis and
960/// runs `ANALYZE` only where it believes the statistics no longer hold. A no-op
961/// when nothing has moved, which is what makes it safe to call on a schedule
962/// rather than only on demand.
963///
964/// Bounded by [`ANALYSIS_LIMIT`] like everything else on the connection.
965pub const OPTIMIZE: &str = "PRAGMA optimize;";
966
967/// The row cap that makes [`ANALYZE`] budgetable.
968///
969/// 400 is SQLite's own documented recommendation. It buys approximate statistics
970/// in roughly constant time instead of exact statistics in time proportional to
971/// the table — and approximate is emphatically enough here, because the decision
972/// being informed is *which of two indices discriminates*, not a cardinality
973/// estimate anyone reads.
974///
975/// Set on the connection rather than around each call, so it also bounds the
976/// analysis [`OPTIMIZE`] triggers internally. A limit that applied only to the
977/// explicit path would leave the scheduled one unbounded, which is the half that
978/// runs without anybody watching.
979///
980/// # Measured in 0.12.23: it is a constant factor, not a bound (D-166)
981///
982/// D-149 claimed this makes `ANALYZE`'s cost "a function of the index count
983/// rather than the table size". Measured on this schema — `examples/analyze_hold.rs`,
984/// which times the crate's own hold beside the same file analysed with the
985/// pragma off and on:
986///
987/// | edges | crate's hold | limit off | limit 400 |
988/// |---|---|---|---|
989/// | 10,000 | 5.26 ms | 18.4 ms | 6.01 ms |
990/// | 40,000 | 19.1 ms | 78.6 ms | 19.4 ms |
991///
992/// The pragma **is** in force — the crate's hold tracks the capped arm and not
993/// the uncapped one, which is how it is established at all, since the
994/// connection that runs `ANALYZE` is the actor's and no test can reach it. It
995/// is worth 3.1× at 10,000 edges and 4.1× at 40,000.
996///
997/// What it does not do is remove the table from the equation: over that 4×
998/// range the capped time grew 3.2×. So `analyze()` on a 40,000-edge ledger
999/// holds the write lock for ~19 ms, about 6× [`crate::CHUNK_BUDGET`], and
1000/// [`crate::metrics::CommandKind::Analyze`] is **not** budget-exempt — it
1001/// appears in `metrics().budget_violations()` and always had.
1002///
1003/// Since 0.13.24 that kind is `analyze()` alone; `optimize()` reports as
1004/// [`crate::metrics::CommandKind::Optimize`] and is separately, deliberately
1005/// not exempt (W10.5,
1006/// [D-197](../../docs/architecture/s13-decision-register.md#d-197)).
1007pub const ANALYSIS_LIMIT: &str = "PRAGMA analysis_limit = 400";
1008
1009/// Every index the schema declares.
1010///
1011/// # Two entries left in v8, and why the list is now allowed to be short
1012///
1013/// `idx_annotations_label` and `idx_lc_tgt_active` were dropped by the v7 → v8
1014/// rung ([D-089](../../docs/architecture/s13-decision-register.md), completed by
1015/// D-118). Neither had a reader anywhere in the crate — `analytics_annotations`
1016/// is never selected from here at all, and no query seeks on
1017/// `links_current.target_id` as a leading column — so each was an index write
1018/// per insert, forever, buying nothing. One of them was on the crate's hottest
1019/// write path.
1020///
1021/// `tests/index_plan_tests.rs` now requires the unread set to be **empty**,
1022/// which turns "these two are known bad" into "an index with no reader is a red
1023/// test". That is the guarantee this list is kept short by.
1024///
1025/// # `idx_links_target` is not `idx_lc_tgt_active` coming back
1026///
1027/// The two look like the same index and are not, which is worth stating because
1028/// the resemblance is the trap. `idx_lc_tgt_active` was `(target_id, valid_to)`
1029/// on **`links_current`**, the materialized projection, and it was dropped
1030/// because *nothing in the crate seeks on it* — no reader, pure write cost.
1031/// `idx_links_target` is `(target_id)` on **`links`**, the ledger, and it exists
1032/// because `CONCEPTS_ARCHIVABLE` seeks on exactly that column and the plan is
1033/// measured before and after.
1034///
1035/// D-089's rule was never "no index on a target column". It was "an index needs
1036/// a named query that seeks on it", and the registry is what enforces the
1037/// difference rather than this paragraph.
1038/// The two indices on `links_current`, named because a rung has to restore
1039/// them (§15.2, D-214).
1040///
1041/// `links_current` is derivative, so the v11 → v12 rung re-creates it rather
1042/// than altering it — and `DROP TABLE` takes the table's indices with it.
1043/// Neither [`CREATE_LINKS_CURRENT_TABLE`] nor `rebuild_within` puts them back:
1044/// the first declares a table and the second fills one. The open-time schema
1045/// verifier is what noticed, which is the argument for having it.
1046///
1047/// `pub(crate)` rather than `pub`: every other const this module publishes
1048/// describes the schema a caller might want to read, and these two exist
1049/// only so a rung can put back what its own `DROP TABLE` removed. The
1050/// published form of an index is still [`CREATE_INDICES`], which contains
1051/// both of these.
1052///
1053/// Named consts rather than a `CREATE_INDICES` scan for `ON links_current`,
1054/// because a rung should state which indices it owes rather than derive the
1055/// list from a definition that will keep changing after it. If a later release
1056/// adds a third index here, that release's rung adds it — this one is a
1057/// statement about v12 and stays one.
1058pub(crate) const LC_TRAVERSAL_COVER: &str = "CREATE INDEX IF NOT EXISTS \
1059 idx_lc_traversal_cover ON links_current \
1060 (source_id, valid_from, valid_to, weight, edge_type, target_id);";
1061
1062/// See [`LC_TRAVERSAL_COVER`].
1063pub(crate) const LC_OPEN_INTERVAL: &str = "CREATE INDEX IF NOT EXISTS \
1064 idx_lc_open_interval ON links_current \
1065 (source_id, target_id, edge_type, valid_to, valid_from);";
1066
1067/// The lineage read's own index (0.14.14, §15.4, D-231, shipped v13 -> v14).
1068///
1069/// See [`CREATE_INDICES`] for what seeks on it and why it is a **second** index
1070/// rather than a column added to [`LC_TRAVERSAL_COVER`], which is what §15.4
1071/// asked for.
1072pub(crate) const LC_LINEAGE_CUT: &str = "CREATE INDEX IF NOT EXISTS \
1073 idx_lc_lineage_cut ON links_current \
1074 (branch_id, recorded_at, source_id, target_id, edge_type, valid_from, \
1075 valid_to, weight);";
1076
1077/// Every index declared `ON links_current`, for the shadow swap to put back
1078/// (0.15.19, review C-12).
1079///
1080/// # Why this exists next to [`CREATE_INDICES`] rather than being a scan of it
1081///
1082/// `integrity::shadow`'s swap does `DROP TABLE links_current`, which takes the
1083/// table's indexes with it, and has to recreate exactly the set the projection
1084/// has **today** — unlike a migration rung, which owes the set its own version
1085/// declared and is right to name them one at a time. So the swap did the one
1086/// thing available to it and filtered `CREATE_INDICES` on
1087/// `stmt.contains("links_current")`.
1088///
1089/// That test has a false positive waiting: any future index on **`links`**
1090/// whose text happens to mention `links_current` — in a partial-index `WHERE`,
1091/// or in the comment above it, since these are one string each — would be
1092/// recreated against the renamed table inside the swap's transaction. It would
1093/// either fail the swap or leave an index nobody declared.
1094///
1095/// A name is a name. The list is spelled out, and
1096/// `every_links_current_index_is_in_the_swap_list` fails the build if
1097/// `CREATE_INDICES` gains an entry on this table that is not here — which is
1098/// the property the substring test was reaching for and could not state.
1099pub(crate) const LINKS_CURRENT_INDICES: &[&str] =
1100 &[LC_TRAVERSAL_COVER, LC_OPEN_INTERVAL, LC_LINEAGE_CUT];
1101
1102/// Every trigger that names `links_current`, with the name to drop it by
1103/// (0.15.19, review C-12).
1104///
1105/// Paired rather than two lists, because the swap needs both halves and needs
1106/// them to agree: `ALTER TABLE … RENAME` re-resolves every trigger body, so a
1107/// trigger naming `links_current` must be **dropped** before the rename and
1108/// **recreated** after it (see the `integrity::shadow` module header, which
1109/// measured that). A trigger dropped but not recreated leaves the projection
1110/// unmaintained; one recreated but not dropped fails the rename. Deriving the
1111/// name from the DDL by string surgery would be the same substring match one
1112/// layer down, so the pair is written out and
1113/// `every_links_current_trigger_is_in_the_swap_list` checks both directions.
1114pub(crate) const LINKS_CURRENT_TRIGGERS: &[(&str, &str)] = &[
1115 ("trg_links_current_sync", CREATE_LINKS_CURRENT_SYNC),
1116 ("trg_links_single_open", CREATE_LINKS_SINGLE_OPEN),
1117];
1118
1119pub const CREATE_INDICES: &[&str] = &[
1120 // Covering index for the traversal CTE (§5.2, D-042).
1121 //
1122 // Column order is load-bearing and was measured with EXPLAIN QUERY PLAN.
1123 // The seek column is `source_id`; everything after it is there so the
1124 // recursive step never touches the base table. The two range columns come
1125 // next and `edge_type` comes *after* them, because `edge_types` is empty
1126 // unless a caller sets it: with `edge_type` in second position SQLite
1127 // declines the index for the unfiltered traversal — the default one — and
1128 // silently falls back to a non-covering plan.
1129 //
1130 // (source_id, edge_type, valid_from, ...) filtered: COVERING
1131 // unfiltered: NOT covering
1132 // (source_id, valid_from, valid_to, weight, edge_type, target_id)
1133 // both: COVERING
1134 //
1135 // This subsumes the former idx_lc_src_active (source_id, valid_to): same
1136 // prefix column, strictly more payload. Keeping both would pay two index
1137 // writes per assertion on a table that already takes three writes.
1138 LC_TRAVERSAL_COVER,
1139 // The single-open-interval probe's own index (D-059, shipped v5 -> v6).
1140 //
1141 // `trg_links_single_open` runs an `EXISTS` on every edge insert, keyed on
1142 // (source_id, target_id, edge_type, valid_to) with valid_from as an
1143 // inequality. Before this index the planner served that probe from
1144 // `idx_lc_traversal_cover` with only `source_id` bound — it wins as a
1145 // covering index over the primary-key autoindex, which lacks `valid_to` —
1146 // so **every insert scanned its source's entire out-degree**. Measured on a
1147 // fixed 90-row chunk: 4.4 ms into an empty table, 18.4 ms into a
1148 // 2,000-edge hub, 47.7 ms into an 8,000-edge one, and 1.06 s into 90,000.
1149 // Growth in the table, not in the chunk.
1150 //
1151 // With this index the same 90 rows into the 8,000-edge hub take 8.0 ms and
1152 // stay flat. It matters beyond bulk import: the probe is on the insert path,
1153 // so an interactive `assert_edge` against a high-degree node paid the same
1154 // scan, and that is the path CHUNK_BUDGET's 3 ms exists to protect.
1155 //
1156 // Column order follows the trigger's WHERE exactly — the three equalities
1157 // first, then `valid_to` which is compared to the sentinel, then
1158 // `valid_from` which is the `<>` and cannot be a seek column. This does not
1159 // subsume `idx_lc_traversal_cover` and is not subsumed by it: that one leads
1160 // on `source_id` alone for the recursive walk, this one needs all three
1161 // equality columns bound. Both are kept, which is a fourth index write per
1162 // assertion buying a scan's removal from the same operation.
1163 LC_OPEN_INTERVAL,
1164 // The lineage read's base scans (0.14.14, W12.14, §15.4, D-231, shipped
1165 // v13 -> v14).
1166 //
1167 // `graph::lineage::churned_cte` and `links_cut_cte` are the only two
1168 // statements in the crate that read `links_current` **by lineage**, and
1169 // five call sites emit them: the traversal, `query_as_of_edges_on`,
1170 // `load_subgraph_with` and `diff`'s two tagged copies. Both drive from the
1171 // materialised `lineage` set — `JOIN lineage g ON g.branch_id =
1172 // lc.branch_id` — and both then compare `lc.recorded_at` to that lineage's
1173 // cutoff. So the seek is `(branch_id, recorded_at)` and the payload is
1174 // every other column the two arms project.
1175 //
1176 // Without it SQLite builds the index itself, three times per branched read:
1177 // `AUTOMATIC PARTIAL COVERING INDEX (branch_id=?)` twice over
1178 // `links_current` and once over the `links_cut` co-routine. The third is
1179 // not a table and no index can serve it; the first two are, and this is
1180 // them. Measured (`examples/branch_index_rung_probe.rs`, 1,110 edges,
1181 // chain of 10, best of 25):
1182 //
1183 // branched read, no post-fork churn 6.50 -> 5.43 ms 1.20x
1184 // branched read, 10% post-fork churn 16.94 -> 7.45 ms 2.28x
1185 // trunk traversal 1.64 -> 1.64 ms unchanged
1186 // 2,000 assertions 18.3 -> 20.6 ms +12.6%
1187 //
1188 // **Why a second index and not a column on `idx_lc_traversal_cover`,
1189 // against what §15.4 and D-219 both say.** D-219 measured three shapes and
1190 // preferred folding `branch_id` in after the range columns; it measured
1191 // them against `branch_id IN (ancestry)`, which that same probe run proved
1192 // is not a resolution and which 0.14.4 therefore did not ship. Under the
1193 // reader that did ship, the walk joins a CTE and never touches this table,
1194 // so the folded shape is never consulted and buys **nothing** — 6.18 vs
1195 // 6.27 ms, inside the run-to-run spread. And every single-index shape that
1196 // leads on `branch_id` — the one §15.3 proposed included — takes the trunk
1197 // walk off its covering index altogether:
1198 //
1199 // SEARCH l USING INDEX idx_lc_open_interval (source_id=?)
1200 //
1201 // one bound column and not covering, which is what
1202 // `the_shipped_traversal_cte_stays_on_the_covering_index` exists to refuse.
1203 // The two shapes stopped sharing an access path when the reader stopped
1204 // walking `links_current` directly, so they can no longer share an index.
1205 LC_LINEAGE_CUT,
1206 "CREATE INDEX IF NOT EXISTS idx_txlog_time ON transaction_log (recorded_at);",
1207 "CREATE INDEX IF NOT EXISTS idx_txlog_entity ON transaction_log (entity_id);",
1208 // The fold's partition and its order (0.15.12, W15.2, review C-4, shipped
1209 // v16 -> v17).
1210 //
1211 // The four folds in `temporal::replay` are one window function:
1212 //
1213 // ROW_NUMBER() OVER (PARTITION BY table_name, entity_id, branch_id
1214 // ORDER BY seq_id DESC)
1215 //
1216 // and a window function needs its input in partition-then-order sequence.
1217 // Nothing supplied that, so every reconstruction sorted the whole log in a
1218 // temp B-tree: `SEARCH transaction_log USING INDEX idx_txlog_time
1219 // (recorded_at<?)` followed by `USE TEMP B-TREE FOR ORDER BY`. This index
1220 // is that sequence, so the sort disappears from the plan entirely.
1221 //
1222 // Measured (`examples/txlog_fold_index_probe.rs`, 30,000 log rows in 12,000
1223 // partitions across three lineages, best of 7):
1224 //
1225 // the fold's SQL 64.2 -> 46.2 ms 1.39x
1226 // reconstruct() end to end 99.5 -> 72.6 ms 1.37x
1227 // the file +8.2%
1228 // 400 single-edge writes +5.0%
1229 //
1230 // **`DESC` on the last column is most of the effect and is not
1231 // decoration.** The ascending form of exactly these columns supplies the
1232 // partition and then has to re-sort inside each one — `USE TEMP B-TREE FOR
1233 // RIGHT PART OF ORDER BY` — and that residue is worth 1.30x: 60.2 ms
1234 // against this shape's 46.2, and 90.2 against 72.6 end to end. It is not
1235 // *slower* than having no index; it is a plausible-looking two thirds of
1236 // the improvement, chosen by the planner, at the same file and write cost
1237 // as the whole of it. That is the shape an edit could drift into and keep
1238 // green, which is why the pin is the sort's **absence** rather than the
1239 // index's presence.
1240 //
1241 // **This is not the index review C-4 asked for, and C-4's was measured.**
1242 // The finding names `idx_txlog_entity_lineage` on `(entity_id, branch_id)`,
1243 // on the stated grounds that the fold partitions on those two columns. It
1244 // partitions on three: `table_name` leads, because a concept's `entity_id`
1245 // is its unvalidated id and a link's is the synthetic
1246 // `source|target|type|valid_from`, and the two namespaces are not disjoint
1247 // — `temporal::replay`'s own note explains what partitioning on the id
1248 // alone silently drops. An index whose leading column is not the
1249 // partition's first column cannot serve the window, and the probe measured
1250 // that: **it never came out faster than having no index** — 73.1 ms
1251 // against 64.2 on the run tabulated above, 64.6 against 63.6 on a second —
1252 // because it is attractive enough to take the fold off `idx_txlog_time`
1253 // and then cannot supply the order it was taken for.
1254 //
1255 // The durable half of that finding is the plan, not the millisecond. With
1256 // C-4's index the fold reads `SCAN transaction_log | USE TEMP B-TREE FOR
1257 // ORDER BY`: a **full scan** plus the sort, where v16 had a range seek plus
1258 // the sort. The magnitude of the penalty moves with the machine; the loss
1259 // of the seek does not.
1260 //
1261 // **What it deliberately does not carry.** Adding `recorded_at, operation,
1262 // payload` makes it covering and takes the fold to 39.3 ms — another 1.18x
1263 // — for a file **51% larger**, because `payload` is the widest column in
1264 // the ledger and this would store a second copy of all of it. Refused on
1265 // that trade, and recorded here so the next reader does not re-derive it:
1266 // the fold is not the crate's hottest path and the log is not the place to
1267 // double.
1268 //
1269 // The other readers of this table were checked before and after and none of
1270 // them moved: the archive's log predicate keeps `idx_txlog_entity`, and the
1271 // two aggregates keep `idx_txlog_time` as a covering scan. `idx_txlog_time`
1272 // does lose the fold, which was the reader `index_plan_tests` recorded for
1273 // it — its entry there now names the aggregates that remain, because an
1274 // index justified by a query that has left it is D-089 waiting to happen.
1275 "CREATE INDEX IF NOT EXISTS idx_txlog_fold_partition ON transaction_log \
1276 (table_name, entity_id, branch_id, seq_id DESC);",
1277 // The archive cutoff's seek column on the ledger table (0.12.6, W3.1,
1278 // D-151, review §2.1, shipped v10 -> v11).
1279 //
1280 // `links` carried a primary key and nothing else. `LINKS_ARCHIVABLE` opens
1281 // with `recorded_at < :cutoff`, and the primary key leads on `source_id`, so
1282 // there was nothing for that bound to seek on: both the archiving SELECT and
1283 // the archiving DELETE scanned the entire ledger. Measured on the
1284 // populated-and-analysed fixture in `tests/index_plan_tests.rs`:
1285 //
1286 // before SCAN links | CORRELATED SCALAR SUBQUERY 1 | SEARCH newer ...
1287 // after SEARCH links USING INDEX idx_links_recorded_at (recorded_at<?)
1288 //
1289 // The inner supersession probe was never the problem — it binds the whole
1290 // primary-key prefix and always did.
1291 //
1292 // **This index is justified on those two queries and not on the clock
1293 // floor.** Review §2.1 led with `recorded_at_floor`, the `MAX(recorded_at)`
1294 // read on every `open()`, and counted it among the scans this would close.
1295 // It is not one: SQLite already served the bare `MAX()` from the primary
1296 // key's covering index without traversing the table, and after this index it
1297 // does the same thing through a different covering index. The startup cost
1298 // the review predicted did not exist, so the justification rests entirely on
1299 // the archive path — see D-150 for how that was caught, and D-089 for why an
1300 // index bought on a believed benefit is the failure mode being avoided.
1301 "CREATE INDEX IF NOT EXISTS idx_links_recorded_at ON links (recorded_at);",
1302 // The other half of the concept-archival reachability check (0.12.6, W3.2,
1303 // D-151, review §2.2, shipped v10 -> v11).
1304 //
1305 // `CONCEPTS_ARCHIVABLE` asks whether any surviving link mentions a concept
1306 // *in either direction*: `links.source_id = concepts.id OR links.target_id =
1307 // concepts.id`. The primary key serves the left arm. Nothing served the
1308 // right one, and an `OR` is only as seekable as its worst arm, so the whole
1309 // correlated subquery degraded to a scan of `links` **once per candidate
1310 // concept** — O(concepts x links).
1311 //
1312 // before CORRELATED SCALAR SUBQUERY 1
1313 // | SCAN links USING COVERING INDEX sqlite_autoindex_links_1
1314 // after CORRELATED SCALAR SUBQUERY 1 | MULTI-INDEX OR
1315 // | INDEX 1 | SEARCH links USING COVERING INDEX
1316 // sqlite_autoindex_links_1 (source_id=?)
1317 // | INDEX 2 | SEARCH links USING INDEX
1318 // idx_links_target (target_id=?)
1319 //
1320 // `MULTI-INDEX OR` is SQLite deciding to run both arms as seeks and union
1321 // the rowids, which is exactly the plan the index was added to make
1322 // available. Both the SELECT and the DELETE form pick it up.
1323 //
1324 // A single-column index on the ledger's hottest write path needs the
1325 // strongest justification available, and it has one beyond the plan shape:
1326 // *before* this index the planner was building `AUTOMATIC COVERING INDEX
1327 // (target_id=?)` at query time to answer the same question. It had already
1328 // concluded the index was worth having and was paying to construct a
1329 // throwaway copy per statement.
1330 "CREATE INDEX IF NOT EXISTS idx_links_target ON links (target_id);",
1331 // The lineage's own rows, and only the lineage's (0.15.30, W16.6, [D-273],
1332 // shipped v17 -> v18).
1333 //
1334 // Six statements in `archive_branch_session` filter on `branch_id = ?` and
1335 // nothing led with that column, so each of them scanned a trunk-sized
1336 // table: `links`' primary key carries `branch_id` last (D-232) and
1337 // `idx_txlog_fold_partition` carries it third. A twenty-row lineage
1338 // therefore cost what the whole ledger cost, and grew with it — **9.5 ms at
1339 // a 2,000-edge trunk against 22.0 ms at 8,000**, on the same twenty rows,
1340 // which is D-271's falsified expectation restated as a plan.
1341 //
1342 // Measured (`examples/branch_archive_index_probe.rs`, 8,000-edge trunk,
1343 // analysed, best of five, through `Database::archive_branch`):
1344 //
1345 // none (v17) 22.0 ms
1346 // links (branch_id) 18.2
1347 // transaction_log (branch_id) 11.9
1348 // both 6.8
1349 // what shipped (both, partial) 12.0
1350 //
1351 // **Neither table alone is enough.** The log is twice the size of `links`
1352 // and is the larger of the two scans, which is why indexing `links` by
1353 // itself moves so little.
1354 //
1355 // # `WHERE branch_id <> 'main'` is what makes these free
1356 //
1357 // The trunk is never archivable — `refuse_unarchivable_branch` refuses it in
1358 // its first three lines, because every lineage's parent chain ends there —
1359 // so the rows that dominate both tables, and that every ordinary assertion
1360 // adds to, do not belong in an index built for archival. Against the full
1361 // form on the same fixture: **the same plans on both tables, before and
1362 // after `ANALYZE`**; a 200-edge batch at 24.8 ms against the unindexed
1363 // 24.9, where the full form costs 27.3; and **+20 KB on disk against
1364 // +260 KB**. An index that holds eighty rows out of a ledger's millions is
1365 // not a write cost anybody has to argue about.
1366 //
1367 // **Its statistics also do not decay.** `ANALYZE` records the full index as
1368 // `9144 1829` — average rows per key, dragged upward by a trunk that is most
1369 // of the table, and heading for the ratio at which the planner declines it —
1370 // against the partial index's `80 20`, which describes branches and stays
1371 // true however large the trunk grows. A full index here would get *less*
1372 // likely to be used as the problem it solves got worse.
1373 //
1374 // **What the partial form does not close, and it is one thing.** The last
1375 // row above is 12.0 against the full pair's 6.8, and adding *full* indexes
1376 // on top of the shipped partial pair recovers exactly that difference —
1377 // 7.2 ms, and flat in the trunk where 12.0 still grows (8.3 ms at a
1378 // 2,000-edge trunk). What only a full index can serve is the **foreign-key
1379 // child search**: `branch_id` on all four ledger tables is
1380 // `REFERENCES branches(branch_id)`, so `DELETE FROM branches` makes SQLite
1381 // look for children in each of them, and that search is SQLite's own text —
1382 // it carries no predicate, so no partial index can be reached from it, and
1383 // it has no `EXPLAIN QUERY PLAN` output to pin. Buying it costs the 10–15%
1384 // above on every write forever, for an operation run by hand, so it is left
1385 // open and named rather than paid for. Three quarters of the repair for
1386 // nothing; the last quarter priced.
1387 //
1388 // **The price is that the five statements have to restate the invariant.**
1389 // SQLite uses a partial index only where the query's `WHERE` *implies* the
1390 // index's, and `branch_id = ?` against a bound parameter implies nothing —
1391 // so `archive_branch_session` says `AND branch_id <> 'main'` explicitly.
1392 // `the_archive_seeks_the_lineage` in `tests/index_plan_tests.rs` is what
1393 // keeps those two texts agreeing.
1394 //
1395 // **And that price is the strongest thing about this shape.** A partial
1396 // index cannot be chosen by a query that does not carry the predicate, so
1397 // neither of these can be reached by the fold ([D-254]), by the branched
1398 // guard's log arm ([D-272], which had to be nailed down with `CROSS JOIN`
1399 // four days ago), or by anything else that reads these two tables. Adding a
1400 // *full* index on the log would have put a new candidate in front of every
1401 // one of them. This is an index that can only be used on purpose.
1402 //
1403 // **Not `concepts (branch_id)` and not `links_current (branch_id)`.** They
1404 // were measured with these two: together they move 6.8 ms to 6.5. The first
1405 // is the shape the planner declines and is right to — a lineage that mints
1406 // no concepts leaves one distinct key in `sqlite_stat1` and the plan reverts
1407 // to a scan — and the second is [D-089]'s table, the crate's hottest write
1408 // path, for 0.3 ms of an operation that runs by hand.
1409 //
1410 // [D-089]: ../../docs/architecture/s13-decision-register.md#d-089
1411 // [D-254]: ../../docs/architecture/s13-decision-register.md#d-254
1412 // [D-272]: ../../docs/architecture/s13-decision-register.md#d-272
1413 // [D-273]: ../../docs/architecture/s13-decision-register.md#d-273
1414 concat!(
1415 "CREATE INDEX IF NOT EXISTS idx_links_branch ON links (branch_id) \
1416 WHERE branch_id <> '",
1417 main_branch!(),
1418 "';"
1419 ),
1420 concat!(
1421 "CREATE INDEX IF NOT EXISTS idx_txlog_branch ON transaction_log (branch_id) \
1422 WHERE branch_id <> '",
1423 main_branch!(),
1424 "';"
1425 ),
1426];
1427
1428/// Every trigger the schema declares.
1429///
1430/// **`IF NOT EXISTS` means a changed body does not reach an existing file.**
1431/// `migrations::verify` checks trigger *presence by name*, which is deliberate
1432/// (a count refuses healthy databases) but does not and cannot notice that a
1433/// trigger present under the right name carries an older body. A database
1434/// stamped v5 by an earlier build therefore keeps whatever trigger text it was
1435/// created with until a rung drops and recreates it.
1436///
1437/// This is why the payload carries a version. Changing a log trigger's payload
1438/// splits the database population in two — files created after the change write
1439/// the new shape, files created before keep writing the old one — and the only
1440/// thing that makes that survivable is that every reader accepts both. A
1441/// payload change that did *not* bump `v` would be indistinguishable at read
1442/// time from corruption, which is the case `DbError::PayloadVersion` exists for.
1443///
1444/// The v1 → v2 concept payload (defect V) is deliberately left to ride along on
1445/// the next rung that has to move `user_version` anyway rather than claiming one
1446/// of its own: an old file loses `embedding_model` from its temporal reads, which
1447/// is exactly the behaviour it had before, and gains it the moment it is
1448/// migrated. Nothing regresses in the meantime.
1449/// `links_current` maintenance, one row per open belief **per lineage**.
1450///
1451/// A named `const` since v12 for [`CREATE_CONCEPTS_LOG_INSERT`]'s reason: the
1452/// rung has to re-issue this exact body, and a rung with its own copy is a copy
1453/// that drifts. The conflict target matches the table's primary key, which now
1454/// ends in `branch_id` — without that, a branch asserting an edge its parent
1455/// already holds would *overwrite* the parent's row instead of adding its own.
1456pub const CREATE_LINKS_CURRENT_SYNC: &str = r#"
1457 CREATE TRIGGER IF NOT EXISTS trg_links_current_sync
1458 AFTER INSERT ON links
1459 BEGIN
1460 INSERT INTO links_current
1461 (source_id, target_id, edge_type, valid_from, valid_to,
1462 weight, properties, recorded_at, branch_id)
1463 VALUES
1464 (NEW.source_id, NEW.target_id, NEW.edge_type, NEW.valid_from,
1465 NEW.valid_to, NEW.weight, NEW.properties, NEW.recorded_at,
1466 NEW.branch_id)
1467 ON CONFLICT(source_id, target_id, edge_type, valid_from, branch_id) DO UPDATE SET
1468 valid_to = excluded.valid_to,
1469 weight = excluded.weight,
1470 properties = excluded.properties,
1471 recorded_at = excluded.recorded_at
1472 WHERE excluded.recorded_at > links_current.recorded_at;
1473 END;
1474"#;
1475
1476/// One open interval per edge **per lineage** (§4.3, branch-scoped at v12).
1477///
1478/// The `branch_id` clause is row-level and deliberately not ancestry-aware. A
1479/// branch that inherits an open interval from its parent and asserts its own is
1480/// not violating this rule — it is superseding a belief, which is the thing a
1481/// branch is for.
1482///
1483/// # The question this comment parked, answered at 0.14.8 (D-225)
1484///
1485/// *Whether the inherited interval should also close.* It should not, and
1486/// cannot: closing the ancestor's row is the parent corruption Doctrine III
1487/// forbids, and `links` is append-only so no statement in the crate could do
1488/// it. What a branch writes instead is its **own** row at the ancestor's key,
1489/// which the read prefers by `dist` — shadow retirement.
1490///
1491/// The half a trigger genuinely cannot answer went to the Rust layer, where
1492/// the ancestry is reachable: `lineage::overlap_candidates_resolved` refuses an
1493/// assertion whose interval overlaps **what the writing lineage can see**,
1494/// which is the read's definition applied to the write. That is a guard against
1495/// callers going through the actor and not against raw SQL, which is the same
1496/// honest cost `reject_overlapping_interval` has carried since D-060 — a
1497/// trigger able to make it would need a recursive ancestry walk on every
1498/// insert, on the path D-059 exists to keep fast.
1499pub const CREATE_LINKS_SINGLE_OPEN: &str = concat!(
1500 r#"
1501 CREATE TRIGGER IF NOT EXISTS trg_links_single_open
1502 BEFORE INSERT ON links
1503 WHEN NEW.valid_to = '9999-12-31T23:59:59.999999Z'
1504 AND EXISTS (
1505 SELECT 1 FROM links_current
1506 WHERE source_id = NEW.source_id
1507 AND target_id = NEW.target_id
1508 AND edge_type = NEW.edge_type
1509 AND branch_id = NEW.branch_id
1510 AND valid_from <> NEW.valid_from
1511 AND valid_to = '9999-12-31T23:59:59.999999Z'
1512 )
1513 BEGIN
1514 SELECT RAISE(ABORT, '"#,
1515 abort_single_open!(),
1516 r#"');
1517 END;
1518 "#
1519);
1520
1521/// The update half of the concepts log. See [`CREATE_CONCEPTS_LOG_INSERT`].
1522///
1523/// Unconditional where its insert sibling is marker-gated, and the asymmetry is
1524/// deliberate: nothing inside an archive session updates a concept, so gating
1525/// this would suppress nothing.
1526///
1527/// `branch_id` is in the column list since v12 and the omission would have been
1528/// expensive. `concepts` permits a **same-lineage** update — the guards refuse
1529/// cross-lineage inserts and `branch_id` changes, not this — so a branch
1530/// correcting a concept it minted would have logged the change against `'main'`,
1531/// putting a branch's own history in the trunk's fold and leaving the row
1532/// invisible to the abandonment sweep that §15.5's `archive` arm performs.
1533pub const CREATE_CONCEPTS_LOG_UPDATE: &str = r#"
1534 CREATE TRIGGER IF NOT EXISTS trg_concepts_log_update
1535 AFTER UPDATE ON concepts
1536 BEGIN
1537 INSERT INTO transaction_log (table_name, entity_id, operation, payload, recorded_at, branch_id)
1538 VALUES ('concepts', NEW.id, 'U',
1539 json_object('v', 2, 'title', NEW.title, 'content', NEW.content,
1540 'valid_from', NEW.valid_from, 'valid_to', NEW.valid_to,
1541 'retired', NEW.retired,
1542 'embedding_model', NEW.embedding_model),
1543 NEW.recorded_at, NEW.branch_id);
1544 END;
1545"#;
1546
1547/// The links log, and the entry whose `entity_id` is composed rather than copied.
1548///
1549/// `source|target|type|valid_from` identifies an edge assertion and carries **no
1550/// lineage**, which is why `branch_id` had to become a column of its own rather
1551/// than a fifth field in that string. Re-keying `entity_id` was the other
1552/// option and was rejected: it changes what a log entry identifies, so rows
1553/// written before the rung would no longer match rows written after it, and the
1554/// fold would silently split one edge's history in two.
1555///
1556/// With the column present, the four folds in `temporal::replay` — a private
1557/// module, so the name is plain text rather than a link that would not resolve —
1558/// partition by `(table_name, entity_id, branch_id)` and two lineages'
1559/// assertions about one edge stay two beliefs. Without it they collapse to
1560/// whichever has the higher `seq_id` — no error, no drift report, just one
1561/// lineage's belief gone.
1562pub const CREATE_LINKS_LOG_INSERT: &str = r#"
1563 CREATE TRIGGER IF NOT EXISTS trg_links_log_insert
1564 AFTER INSERT ON links
1565 BEGIN
1566 INSERT INTO transaction_log (table_name, entity_id, operation, payload, recorded_at, branch_id)
1567 VALUES ('links',
1568 NEW.source_id || '|' || NEW.target_id || '|' || NEW.edge_type || '|' || NEW.valid_from,
1569 'I',
1570 json_object('v', 1, 'source_id', NEW.source_id, 'target_id', NEW.target_id,
1571 'edge_type', NEW.edge_type, 'valid_from', NEW.valid_from,
1572 'valid_to', NEW.valid_to, 'weight', NEW.weight,
1573 'properties', json(NEW.properties)),
1574 NEW.recorded_at, NEW.branch_id);
1575 END;
1576"#;
1577
1578/// The ledger's delete guard, named since v15 (0.14.15, [D-232]).
1579///
1580/// An anonymous entry in [`CREATE_TRIGGERS`] until a rung needed to put it
1581/// back: the v14 → v15 rung rebuilds `links`, `DROP TABLE` takes its four
1582/// triggers with it, and a rung cannot re-issue a body it has no name for.
1583/// Promoted rather than copied, which is the rule
1584/// [`CREATE_LINKS_CURRENT_SYNC`] states — a rung with its own copy of a trigger
1585/// is a copy that drifts. The v12 rung promoted four bodies for exactly this
1586/// reason; this is the fifth.
1587///
1588/// D-008 (revised): probe `main.sqlite_master` for the archive-session marker.
1589/// SQLite forbids a trigger in `main` from referencing objects in another
1590/// database, temp included, so the original `temp.sqlite_master` probe fails at
1591/// `CREATE TRIGGER` time and is unimplementable.
1592pub const CREATE_LINKS_GUARD_DELETE: &str = concat!(
1593 r#"
1594 CREATE TRIGGER IF NOT EXISTS trg_links_guard_delete
1595 BEFORE DELETE ON links
1596 WHEN NOT EXISTS (
1597 SELECT 1 FROM sqlite_master
1598 WHERE type = 'table' AND name = 'macrame_archive_session'
1599 )
1600 BEGIN
1601 SELECT RAISE(ABORT, '"#,
1602 abort_delete_guard!(),
1603 r#"');
1604 END;
1605 "#
1606);
1607
1608pub const CREATE_TRIGGERS: &[&str] = &[
1609 CREATE_LINKS_CURRENT_SYNC,
1610 CREATE_LINKS_SINGLE_OPEN,
1611 concat!(
1612 r#"
1613 CREATE TRIGGER IF NOT EXISTS trg_concepts_monotonic_ra
1614 BEFORE UPDATE ON concepts
1615 WHEN NEW.recorded_at <= OLD.recorded_at
1616 BEGIN
1617 SELECT RAISE(ABORT, '"#,
1618 abort_monotonic_ra!(),
1619 r#"');
1620 END;
1621 "#
1622 ),
1623 // Payload v2 adds `embedding_model` (defect V). Before it, the field was
1624 // written by nobody and read by two — `replay::fold_delta` and
1625 // `as_of::hydrate_attributes` both asked the payload for it and both always
1626 // saw null, so `AttributeMode::AtTime`, the faithful mode Doctrine VIII
1627 // exists to offer, returned a *less* complete record than `Current`.
1628 //
1629 // The version number moves because the shape is a compat surface: readers
1630 // must be able to tell "this build wrote no model" from "this payload
1631 // predates the field". v1 is still accepted and folds with the field absent,
1632 // which is what makes this safe without a migration rung — see the note on
1633 // [`CREATE_TRIGGERS`].
1634 CREATE_CONCEPTS_LOG_INSERT,
1635 CREATE_CONCEPTS_LOG_UPDATE,
1636 CREATE_LINKS_LOG_INSERT,
1637 CREATE_CONCEPTS_GUARD_DELETE,
1638 // v12 (§15.2, D-214). Order matters only in that every one of these names a
1639 // table the baseline has already created; `verify` recovers the names from
1640 // this array, so a trigger added here is a trigger the ladder must produce.
1641 CREATE_CONCEPTS_GUARD_LINEAGE,
1642 CREATE_CONCEPTS_GUARD_BRANCH,
1643 CREATE_BRANCHES_GUARD_UPDATE,
1644 CREATE_BRANCHES_GUARD_DELETE,
1645 CREATE_LINKS_GUARD_DELETE,
1646 // v16 (W14.5, D-249). After the guard below in effect as well as in this
1647 // list: the guard is BEFORE DELETE and aborts, so a refused delete never
1648 // reaches this one.
1649 CREATE_TXLOG_MARK_GAP,
1650 concat!(
1651 r#"
1652 CREATE TRIGGER IF NOT EXISTS trg_txlog_guard_delete
1653 BEFORE DELETE ON transaction_log
1654 WHEN NOT EXISTS (
1655 SELECT 1 FROM sqlite_master
1656 WHERE type = 'table' AND name = 'macrame_archive_session'
1657 )
1658 BEGIN
1659 SELECT RAISE(ABORT, '"#,
1660 abort_delete_guard!(),
1661 r#"');
1662 END;
1663 "#
1664 ),
1665 // --- FTS sync (§5.9) ------------------------------------------------
1666 //
1667 // These write to `concepts_fts` and to nothing else. In particular they do
1668 // not touch `transaction_log`: an FTS index is derived from concept text
1669 // the ledger already records, so logging it would record the same fact
1670 // twice — the reasoning Doctrine VII applies to embeddings, and the reason
1671 // `doctrine_static_tests` scans this array.
1672 r#"
1673 CREATE TRIGGER IF NOT EXISTS trg_concepts_fts_insert
1674 AFTER INSERT ON concepts
1675 BEGIN
1676 INSERT INTO concepts_fts (rowid, title, content)
1677 VALUES (NEW.rowid_pk, NEW.title, NEW.content);
1678 END;
1679 "#,
1680 // The retraction is not optional and not symmetric with the insert. An
1681 // external-content FTS5 index stores terms, not text, so replacing a row
1682 // means telling it which terms to *remove* — and it needs the old column
1683 // values to work that out. Omit this and the index keeps matching words the
1684 // concept no longer contains, with no error and no way to notice except by
1685 // searching for something that is no longer there.
1686 r#"
1687 CREATE TRIGGER IF NOT EXISTS trg_concepts_fts_update
1688 AFTER UPDATE ON concepts
1689 BEGIN
1690 INSERT INTO concepts_fts (concepts_fts, rowid, title, content)
1691 VALUES ('delete', OLD.rowid_pk, OLD.title, OLD.content);
1692 INSERT INTO concepts_fts (rowid, title, content)
1693 VALUES (NEW.rowid_pk, NEW.title, NEW.content);
1694 END;
1695 "#,
1696 // The third trigger, installed **inert** by v8 (§4.6, D-119).
1697 //
1698 // Through v7 this array had no delete trigger, and the stated reason was
1699 // that `trg_concepts_guard_delete` is unconditional (D-022) so no delete
1700 // path exists to keep in sync. That was true and it was the wrong shape:
1701 // the index's correctness depended on a *different* trigger staying
1702 // unconditional, and nothing connected the two except a comment.
1703 //
1704 // It cannot fire today — the guard is a `BEFORE DELETE` that always aborts,
1705 // so the statement never reaches `AFTER DELETE`. It is here because 0.9.0's
1706 // archive session is what makes the guard conditional, and the moment that
1707 // lands the index would go silently stale without this. Installing the
1708 // capability in the rung that is already rebuilding the table costs nothing.
1709 //
1710 // It does **not** mean 0.9.0 needs no migration of its own — that claim was
1711 // written here and it is wrong (D-126, corrected 0.8.0 pre-tag). This trigger
1712 // is C2's step 3; step 2 is making `trg_concepts_guard_delete` conditional,
1713 // and that is a `v8 → v9` rung, because `CREATE TRIGGER IF NOT EXISTS` on an
1714 // existing name keeps the **old body** and `verify()` compares names only, so
1715 // a re-issued baseline would leave the unconditional guard in place and pass.
1716 // Deliberately not fixed here: the archive-session marker exists during
1717 // *links* archival too, so a conditional concepts guard shipped in 0.8.0
1718 // would leave concepts deletable during those sessions.
1719 //
1720 // `the_fts_delete_trigger_is_installed_and_inert` (wave1_regression_tests)
1721 // pins both halves rather than assuming either.
1722 r#"
1723 CREATE TRIGGER IF NOT EXISTS trg_concepts_fts_delete
1724 AFTER DELETE ON concepts
1725 BEGIN
1726 INSERT INTO concepts_fts (concepts_fts, rowid, title, content)
1727 VALUES ('delete', OLD.rowid_pk, OLD.title, OLD.content);
1728 END;
1729 "#,
1730];
1731
1732#[cfg(test)]
1733mod tests {
1734 use crate::util::timestamp::{CANONICAL_TS_GLOB, OPEN_SENTINEL};
1735
1736 /// The DDL's CHECK pattern and the Rust-side pattern must be the same
1737 /// pattern. If they drift, one layer accepts what the other rejects and the
1738 /// canonical-form invariant is enforced in name only.
1739 #[test]
1740 fn ddl_glob_matches_the_rust_canonical_pattern() {
1741 assert_eq!(format!("'{}'", ts_glob!()), CANONICAL_TS_GLOB);
1742 }
1743
1744 /// Every DDL statement that declares a temporal default must use the
1745 /// canonical sentinel; a second-precision default would be rejected by the
1746 /// very CHECK sitting next to it.
1747 #[test]
1748 fn ddl_defaults_use_the_canonical_sentinel() {
1749 for ddl in [
1750 super::CREATE_CONCEPTS_TABLE,
1751 super::CREATE_LINKS_TABLE,
1752 super::CREATE_LINKS_CURRENT_TABLE,
1753 super::CREATE_TRANSACTION_LOG_TABLE,
1754 ] {
1755 assert!(
1756 !ddl.contains("9999-12-31T23:59:59Z"),
1757 "DDL still carries the pre-0.5.4 second-precision sentinel: {ddl}"
1758 );
1759 }
1760 for trigger in super::CREATE_TRIGGERS {
1761 assert!(
1762 !trigger.contains("9999-12-31T23:59:59Z"),
1763 "trigger still carries the pre-0.5.4 sentinel: {trigger}"
1764 );
1765 }
1766 assert!(super::CREATE_LINKS_TABLE.contains(OPEN_SENTINEL));
1767 }
1768
1769 /// The swap's index list must be every index on `links_current`.
1770 ///
1771 /// `integrity::shadow` drops the table and puts these back by name. An
1772 /// index added to [`CREATE_INDICES`] on this table and not added here
1773 /// would be dropped by the swap and never recreated — a projection that is
1774 /// still correct and silently unindexed, which is the failure the open-time
1775 /// verifier was written for and which this catches a build earlier.
1776 #[test]
1777 fn every_links_current_index_is_in_the_swap_list() {
1778 for sql in super::CREATE_INDICES {
1779 if sql.contains("ON links_current") {
1780 assert!(
1781 super::LINKS_CURRENT_INDICES.contains(sql),
1782 "an index on links_current is missing from LINKS_CURRENT_INDICES, so the shadow swap would drop it and not put it back: {sql}"
1783 );
1784 }
1785 }
1786 for sql in super::LINKS_CURRENT_INDICES {
1787 assert!(
1788 sql.contains("ON links_current"),
1789 "LINKS_CURRENT_INDICES carries something that is not on that table: {sql}"
1790 );
1791 }
1792 }
1793
1794 /// The swap's trigger list must be every trigger that names
1795 /// `links_current`, and each pair must agree.
1796 ///
1797 /// Both directions matter and they fail differently. A trigger missing from
1798 /// the list is left in place across `ALTER TABLE … RENAME`, which fails the
1799 /// rename outright — loud, but inside a maintenance operation. A pair whose
1800 /// name does not match its body drops one trigger and recreates another,
1801 /// which is silent.
1802 #[test]
1803 fn every_links_current_trigger_is_in_the_swap_list() {
1804 for trigger in super::CREATE_TRIGGERS {
1805 if trigger.contains("links_current") {
1806 assert!(
1807 super::LINKS_CURRENT_TRIGGERS
1808 .iter()
1809 .any(|(_, ddl)| ddl == trigger),
1810 "a trigger naming links_current is missing from LINKS_CURRENT_TRIGGERS, so the shadow swap's rename would fail on it: {trigger}"
1811 );
1812 }
1813 }
1814 for (name, ddl) in super::LINKS_CURRENT_TRIGGERS {
1815 assert!(
1816 ddl.contains(name),
1817 "LINKS_CURRENT_TRIGGERS pairs {name} with a body that does not declare it"
1818 );
1819 assert!(
1820 super::CREATE_TRIGGERS.contains(ddl),
1821 "LINKS_CURRENT_TRIGGERS carries a trigger the schema does not create: {name}"
1822 );
1823 }
1824 }
1825
1826 /// Every abort message the classifier matches on must actually appear in the
1827 /// DDL that emits it. `concat!` makes this true by construction today; the
1828 /// test is what keeps it true if someone re-inlines a literal.
1829 #[test]
1830 fn every_abort_message_appears_in_a_trigger() {
1831 for msg in [
1832 super::ABORT_SINGLE_OPEN,
1833 super::ABORT_MONOTONIC_RA,
1834 super::ABORT_DELETE_GUARD,
1835 ] {
1836 assert!(
1837 super::CREATE_TRIGGERS.iter().any(|t| t.contains(msg)),
1838 "no trigger emits {msg:?}, so its typed error is unreachable"
1839 );
1840 }
1841 }
1842}