Skip to main content

macrame/temporal/
archive.rs

1use std::path::Path;
2
3use libsql::TransactionBehavior;
4
5use crate::error::{DbError, Result, WriteOp};
6use crate::schema::ddl::ARCHIVE_SESSION_MARKER;
7use crate::util::limits::HYDRATE_CHUNK;
8
9/// Outcome of one archive session.
10#[derive(Debug, Clone, PartialEq, Eq)]
11#[non_exhaustive]
12pub struct ArchiveReport {
13    pub links_archived: usize,
14    /// Concepts moved to `cold.concepts` (0.9.0, C2). Always `0` before v9,
15    /// where no concept could leave the hot table at all.
16    pub concepts_archived: usize,
17    pub log_entries_archived: usize,
18    /// Oldest `transaction_log.seq_id` still present in the hot file after the
19    /// session, i.e. the new horizon (see glossary). `None` if the hot log is empty.
20    pub horizon: Option<i64>,
21}
22
23/// Schema of the cold database. Deliberately trigger-free and FK-free.
24///
25/// **Corrected 2026-08-07.** This comment used to justify the FK-free part with
26/// *"concepts are never archived (D-022)"*, which stopped being true in 0.9.0
27/// when C2 added `cold.concepts` — the table declared a few lines below. The
28/// reasons that survive are the other two, and they are the load-bearing ones:
29/// a FK from `cold.links` to `concepts` still could not be satisfied, because
30/// the cold file holds only the concepts that have gone cold and `cold.links`
31/// may name any of them; and the delete guards must not exist on a file whose
32/// whole purpose is to receive rows and, on rehydration, to give them back.
33const COLD_SCHEMA: &[&str] = &[
34    // `weight` carries the same CHECK as the hot table (T2.1, D-083). Not
35    // symmetry for its own sake: the cold file is read back by `reconstruct`
36    // through the same `f64` decode, so a text weight is the same panic there
37    // as it is here, and a negative one is the same unsound shortest path.
38    //
39    // The hot table's constraint does not protect this one. Rows arrive by
40    // `INSERT … SELECT` across an ATTACH, which re-checks against *this*
41    // table's constraints — and a cold file may predate the hot file's rung, or
42    // have been written by a version that had neither.
43    //
44    // `IF NOT EXISTS` means an existing cold database keeps whatever definition
45    // it was created with; this constrains new cold files, and the loader guard
46    // is what covers the old ones. That is the same division of labour §4.7
47    // describes, and the reason the guard stays.
48    //
49    // **`branch_id` is in the key since v15** (0.14.15, D-232), and it had to
50    // move with the hot table rather than after it. The hot key admitted the
51    // pair, so archiving became the one operation that could still refuse it:
52    // two lineages' rows about one edge at one `recorded_at` are legal in
53    // `links` and would have collided on the way out, turning a write the crate
54    // now accepts into a maintenance failure the caller cannot act on.
55    // `upgrade_cold_lineage` carries existing cold files across.
56    r#"CREATE TABLE IF NOT EXISTS cold.links (
57        source_id   TEXT NOT NULL,
58        target_id   TEXT NOT NULL,
59        edge_type   TEXT NOT NULL,
60        valid_from  TEXT NOT NULL,
61        recorded_at TEXT NOT NULL,
62        valid_to    TEXT NOT NULL,
63        weight      REAL NOT NULL CHECK (weight >= 0.0 AND weight < 9e999 AND typeof(weight) = 'real'),
64        properties  TEXT NOT NULL,
65        branch_id   TEXT NOT NULL DEFAULT 'main',
66        PRIMARY KEY (source_id, target_id, edge_type, valid_from, recorded_at, branch_id)
67    )"#,
68    // Concepts, as of v9 (C2). Trigger-free and FK-free like `cold.links`, and
69    // for the same reasons -- but note what it does NOT drop.
70    //
71    // **Every column crosses, `content` included.** Archival is a move, not a
72    // rewrite (2.3), and a move that drops a column is a rewrite. The log
73    // payload for a concept carries its `content` (4.3), so a cold concept
74    // whose text had been dropped would contradict `cold.transaction_log` about
75    // itself, and rehydration would return a concept the ledger never recorded:
76    // empty text where the log says there was text. That is the unexplained
77    // absence Doctrine V exists to prevent.
78    //
79    // The tension with D-116 is apparent rather than real. D-116 governs the
80    // *in-memory* `NodeData` representation -- `content` is not loaded by
81    // default because most readers do not want it. This is *on-disk* storage.
82    // Disk carries the text; memory does not populate it until asked. Two
83    // independent defaults, and conflating them would make rehydration lossy to
84    // save a read nobody was performing.
85    //
86    // `rowid_pk` crosses as the record of what the rowid *was*. Restoring it is
87    // C3's problem and not obviously safe: `concepts.rowid_pk` is a plain
88    // INTEGER PRIMARY KEY, so SQLite may reuse a freed value, and archiving the
89    // highest rowids can leave a later insert holding one a cold row still
90    // claims. The column is carried because a move must not lose it.
91    //
92    // **The hazard has two exits and C3 must take one of them explicitly.**
93    // Either reinstate the original `rowid_pk` when it is still free, or assign
94    // a fresh one — and in the second case **update `concepts_fts`'s
95    // `content_rowid` mapping to match**, because the FTS index is
96    // external-content keyed on this column (4.6, D-119). A rehydration that
97    // reassigns the rowid without re-pointing the index leaves the search index
98    // silently describing the wrong row, which is the exact failure `rowid_pk`
99    // was made explicit to prevent. Named here so C3 meets both exits rather
100    // than rediscovering the FTS coupling.
101    r#"CREATE TABLE IF NOT EXISTS cold.concepts (
102        rowid_pk         INTEGER,
103        id               TEXT NOT NULL PRIMARY KEY,
104        title            TEXT NOT NULL,
105        content          TEXT NOT NULL DEFAULT '',
106        embedding_model  TEXT,
107        valid_from       TEXT NOT NULL,
108        valid_to         TEXT NOT NULL,
109        recorded_at      TEXT NOT NULL,
110        retired          INTEGER NOT NULL DEFAULT 0,
111        branch_id        TEXT NOT NULL DEFAULT 'main'
112    )"#,
113    // seq_id is carried over verbatim from the hot log, so it is a plain
114    // INTEGER PRIMARY KEY -- never AUTOINCREMENT, which would renumber history.
115    r#"CREATE TABLE IF NOT EXISTS cold.transaction_log (
116        seq_id      INTEGER PRIMARY KEY,
117        table_name  TEXT NOT NULL,
118        entity_id   TEXT NOT NULL,
119        operation   TEXT NOT NULL,
120        payload     TEXT NOT NULL,
121        recorded_at TEXT NOT NULL,
122        branch_id   TEXT NOT NULL DEFAULT 'main'
123    )"#,
124    "CREATE INDEX IF NOT EXISTS cold.idx_cold_txlog_entity ON transaction_log (entity_id)",
125    "CREATE INDEX IF NOT EXISTS cold.idx_cold_txlog_time ON transaction_log (recorded_at)",
126    // The fold's partition index is **not** here; see [`COLD_LINEAGE_INDICES`].
127    // It names `branch_id`, this list runs before the column exists, and the
128    // two indexes above name only columns a v11 cold file already had.
129    // Lineages, as of 0.14.13 (§15.4, D-230). The one cold table that is not a
130    // mirror of a hot one: `branches` carries no `archived_at` and this needs
131    // one, because the hot row's `created_at` says when the lineage began and
132    // nothing on it can say when the ledger stopped knowing about it.
133    //
134    // **This is the table `upgrade_cold_lineage` predicted.** Its note says a
135    // cold row records *that* it belonged to a lineage without recording what
136    // that lineage was, and that "the abandonment arm makes forgetting a branch
137    // an ordinary operation, and a cold row stamped with a name nothing
138    // resolves is the shape that falls out of it". This is what resolves the
139    // name: the cold file carries the lineage record itself, so
140    // `cold.links.branch_id` names a row in the same file rather than a string
141    // whose meaning was left behind in the hot database.
142    //
143    // FK-free like the rest of the cold schema, `parent_id` included — the
144    // parent is normally still hot, which is the whole point of the operation,
145    // so a self-referencing key here would refuse every row this table exists
146    // to hold.
147    r#"CREATE TABLE IF NOT EXISTS cold.branches (
148        branch_id   TEXT NOT NULL PRIMARY KEY,
149        parent_id   TEXT,
150        forked_at   TEXT,
151        created_at  TEXT NOT NULL,
152        archived_at TEXT NOT NULL
153    )"#,
154    r#"CREATE TABLE IF NOT EXISTS cold.archive_horizon (
155        archived_at TEXT NOT NULL,
156        cutoff      TEXT NOT NULL,
157        horizon     INTEGER
158    )"#,
159];
160
161/// A links assertion is archivable when it is older than the cutoff AND it is
162/// either superseded by a later assertion **of its own lineage** for the same
163/// interval key, or it is the current belief for an interval that closed before
164/// the cutoff.
165///
166/// This keeps every row that `links_current` still projects (Doctrine VI: the
167/// materialization must stay rebuildable from `links`) while moving exactly the
168/// "closed intervals, superseded history" the §2 diagram assigns to the cold file.
169///
170/// # `newer.branch_id = links.branch_id`, added at 0.14.12 ([D-229])
171///
172/// Without it this predicate archived rows the ledger still believed. `links_current`
173/// is keyed by `(source, target, type, valid_from, branch_id)` and the four folds in
174/// `temporal::replay` partition by `(table_name, entity_id, branch_id)`, but a link's
175/// `entity_id` is `source|target|type|valid_from` and carries **no lineage**
176/// ([`crate::schema::ddl::CREATE_LINKS_LOG_INSERT`] says why re-keying it was
177/// refused). So "a later assertion for the same interval key" matched **across**
178/// lineages, and a branch asserting at an ancestor's key made the ancestor's own
179/// open, current row look superseded.
180///
181/// Measured before the repair, on a two-row fixture: the trunk asserts `a → b`, a
182/// branch forks and asserts at the same key, one `archive` runs, and the **trunk**
183/// stops reaching `b`. `audit_current` reports **0**, which is why nothing caught
184/// it — `links_current` is honestly re-derived from a `links` table that has been
185/// wrongly pruned, so the projection is correct with respect to what survives and
186/// the drift check has nothing to compare against. Doctrine VI's audit answers
187/// "is the projection the image of the ledger", never "is the ledger complete".
188///
189/// **Exact-branch equality, not ancestry**, and for
190/// [`crate::schema::ddl::CREATE_CONCEPTS_GUARD_LINEAGE`]'s reason. A descendant's
191/// row shadows an ancestor's *for the descendant's reads*; the ancestor still
192/// believes its own row, and Doctrine III is precisely that shadowing never
193/// touches it. A predicate that let a descendant supersede an ancestor would
194/// archive the parent's belief because a child disagreed.
195///
196/// [D-229]: ../../docs/architecture/s13-decision-register.md#d-229
197///
198/// # The closed-interval arm, and the row it must not take
199///
200/// "A closed interval is history" is true of a lineage that holds the only row
201/// at its key, and false of a **shadow**. A branch retires an inherited edge by
202/// writing its own closed row at the ancestor's key — the only cross-lineage
203/// retirement [Doctrine III] permits, because it never touches the parent's row.
204/// Archiving that row does not send history cold; it removes the branch's
205/// disbelief and lets the ancestor's open row win the resolution again.
206///
207/// Measured before the repair: a branch retires `b → c` over `[EPOCH, T1)`, one
208/// archive runs, and at `T2` the branch reaches `c` — an edge it had stopped
209/// believing, restored by a maintenance operation that mints no assertions. That
210/// is the resurrection [`crate::schema::ddl::CREATE_CONCEPTS_LOG_INSERT`] gates
211/// the rehydration insert against, reached down the other path.
212///
213/// So the arm stands down whenever **another lineage holds a hot row at the same
214/// interval key**. Conservative rather than exact: what strictly matters is an
215/// *ancestor's* row surviving this session, and both halves of that are more than
216/// this predicate can see. Ancestry would mean resolving `graph::lineage`'s chain
217/// for every branch, in a whole-database operation that takes no branch
218/// parameter; "surviving this session" is self-referential, since what survives
219/// is the answer this predicate is computing. Leaving rows hot costs file size
220/// and is never wrong, so the rule is the one that needs neither. A key held by
221/// exactly one lineage — every key on a ledger that has never forked — is
222/// unaffected, which the tests measure rather than argue from a column default.
223///
224/// ## What being conservative costs, stated (0.15.19, review C-21)
225///
226/// "Never wrong" was the whole of what this said, and the cost has a shape a
227/// deployment is better off knowing before it meets it. The `NOT EXISTS` runs
228/// against `links`, not against `cold`, so a closed row is held hot by **any**
229/// other lineage's hot row at the same interval key — the trunk's own included,
230/// and including rows that have nothing to do with the shadow retirement this
231/// arm exists for.
232///
233/// * On a ledger that has never forked, nothing is held: every key belongs to
234///   one lineage and the sub-select is empty. This is the common case and it
235///   pays nothing at all.
236/// * With live branches, every key **any** of them has written stays hot on
237///   **every** lineage that shares it. The bound is the number of distinct
238///   interval keys live branches have touched — not the size of the ledger, and
239///   not a function of how long the branch has been open.
240/// * It clears itself, with no operator step. [`archive_branch`] takes a
241///   lineage's hot rows cold, which removes them from the table this guard
242///   consults, so the very next ordinary `archive` session finds the
243///   `NOT EXISTS` satisfied and takes the rows that were being held. Nothing
244///   accumulates across that boundary.
245///
246/// The alternative's failure mode is the one measured above — an edge a branch
247/// had stopped believing coming back because a maintenance operation ran — so
248/// the trade is disk against a wrong answer. Long-lived branches writing at
249/// keys the trunk also holds are the only shape where the disk side is visible.
250///
251/// [Doctrine III]: ../../docs/architecture/README.md
252/// [`archive_branch`]: crate::temporal::archive_branch
253const LINKS_ARCHIVABLE: &str = r#"
254    recorded_at < :cutoff AND (
255        EXISTS (
256            SELECT 1 FROM links newer
257            WHERE newer.source_id   = links.source_id
258              AND newer.target_id   = links.target_id
259              AND newer.edge_type   = links.edge_type
260              AND newer.valid_from  = links.valid_from
261              AND newer.branch_id   = links.branch_id
262              AND newer.recorded_at > links.recorded_at
263              AND NOT EXISTS (
264                    SELECT 1 FROM branches b
265                    WHERE b.forked_at IS NOT NULL
266                      AND b.forked_at >= links.recorded_at
267                      AND b.forked_at <  newer.recorded_at
268                  )
269        )
270        OR (valid_to <> '9999-12-31T23:59:59.999999Z' AND valid_to <= :cutoff
271            AND NOT EXISTS (
272                SELECT 1 FROM links other
273                WHERE other.source_id  = links.source_id
274                  AND other.target_id  = links.target_id
275                  AND other.edge_type  = links.edge_type
276                  AND other.valid_from = links.valid_from
277                  AND other.branch_id <> links.branch_id
278            )
279            AND NOT EXISTS (
280                SELECT 1 FROM links older, branches b
281                WHERE older.source_id   = links.source_id
282                  AND older.target_id   = links.target_id
283                  AND older.edge_type   = links.edge_type
284                  AND older.valid_from  = links.valid_from
285                  AND older.branch_id   = links.branch_id
286                  AND older.recorded_at < links.recorded_at
287                  AND b.forked_at IS NOT NULL
288                  AND b.forked_at >= older.recorded_at
289                  AND b.forked_at <  links.recorded_at
290            ))
291    )
292"#;
293
294/// A log entry is archivable when it is older than the cutoff and a later entry
295/// exists **for the same entity on the same lineage**, i.e. it is superseded.
296/// The newest entry per fold partition always stays hot so that
297/// `reconstruct(now)` never needs the cold file.
298///
299/// # The lineage clause, added at 0.14.12 ([D-229])
300///
301/// The sentence above used to say "per entity", and the four folds in
302/// `temporal::replay` have partitioned by `(table_name, entity_id, branch_id)`
303/// since v12 — so the predicate stopped keeping the newest entry per *partition*
304/// hot the moment lineage arrived, and nothing said so. A branch writing at an
305/// ancestor's edge key made the ancestor's newest entry archivable, which is the
306/// same defect [`LINKS_ARCHIVABLE`] carried, reached from the log side.
307///
308/// It changes nothing for `concepts` entries and that is worth stating rather
309/// than leaving to be rediscovered: a concept's `entity_id` is its `id`, and
310/// [`crate::schema::ddl::CREATE_CONCEPTS_GUARD_LINEAGE`] refuses a branch
311/// restating an inherited one, so every log entry for one concept already
312/// carries one lineage. The clause is a no-op there by construction, not by
313/// accident.
314///
315/// # The fork-point clause, added at 0.15.26 ([D-269])
316///
317/// [D-229] made supersession ask **whose** entry is newer. It did not ask
318/// **when**, and a fork point is a `recorded_at` that a descendant is pinned to:
319/// a branch reads its ancestors through `recorded_at <= branches.forked_at`, so
320/// it folds whichever entry was newest *at the moment it forked*, not whichever
321/// is newest now. Supersession on one lineage therefore says nothing about
322/// whether the superseded entry is still in use — a descendant that forked
323/// between the two writes is reading the older one, and it is the only reader
324/// that is.
325///
326/// Found by `lineage_property_tests.rs` on its first run against a clean tree,
327/// and it needs **no cross-lineage write anywhere in the history**: the trunk
328/// restates an edge it already holds, one ordinary `archive` runs, and the fork
329/// stops reaching a node the trunk still reaches. The branch did nothing except
330/// exist across a restatement. That is the consequence [D-229] refused for a
331/// retirement — *"a different, older belief"* — here degraded to no belief at
332/// all, and reached without anybody writing on a branch.
333///
334/// So a superseded entry stays hot while any lineage's fork point falls in
335/// `[entry.recorded_at, newer.recorded_at)`, the half-open interval whose
336/// readers are exactly the branches pinned to `entry`. Conservative in
337/// [D-229]'s sense and for its reasons: it asks about *any* branch rather than
338/// resolving which descend from `transaction_log.branch_id`, because this
339/// operation takes no branch parameter and `graph::lineage` cannot be reused by
340/// one that has none. Holding an entry nobody needs costs file size; releasing
341/// one somebody needs costs an answer. It clears itself exactly as
342/// [`LINKS_ARCHIVABLE`]'s closed-interval arm does: `archive_branch` removes the
343/// row from `branches`, and the next ordinary session takes what was held.
344///
345/// **This clause is the one that closes the finding, and which predicate that
346/// is was measured rather than reasoned.** The obvious guess is that the gap
347/// lives on [`LINKS_ARCHIVABLE`], since that is where [D-229] was. The first
348/// repair was written on that guess and **fixed nothing**: with the guard on
349/// `links` alone the ledger kept every row and the branch still lost the edge,
350/// because a forked read does not resolve through `links_current` and the only
351/// table the session had changed was `transaction_log`. So the order matters —
352/// the log side first, on evidence.
353///
354/// [`LINKS_ARCHIVABLE`] carries the same guard too, and for a different history
355/// rather than for symmetry: a *retirement* on the trunk after a fork reaches
356/// the links side, where a restatement does not. That clause then forced a
357/// third, on the closed-interval arm, because an older open row held hot lets
358/// the arm take the row that closed it and resurrect the retired belief —
359/// [D-229]'s own symptom, reached through this entry's repair. The generator
360/// found that one as well, on the run after the first fix.
361///
362/// [D-269]: ../../docs/architecture/s13-decision-register.md#d-269
363/// [D-229]: ../../docs/architecture/s13-decision-register.md#d-229
364const LOG_ARCHIVABLE: &str = r#"
365    recorded_at < :cutoff AND EXISTS (
366        SELECT 1 FROM transaction_log newer
367        WHERE newer.entity_id = transaction_log.entity_id
368          AND newer.branch_id = transaction_log.branch_id
369          AND newer.seq_id    > transaction_log.seq_id
370          AND NOT EXISTS (
371                SELECT 1 FROM branches b
372                WHERE b.forked_at IS NOT NULL
373                  AND b.forked_at >= transaction_log.recorded_at
374                  AND b.forked_at <  newer.recorded_at
375              )
376    )
377"#;
378
379/// A concept is archivable when it is `retired`, both its clocks are behind the
380/// cutoff, **and no surviving row of hot `links` mentions it in either
381/// direction** (C1, D-128).
382///
383/// # Why reachability, and not a closed interval
384///
385/// A link assertion has a closed interval, so [`LINKS_ARCHIVABLE`] can ask
386/// whether the interval ended. A concept is an *entity*, and has no closed
387/// state: `retired = 1` says belief in it stopped, which is not the same claim
388/// as "nothing points at it any more". The two `links` foreign keys are what
389/// make that difference matter — archiving a concept physically removes its row,
390/// and a surviving hot link naming it would leave the key unsatisfiable.
391/// `ON DELETE CASCADE` is not the way out, because the rows it would cascade
392/// onto are ledger rows.
393///
394/// So concept archival is **strictly downstream of link archival**: a concept
395/// becomes eligible only once every edge mentioning it has itself gone cold.
396/// Inside a session this predicate is therefore evaluated *after* the `links`
397/// delete and never before it, and the same question asked before and after one
398/// session legitimately gives two different answers. That is a property of the
399/// predicate, not a race.
400///
401/// # The other two foreign keys, and why they are not clauses here
402///
403/// `concepts` also has inbound keys from `analytics_annotations` and from every
404/// registered `embeddings_*` table ([`crate::schema::migrations`] lists all
405/// four). Neither appears above, and the distinction is the point: those hold
406/// **derived** rows. Doctrine VII makes an embedding an artifact of a model
407/// applied to content, and an annotation is the output of an algorithm that read
408/// `concepts` in the first place. A derived row is removed and recomputed; a
409/// ledger row is neither. Making archivability wait on a recomputable artifact
410/// would answer "not yet" forever for any concept that had ever been embedded.
411///
412/// # Both clocks, because one of them is not enough
413///
414/// The specification for this predicate named `valid_to` alone.
415/// `recorded_at < :cutoff` is here as well, mirroring [`LINKS_ARCHIVABLE`]:
416/// a concept retired with its `valid_to` behind the cutoff but *recorded* at or
417/// after it is a fact the session is not meant to touch yet, and archiving it
418/// would send the concept cold while the log entries describing it stayed hot.
419/// That is the same two-clock mismatch the `links_current` compensation carried
420/// until Wave 4.5 (see [`archive_session`]), reached from the other side.
421/// Doctrine II: two clocks, never mixed.
422///
423/// The open sentinel needs no clause of its own — `9999-12-31T23:59:59.999999Z`
424/// sorts above every canonical stamp (D-029), so a concept whose validity is
425/// still open fails `valid_to < :cutoff` for any cutoff a caller can pass.
426const CONCEPTS_ARCHIVABLE: &str = r#"
427    retired = 1
428    AND recorded_at < :cutoff
429    AND valid_to    < :cutoff
430    AND NOT EXISTS (
431        SELECT 1 FROM links
432        WHERE links.source_id = concepts.id
433           OR links.target_id = concepts.id
434    )
435"#;
436
437/// The ids of every concept that `CONCEPTS_ARCHIVABLE` admits at `cutoff`, in
438/// `id` order.
439///
440/// **Read-only, and deliberately available before anything can act on it.**
441/// Concept archival is the one operation in this crate a caller cannot undo
442/// without a cold file to hand, so the predicate that decides it is observable
443/// on its own rather than only as a count in a report after the fact.
444///
445/// The answer is a function of the hot state *now*. Archiving links first will
446/// generally enlarge it — that is the downstream relationship
447/// `CONCEPTS_ARCHIVABLE` describes, not an inconsistency — so a caller
448/// planning a session should ask after the link archive, not before it.
449pub async fn archivable_concepts(conn: &libsql::Connection, cutoff: &str) -> Result<Vec<String>> {
450    let mut rows = conn
451        .query(
452            &format!("SELECT id FROM concepts WHERE {CONCEPTS_ARCHIVABLE} ORDER BY id"),
453            libsql::named_params! {":cutoff": cutoff},
454        )
455        .await?;
456
457    let mut ids = Vec::new();
458    while let Some(row) = rows.next().await? {
459        ids.push(row.get::<String>(0)?);
460    }
461    Ok(ids)
462}
463
464/// Whether there is an archive at `path`, as opposed to a file (0.15.19, C-13).
465///
466/// # An empty file is not an archive, and saying otherwise broke reads
467///
468/// Every reader used to ask `path.exists()`. `ATTACH DATABASE` **creates the
469/// file** — before any DDL, and whether or not the session that attached it
470/// ever writes a row — so `exists()` answers "an archive session once began
471/// here", which is not the question any of them is asking.
472///
473/// The gap is reachable through the public API with nothing failing
474/// unexpectedly. [`rehydrate`] on a ledger that has never been archived used to
475/// ATTACH, ask `cold.concepts` a question, be told the table does not exist,
476/// and return that error — leaving a **0-byte file** behind. From then on every
477/// `reconstruct` at an instant below the newest hot stamp took the cold arm and
478/// failed with a raw `no such table: cold.transaction_log`: a database whose
479/// entire history became unreadable because a caller asked to rehydrate
480/// something that was never archived. Measured in
481/// `examples/cold_file_reach_probe.rs`, before and after.
482///
483/// A zero-length file is an empty SQLite database with no tables in it, which
484/// is exactly "no archive". The check is one metadata stat, so it costs what
485/// `exists()` cost, and it is the *healing* half of this repair: a database
486/// already carrying a stray file starts reading correctly again the next time
487/// it is opened, with no operator step and nothing to delete by hand.
488///
489/// It does not open the file. A file that is non-empty but not a cold ledger is
490/// a different failure and belongs to whoever wrote it; what makes that case
491/// narrow is that [`archive_session`] now writes the schema **inside** its own
492/// transaction, so a schema pass cannot half-survive.
493pub(crate) fn archive_present(path: &Path) -> bool {
494    std::fs::metadata(path).is_ok_and(|m| m.len() > 0)
495}
496
497/// Move closed edge intervals and superseded log rows older than `cutoff` into
498/// the cold database at `archive_path` (§5.7, D-012, D-022).
499///
500/// The whole session is one `BEGIN IMMEDIATE … COMMIT` transaction (D-012):
501/// copy-then-delete must be atomic, or a crash between the phases duplicates or
502/// loses rows. The archive-session marker that unlocks the delete guards
503/// (D-008 revised) is created as the first statement of that transaction and
504/// dropped as the last, so it never exists as committed state — commit drops
505/// it, rollback discards it, and there is no crash path that leaves the guards
506/// disarmed.
507///
508/// ATTACH is issued outside the transaction and DETACH is issued unconditionally
509/// on the way out, including on error: ATTACH is not transactional and survives
510/// ROLLBACK, so a leaked handle would make every later archive or cold-DB
511/// reconstruct fail with "database cold is already in use".
512/// `archived_at` is **when the session ran**; `cutoff` is the boundary it used.
513///
514/// Both go into `cold.archive_horizon`, and until Wave 4.5 both columns were
515/// written with the cutoff — so the table recorded that every archive had run at
516/// the instant it was archiving *up to*, which is the one time it certainly did
517/// not run. The two are different clocks (Doctrine II) and the row exists to
518/// carry both: the cutoff says what was moved, `archived_at` says when the
519/// decision was taken, and only the second can answer "how stale is this cold
520/// file". The column was there, correctly named, holding the wrong value.
521pub async fn archive(
522    conn: &libsql::Connection,
523    cutoff: &str,
524    archived_at: &str,
525    archive_path: &Path,
526) -> Result<ArchiveReport> {
527    crate::temporal::replay::detach_stale_cold(conn).await;
528
529    // ATTACH creates the cold file if it does not exist.
530    conn.execute(
531        "ATTACH DATABASE ?1 AS cold",
532        libsql::params![archive_path.to_string_lossy().as_ref()],
533    )
534    .await?;
535
536    let result = archive_session(conn, cutoff, archived_at).await;
537
538    // Unconditional: see the DETACH note above.
539    if let Err(e) = conn.execute("DETACH DATABASE cold", ()).await {
540        tracing::warn!("archive: failed to DETACH cold database: {e}");
541    }
542
543    result
544}
545
546/// Cold indexes that name a column [`upgrade_cold_lineage`] may still be about
547/// to add, so they run *after* it rather than in [`COLD_SCHEMA`] (0.15.12,
548/// W15.2, [D-254]).
549///
550/// # Why the split exists at all
551///
552/// `COLD_SCHEMA` runs before the transaction opens, against a file that may
553/// have been written by any version — including one that predates `branch_id`.
554/// Its two existing `transaction_log` indexes name `entity_id` and
555/// `recorded_at`, which a v11 cold file already had, so the question never came
556/// up. This one names `branch_id`, and `CREATE INDEX` over a column that is not
557/// there yet is `no such column: branch_id` — an archive that refuses a cold
558/// file the previous release accepted, which is the failure this ordering
559/// exists to prevent. Both archive arms upgrade before they insert; running the
560/// index with the upgrade puts it on the far side of that boundary.
561///
562/// # Why the cold file earns an index of its own
563///
564/// Not symmetry with the hot side. Once a database has been archived,
565/// `reconstruct` folds a `UNION ALL` of the two logs — and SQLite compiles that
566/// union as a `MERGE`, which sorts **each side independently**. The two indexes
567/// therefore remove two different sorts, which the probe measured across all
568/// three states (`examples/txlog_fold_index_probe.rs`, 30,000 rows a side):
569///
570/// ```text
571/// neither side indexed   127.2 ms
572/// hot side only          110.1 ms
573/// both sides              96.3 ms
574/// ```
575///
576/// A cold index shipped on the strength of the hot one's numbers would be the
577/// unread index [D-089] exists to refuse. With the union's shape measured, it
578/// has a named reader and is the second half of one improvement.
579///
580/// No cold-side rung is needed and none exists: this runs on every archive
581/// session, so an existing cold database picks the index up the next time
582/// anything is archived into it. A *table* change could not be made that way,
583/// which is the distinction `upgrade_cold_lineage` draws.
584///
585/// [D-089]: ../../docs/architecture/s13-decision-register.md#d-089
586/// [D-254]: ../../docs/architecture/s13-decision-register.md#d-254
587const COLD_LINEAGE_INDICES: &[&str] = &[
588    "CREATE INDEX IF NOT EXISTS cold.idx_cold_txlog_fold_partition ON transaction_log \
589     (table_name, entity_id, branch_id, seq_id DESC)",
590];
591
592/// Bring an existing cold file up to the v12 shape, inside the session's own
593/// transaction (§15.2, D-217).
594///
595/// # Why this is not `CREATE TABLE IF NOT EXISTS`'s job
596///
597/// It cannot be. [`COLD_SCHEMA`] runs against a file that may already hold
598/// these tables, and `IF NOT EXISTS` on an existing name **keeps the old
599/// definition and reports success** — probe §10. A v11 cold file would sail
600/// through the schema pass and then refuse the first insert with `table
601/// cold.transaction_log has no column named branch_id` (probe §11), which is at
602/// least loud; shorten the column list to avoid the error and the lineage is
603/// dropped in silence instead.
604///
605/// # Why it is safe to do here
606///
607/// Probe §12–13 measured both halves on libSQL: `ALTER TABLE cold.… ADD COLUMN`
608/// is accepted inside `BEGIN IMMEDIATE`, an insert in the same transaction sees
609/// the new column, and **`ROLLBACK` takes the DDL with it** — columns and rows
610/// both revert. So a session that fails partway leaves the cold file exactly as
611/// it found it, which is the property that lets an upgrade ride along with an
612/// archive instead of needing a migration of its own.
613///
614/// Detection is column presence. A cold file carries no version stamp worth
615/// trusting: it is a file whose whole purpose is to be moved (D-026).
616///
617/// No foreign key on these columns, unlike their hot counterparts. `branches`
618/// does not exist in the cold file, and a cold file therefore records *that* a
619/// row belonged to a lineage without recording what that lineage was. Named in
620/// §15.5's carry rather than left to be discovered: the abandonment arm makes
621/// forgetting a branch an ordinary operation, and a cold row stamped with a
622/// name nothing resolves is the shape that falls out of it.
623async fn upgrade_cold_lineage(tx: &libsql::Transaction) -> Result<()> {
624    for table in ["links", "concepts", "transaction_log"] {
625        if !cold_has_branch(tx, table).await? {
626            tx.execute(
627                &format!(
628                    "ALTER TABLE cold.{table} ADD COLUMN branch_id TEXT NOT NULL DEFAULT 'main'"
629                ),
630                (),
631            )
632            .await?;
633        }
634    }
635
636    // The column is not the whole of v15. A cold file written by 0.14.8 through
637    // 0.14.14 has `branch_id` and a key that does not mention it, so it passes
638    // the loop above and still refuses the pair the hot table now accepts —
639    // which would make `archive` the operation that fails on a database nothing
640    // else complains about.
641    //
642    // A rebuild rather than an `ALTER`, because SQLite has no way to add a
643    // column to a primary key; the same reason the hot rung is a rebuild. It is
644    // safe in this transaction for the reason above: probe §12–13 established
645    // that `ROLLBACK` takes cold DDL with it, and this adds `CREATE`, `INSERT
646    // … SELECT`, `DROP` and `RENAME` to the `ADD COLUMN` already covered.
647    // `cold.links` carries no trigger and no index, so nothing else has to be
648    // put back.
649    if !cold_links_keyed_by_lineage(tx).await? {
650        tx.execute(COLD_LINKS_V15, ()).await?;
651        tx.execute(
652            "INSERT INTO cold.links_v15 \
653             (source_id, target_id, edge_type, valid_from, recorded_at, \
654              valid_to, weight, properties, branch_id) \
655             SELECT source_id, target_id, edge_type, valid_from, recorded_at, \
656                    valid_to, weight, properties, branch_id FROM cold.links",
657            (),
658        )
659        .await?;
660        tx.execute("DROP TABLE cold.links", ()).await?;
661        tx.execute("ALTER TABLE cold.links_v15 RENAME TO links", ())
662            .await?;
663    }
664
665    // Last, when every column these name is certainly present whatever vintage
666    // the file arrived as. See `COLD_LINEAGE_INDICES` for why they are not in
667    // `COLD_SCHEMA` with the rest of the schema pass.
668    for ddl in COLD_LINEAGE_INDICES {
669        tx.execute(ddl, ()).await?;
670    }
671
672    Ok(())
673}
674
675/// The v15 cold ledger, spelled out because the rebuild needs a second name.
676///
677/// Not derived from [`COLD_SCHEMA`] by string surgery: the two would then be
678/// one definition read two ways, and the failure mode of getting that wrong is
679/// a cold file silently rebuilt into a shape the schema pass does not declare.
680const COLD_LINKS_V15: &str = r#"CREATE TABLE cold.links_v15 (
681        source_id   TEXT NOT NULL,
682        target_id   TEXT NOT NULL,
683        edge_type   TEXT NOT NULL,
684        valid_from  TEXT NOT NULL,
685        recorded_at TEXT NOT NULL,
686        valid_to    TEXT NOT NULL,
687        weight      REAL NOT NULL CHECK (weight >= 0.0 AND weight < 9e999 AND typeof(weight) = 'real'),
688        properties  TEXT NOT NULL,
689        branch_id   TEXT NOT NULL DEFAULT 'main',
690        PRIMARY KEY (source_id, target_id, edge_type, valid_from, recorded_at, branch_id)
691    )"#;
692
693/// Whether `cold.links` has `branch_id` **in its primary key**.
694///
695/// `PRAGMA table_info`'s sixth column is the column's 1-based position in the
696/// key, or 0. Asked of the pragma rather than of the stored SQL for
697/// [`cold_has_branch`]'s reason — a cold file is a file this crate may not have
698/// written, and matching its text would be matching someone else's formatting.
699async fn cold_links_keyed_by_lineage(conn: &libsql::Connection) -> Result<bool> {
700    let mut rows = conn.query("PRAGMA cold.table_info(links)", ()).await?;
701    while let Some(row) = rows.next().await? {
702        let named = row.get::<String>(1).is_ok_and(|name| name == "branch_id");
703        if named && row.get::<i64>(5).is_ok_and(|pk| pk > 0) {
704            return Ok(true);
705        }
706    }
707    Ok(false)
708}
709
710/// Whether one cold table already carries `branch_id`.
711///
712/// Split out because rehydration asks the same question for the opposite
713/// reason: the writer asks so it can upgrade, the reader asks so it can
714/// **avoid** upgrading. A cold file may be read-only media or sit on a share,
715/// and a read path that mutates it is a new failure class.
716async fn cold_has_branch(conn: &libsql::Connection, table: &str) -> Result<bool> {
717    let mut rows = conn
718        .query(&format!("PRAGMA cold.table_info({table})"), ())
719        .await?;
720    while let Some(row) = rows.next().await? {
721        if row.get::<String>(1).is_ok_and(|name| name == "branch_id") {
722            return Ok(true);
723        }
724    }
725    Ok(false)
726}
727
728/// The temp table the keyed repair reads, and the statement that fills it.
729///
730/// Temp rather than a `WITH`: it has to be read **after** the `DELETE` that
731/// makes the rows it names disappear, so the key set must be materialised
732/// before then. It lives in the connection's `temp` database, which the
733/// archive's own `BEGIN IMMEDIATE` covers, and is dropped before the session
734/// ends so a second archive on the same connection starts from nothing.
735const ARCHIVED_KEYS: &str = "archived_keys";
736
737/// Collect the keys a `DELETE FROM links WHERE {clause}` is about to disturb.
738///
739/// Run before the delete, in its transaction, with the delete's own parameters:
740/// the two statements must see the same rows, and the only way to be sure of
741/// that is to give them the same predicate rather than a description of it.
742async fn collect_archived_keys(
743    tx: &libsql::Transaction,
744    clause: &str,
745    params: impl libsql::params::IntoParams,
746) -> Result<()> {
747    tx.execute(&format!("DROP TABLE IF EXISTS temp.{ARCHIVED_KEYS}"), ())
748        .await?;
749    tx.execute(
750        &format!(
751            "CREATE TEMP TABLE {ARCHIVED_KEYS} AS \
752             SELECT DISTINCT {key} FROM links WHERE {clause}",
753            key = crate::integrity::rebuild::PROJECTION_KEY
754        ),
755        params,
756    )
757    .await?;
758    Ok(())
759}
760
761/// Re-derive the projection at the collected keys, then drop the key table.
762///
763/// Called only when the `DELETE` removed something, for
764/// [D-080](../../docs/architecture/s13-decision-register.md#d-080)'s reason:
765/// `links_current` is a function of `links`, so a delete that removed nothing
766/// left nothing to repair. What changed at 0.15.3 is what "repair" costs —
767/// O(keys the session archived) rather than O(every link that survived it).
768async fn repair_archived_keys(tx: &libsql::Transaction) -> Result<()> {
769    crate::integrity::rebuild::repair_keys_within(tx, ARCHIVED_KEYS).await?;
770    tx.execute(&format!("DROP TABLE temp.{ARCHIVED_KEYS}"), ())
771        .await?;
772    Ok(())
773}
774
775/// `conn` is passed alongside `tx` only so [`delete_guarded`] can hand it to
776/// `classify`, which queries on the error path. Both name the same connection.
777async fn archive_session(
778    conn: &libsql::Connection,
779    cutoff: &str,
780    archived_at: &str,
781) -> Result<ArchiveReport> {
782    let tx = conn
783        .transaction_with_behavior(TransactionBehavior::Immediate)
784        .await?;
785
786    // **Inside the transaction, not before it** (0.15.19, review C-13). Cold
787    // DDL is transactional on libSQL — probe §12–13, which `upgrade_cold_lineage`
788    // below already relies on — so a session that fails partway through the
789    // schema pass now leaves the cold file exactly as it found it, instead of
790    // committing a half-declared schema that the next reader would meet as a
791    // missing table. It is not what puts the file on disk (`ATTACH` does that,
792    // and `archive_present` is the answer to it), but it is what makes
793    // "the file is non-empty" mean "the schema is all there".
794    for ddl in COLD_SCHEMA {
795        tx.execute(ddl, ()).await?;
796    }
797
798    // Before the marker, before any insert: an existing cold file may predate
799    // the lineage column, and `IF NOT EXISTS` above will not have added it.
800    upgrade_cold_lineage(&tx).await?;
801
802    // --- archive session opens: the delete guards are now satisfied ---
803    tx.execute(&format!("CREATE TABLE {ARCHIVE_SESSION_MARKER} (x)"), ())
804        .await?;
805
806    let links_archived = tx
807        .execute(
808            &format!(
809                "INSERT OR IGNORE INTO cold.links
810                     (source_id, target_id, edge_type, valid_from, recorded_at,
811                      valid_to, weight, properties, branch_id)
812                 SELECT source_id, target_id, edge_type, valid_from, recorded_at,
813                        valid_to, weight, properties, branch_id
814                 FROM links WHERE {LINKS_ARCHIVABLE}"
815            ),
816            libsql::named_params! {":cutoff": cutoff},
817        )
818        .await? as usize;
819
820    // The keys this session is about to disturb, taken with the delete's own
821    // predicate and before the delete runs. See `collect_archived_keys`.
822    collect_archived_keys(
823        &tx,
824        LINKS_ARCHIVABLE,
825        libsql::named_params! {":cutoff": cutoff},
826    )
827    .await?;
828
829    let links_deleted = delete_guarded(
830        &tx,
831        conn,
832        &format!("DELETE FROM links WHERE {LINKS_ARCHIVABLE}"),
833        libsql::named_params! {":cutoff": cutoff},
834        "links",
835    )
836    .await?;
837
838    // links_current is derivative (Doctrine VI) and must equal the latest-belief
839    // projection of whatever remains in links, or audit_current() reports drift
840    // the moment an archive runs. Re-derive it rather than trying to describe
841    // the deletion's shadow: this used to be a hand-written
842    // `DELETE FROM links_current WHERE valid_to <= :cutoff`, which filters on
843    // *valid* time while LINKS_ARCHIVABLE also requires `recorded_at < :cutoff`.
844    // A row closed at the cutoff but recorded at or after it therefore survived
845    // in links and was deleted from links_current — permanent drift no later
846    // audit could explain, from a compensation that had quietly stopped being
847    // the image of the thing it compensated for. Doctrine II: two clocks, never
848    // mixed. Deriving from the definition cannot drift from the definition.
849    //
850    // **Skipped when the DELETE removed nothing (T1.1, D-080).** `links_current`
851    // is a function of `links`, so if `links` did not change its projection did
852    // not either, and there is no drift for a rebuild to repair. This was
853    // harmless while `archive()` was called once against a whole backlog,
854    // because the one session always had work. It stops being harmless the
855    // moment the caller windows: `rebuild_within` costs O(surviving `links`)
856    // regardless of how much the session archived (D-077), so without this a run
857    // of twenty windows over a quiet stretch of history pays twenty full
858    // reprojections to delete nothing — and windowing makes the archive slower
859    // in total than not windowing. `log_entries_archived` deliberately does not
860    // enter into it: archiving the transaction log cannot change `links`.
861    //
862    // **And the repair is keyed since 0.15.3 (D-245).** The skip above bounded
863    // *how often* the full reprojection ran; it could not bound what one costs,
864    // and a session that archives ten rows from a million-row ledger still paid
865    // for the million. The projection is pointwise in the key, so re-deriving
866    // at the disturbed keys is the same answer — the reasoning is in
867    // `repair_keys_within`, and `audit_current` is what checks it.
868    if links_deleted > 0 {
869        repair_archived_keys(&tx).await?;
870    }
871
872    // Concepts, and **only now** — after the `links` delete, never before it
873    // ([D-128](../../docs/architecture/s13-decision-register.md)). A concept is
874    // archivable when nothing in hot `links` names it, so evaluating the
875    // predicate before the edges have gone cold archives strictly less than the
876    // session is entitled to. This ordering is the whole content of "concept
877    // archival is downstream of link archival".
878    let concepts_archived = archive_concepts(&tx, conn, cutoff).await?;
879
880    let log_entries_archived = tx
881        .execute(
882            &format!(
883                "INSERT OR IGNORE INTO cold.transaction_log
884                     (seq_id, table_name, entity_id, operation, payload, recorded_at, branch_id)
885                 SELECT seq_id, table_name, entity_id, operation, payload, recorded_at, branch_id
886                 FROM transaction_log WHERE {LOG_ARCHIVABLE}"
887            ),
888            libsql::named_params! {":cutoff": cutoff},
889        )
890        .await? as usize;
891
892    delete_guarded(
893        &tx,
894        conn,
895        &format!("DELETE FROM transaction_log WHERE {LOG_ARCHIVABLE}"),
896        libsql::named_params! {":cutoff": cutoff},
897        "transaction_log",
898    )
899    .await?;
900
901    // Record the new horizon in the cold file so a pre-horizon reconstruct can
902    // tell "archived" from "never existed" (glossary; R14).
903    let horizon: Option<i64> = tx
904        .query("SELECT MIN(seq_id) FROM transaction_log", ())
905        .await?
906        .next()
907        .await?
908        .and_then(|row| row.get(0).ok());
909
910    tx.execute(
911        "INSERT INTO cold.archive_horizon (archived_at, cutoff, horizon) VALUES (?1, ?2, ?3)",
912        libsql::params![archived_at, cutoff, horizon],
913    )
914    .await?;
915
916    // --- archive session closes: the guards re-arm before COMMIT ---
917    tx.execute(&format!("DROP TABLE {ARCHIVE_SESSION_MARKER}"), ())
918        .await?;
919
920    tx.commit().await?;
921
922    Ok(ArchiveReport {
923        links_archived,
924        concepts_archived,
925        log_entries_archived,
926        horizon,
927    })
928}
929
930/// Move every concept [`CONCEPTS_ARCHIVABLE`] admits into `cold.concepts`, and
931/// dispose of its derived rows (C2).
932///
933/// # The partition, which is the decision this function encodes
934///
935/// **Entity data crosses; derivative data does not.** The concept row itself is
936/// moved column for column — a move that drops a column is a rewrite, and
937/// [Doctrine V] does not permit an absence the ledger cannot explain. Its
938/// `analytics_annotations` and `embeddings_*` rows are *deleted* rather than
939/// moved, because [Doctrine VII] makes both recomputable from the content that
940/// did cross. Carrying them would also be unimplementable for the vectors:
941/// `F32_BLOB` and DiskANN are libSQL-specific and the cold file is a plain
942/// database opened through `ATTACH`.
943///
944/// The disposal is not incidental to the move — it is what makes the move
945/// legal. `concepts` has four inbound foreign keys, and the two derived ones
946/// would refuse the `DELETE` outright.
947///
948/// # Why the deletes are not logged, and why that is right
949///
950/// `concepts` carries log triggers on insert and update but **not** on delete —
951/// there was no delete path to log while the guard was unconditional, and there
952/// deliberately still is not. Archival mints no transaction-time facts: the
953/// concept is in the cold file, the log entries describing it are either still
954/// hot or in `cold.transaction_log`, and nothing about what was believed, or
955/// when, has changed. A log entry here would assert that something happened to
956/// the concept at archive time, which is exactly the lie [Doctrine III] forbids.
957async fn archive_concepts(
958    tx: &libsql::Transaction,
959    conn: &libsql::Connection,
960    cutoff: &str,
961) -> Result<usize> {
962    let moved = tx
963        .execute(
964            &format!(
965                "INSERT OR IGNORE INTO cold.concepts
966                     (rowid_pk, id, title, content, embedding_model,
967                      valid_from, valid_to, recorded_at, retired, branch_id)
968                 SELECT rowid_pk, id, title, content, embedding_model,
969                        valid_from, valid_to, recorded_at, retired, branch_id
970                 FROM concepts WHERE {CONCEPTS_ARCHIVABLE}"
971            ),
972            libsql::named_params! {":cutoff": cutoff},
973        )
974        .await? as usize;
975
976    if moved == 0 {
977        return Ok(0);
978    }
979
980    // The derived rows, before the concept they hang off. `embeddings_*` is
981    // enumerated from the catalogue rather than from a list, because the set is
982    // whatever `register_model` has created on *this* database and a hard-coded
983    // list would silently miss a model the caller added.
984    let mut derived: Vec<String> = vec!["analytics_annotations".to_string()];
985    let mut rows = tx
986        .query(
987            "SELECT name FROM sqlite_master WHERE type = 'table' \
988             AND name LIKE 'embeddings\\_%' ESCAPE '\\'",
989            (),
990        )
991        .await?;
992    while let Some(row) = rows.next().await? {
993        derived.push(row.get::<String>(0)?);
994    }
995    drop(rows);
996
997    for table in &derived {
998        tx.execute(
999            &format!(
1000                "DELETE FROM {table} WHERE concept_id IN \
1001                 (SELECT id FROM cold.concepts)"
1002            ),
1003            (),
1004        )
1005        .await?;
1006    }
1007
1008    // `trg_concepts_fts_delete` fires on this and keeps the search index
1009    // correct — the capability v8 installed inert and this rung made reachable.
1010    let deleted = delete_guarded(
1011        tx,
1012        conn,
1013        &format!("DELETE FROM concepts WHERE {CONCEPTS_ARCHIVABLE}"),
1014        libsql::named_params! {":cutoff": cutoff},
1015        "concepts",
1016    )
1017    .await? as usize;
1018
1019    debug_assert_eq!(
1020        moved, deleted,
1021        "the predicate selected a different set for the copy than for the delete"
1022    );
1023
1024    Ok(deleted)
1025}
1026
1027/// Move one lineage's whole ledger to the cold file and forget the lineage
1028/// (0.14.13, §15.4, [D-230]).
1029///
1030/// The abandonment arm §15.4 asks for. A conversation tree discards most of what
1031/// it grows, and until now the only way to reclaim an abandoned branch's space
1032/// was [`archive`], which is indexed by *time* and therefore takes the trunk's
1033/// old history along with it — or leaves the branch's recent history behind,
1034/// which is the usual case and the reason the arm exists.
1035///
1036/// # It is all-or-nothing, and that was forced rather than chosen
1037///
1038/// The road map's justification was that "an abandoned branch's rows are a
1039/// contiguous archivable set by construction, which is the cheapest archive
1040/// predicate in the crate". `branch_id = :branch` really is the cheapest
1041/// predicate in the crate. **Contiguous by construction is false in both of its
1042/// senses**, and each refutation moved this design:
1043///
1044/// 1. *Not closed under `concepts(id)`.* `concepts` is keyed by identity
1045///    globally ([D-214]), so a concept minted on a branch may be named by a
1046///    trunk edge or a sibling's edge — measured by probe, both succeed. The set
1047///    is therefore not FK-closed, and the refusal below is the direct
1048///    expression of that: a lineage another lineage's hot edges still depend on
1049///    is not abandoned, whatever its author believes.
1050/// 2. *Not a prefix of the log.* A branch's `transaction_log` rows are
1051///    scattered through the sequence, exactly as `LOG_ARCHIVABLE`'s are, which
1052///    is what [`crate::temporal::replay`]'s reach test was rewritten for in
1053///    0.5.5.
1054///
1055/// What follows is a chain with no branch points. The links must go — that is
1056/// the operation. If the links go and the log stays, `reconstruct(now)` folds
1057/// the log, yields the branch's open edges, and disagrees with `links_current`
1058/// about present belief; so the log must go too. But `hot_log_reach`'s
1059/// soundness rests on **the newest row per entity is never archivable**, which
1060/// is true of a predicate needing a later row to exist and false of one that
1061/// takes a whole lineage; so the `branches` row must go as well, which is what
1062/// makes a hot fold that omits the lineage *correct rather than silently
1063/// short*. Every read and write naming the name then raises
1064/// [`DbError::UnknownBranch`] — a refusal, which a caller can act on, in place
1065/// of an answer that is quietly missing rows.
1066///
1067/// The `branches` row moving is why v13 exists: `trg_branches_frozen_delete`
1068/// was unconditional, on a docstring that said no session could ever legally
1069/// remove a lineage record. See
1070/// [`crate::schema::ddl::CREATE_BRANCHES_GUARD_DELETE`].
1071///
1072/// # What it refuses
1073///
1074/// * **The trunk.** Every lineage's `parent_id` chain ends there and every
1075///   default `branch_id` names it; there is no ledger left after it goes.
1076/// * **A name that is not registered** — [`DbError::UnknownBranch`], the same
1077///   answer every other branch-taking surface gives, rather than a silent
1078///   success archiving nothing.
1079/// * **A branch with descendants.** A child reads through its parent, so
1080///   archiving the parent would delete rows the child still believes — the same
1081///   loss [D-229] repaired in the time-indexed predicates, arrived at from the
1082///   other direction.
1083/// * **A branch whose concepts another lineage's hot link names.** This is
1084///   refutation 1 above, and the refusal is what makes the post-condition
1085///   uniform: after this returns `Ok`, nothing hot names the lineage and
1086///   nothing hot names anything it minted.
1087///
1088/// All four are checked **inside the session transaction**, before the marker
1089/// is created, so a concurrent fork cannot slip a descendant in between the
1090/// check and the delete.
1091///
1092/// # No `archive_horizon` row, deliberately
1093///
1094/// That table records a **cutoff** and the horizon it produced. This session
1095/// has no cutoff — its boundary is a lineage, not an instant — and writing
1096/// `archived_at` into the `cutoff` column would be the Wave 4.5 defect the
1097/// column's own comment describes, committed a second time on purpose. What
1098/// there is to record is recorded better: `cold.branches` carries the lineage
1099/// and when it was forgotten, and the horizon itself is still readable from the
1100/// hot log, which is where `archive_hint` reads it from anyway.
1101///
1102/// [D-230]: ../../docs/architecture/s13-decision-register.md#d-230
1103/// [D-229]: ../../docs/architecture/s13-decision-register.md#d-229
1104/// [D-214]: ../../docs/architecture/s13-decision-register.md#d-214
1105pub async fn archive_branch(
1106    conn: &libsql::Connection,
1107    branch: &str,
1108    archived_at: &str,
1109    archive_path: &Path,
1110) -> Result<ArchiveReport> {
1111    crate::temporal::replay::detach_stale_cold(conn).await;
1112
1113    conn.execute(
1114        "ATTACH DATABASE ?1 AS cold",
1115        libsql::params![archive_path.to_string_lossy().as_ref()],
1116    )
1117    .await?;
1118
1119    let result = archive_branch_session(conn, branch, archived_at).await;
1120
1121    // Unconditional, for [`archive`]'s reason: a live `cold` handle makes every
1122    // later archive and cold reconstruct fail with "database cold is already in
1123    // use", and the refusals above are the *expected* way out of this function.
1124    if let Err(e) = conn.execute("DETACH DATABASE cold", ()).await {
1125        tracing::warn!("archive_branch: failed to DETACH cold database: {e}");
1126    }
1127
1128    result
1129}
1130
1131fn not_archivable(branch: &str, reason: impl Into<String>) -> DbError {
1132    DbError::BranchNotArchivable {
1133        branch: branch.to_string(),
1134        reason: reason.into(),
1135    }
1136}
1137
1138/// Whether `sql` — a `SELECT 1 … WHERE … = :branch` — matches anything.
1139async fn any_row(tx: &libsql::Transaction, sql: &str, branch: &str) -> Result<bool> {
1140    Ok(tx
1141        .query(sql, libsql::named_params! {":branch": branch})
1142        .await?
1143        .next()
1144        .await?
1145        .is_some())
1146}
1147
1148/// The four refusals, in the order that makes the message most specific.
1149///
1150/// Order is not cosmetic. The trunk check comes first because `main` is
1151/// registered and childless on a ledger that has never forked, so every later
1152/// check would pass it. Registration comes next, because "not registered" is a
1153/// better answer than "has no descendants" for a typo. Descendants before
1154/// concepts because it is the cheaper query and the commoner mistake.
1155async fn refuse_unarchivable_branch(tx: &libsql::Transaction, branch: &str) -> Result<()> {
1156    if branch == crate::schema::ddl::MAIN_BRANCH {
1157        return Err(not_archivable(
1158            branch,
1159            "it is the trunk: every lineage's parent chain ends there and every \
1160             default branch_id names it, so there is no ledger left after it goes",
1161        ));
1162    }
1163
1164    if !any_row(
1165        tx,
1166        "SELECT 1 FROM branches WHERE branch_id = :branch",
1167        branch,
1168    )
1169    .await?
1170    {
1171        return Err(DbError::UnknownBranch(branch.to_string()));
1172    }
1173
1174    if any_row(
1175        tx,
1176        "SELECT 1 FROM branches WHERE parent_id = :branch",
1177        branch,
1178    )
1179    .await?
1180    {
1181        return Err(not_archivable(
1182            branch,
1183            "it has descendants, which read through it: archiving it would delete \
1184             rows they still believe. Archive the descendants first",
1185        ));
1186    }
1187
1188    // The refutation of "contiguous by construction", as a query. A concept is
1189    // keyed by identity across the whole ledger (D-214), so an edge on any
1190    // lineage may name one minted here.
1191    let mut rows = tx
1192        .query(
1193            "SELECT c.id FROM concepts c
1194             WHERE c.branch_id = :branch
1195               AND EXISTS (
1196                   SELECT 1 FROM links l
1197                   WHERE l.branch_id <> :branch
1198                     AND (l.source_id = c.id OR l.target_id = c.id)
1199               )
1200             LIMIT 1",
1201            libsql::named_params! {":branch": branch},
1202        )
1203        .await?;
1204    if let Some(row) = rows.next().await? {
1205        let id: String = row.get(0)?;
1206        return Err(not_archivable(
1207            branch,
1208            format!(
1209                "concept {id} was minted here and a hot edge on another lineage \
1210                 names it. A lineage other lineages still depend on is not \
1211                 abandoned; retire those edges first"
1212            ),
1213        ));
1214    }
1215
1216    Ok(())
1217}
1218
1219/// `conn` is passed alongside `tx` for [`delete_guarded`]'s sake, exactly as in
1220/// [`archive_session`]. Both name the same connection.
1221async fn archive_branch_session(
1222    conn: &libsql::Connection,
1223    branch: &str,
1224    archived_at: &str,
1225) -> Result<ArchiveReport> {
1226    let tx = conn
1227        .transaction_with_behavior(TransactionBehavior::Immediate)
1228        .await?;
1229
1230    // **Inside the transaction, not before it** (0.15.19, review C-13). Cold
1231    // DDL is transactional on libSQL — probe §12–13, which `upgrade_cold_lineage`
1232    // below already relies on — so a session that fails partway through the
1233    // schema pass now leaves the cold file exactly as it found it, instead of
1234    // committing a half-declared schema that the next reader would meet as a
1235    // missing table. It is not what puts the file on disk (`ATTACH` does that,
1236    // and `archive_present` is the answer to it), but it is what makes
1237    // "the file is non-empty" mean "the schema is all there".
1238    for ddl in COLD_SCHEMA {
1239        tx.execute(ddl, ()).await?;
1240    }
1241
1242    upgrade_cold_lineage(&tx).await?;
1243
1244    // Before the marker: a refusal must not be able to leave the guards
1245    // disarmed, and these are reads, which need no session.
1246    refuse_unarchivable_branch(&tx, branch).await?;
1247
1248    // --- archive session opens: the delete guards are now satisfied ---
1249    tx.execute(&format!("CREATE TABLE {ARCHIVE_SESSION_MARKER} (x)"), ())
1250        .await?;
1251
1252    // **`AND branch_id <> 'main'` on all five of the statements below is not
1253    // redundant, however much it reads like it** (0.15.30, [D-273]).
1254    //
1255    // It is redundant as *logic* — `refuse_unarchivable_branch` refused the
1256    // trunk three statements ago, so `branch_id = :branch` already cannot match
1257    // a trunk row — and that is exactly why it can be written. It is there to
1258    // reach `idx_links_branch` and `idx_txlog_branch`, which are **partial**
1259    // indexes over the same predicate: SQLite uses one only where the query's
1260    // `WHERE` implies the index's, and `branch_id = :branch` against a bound
1261    // parameter implies nothing it can prove.
1262    //
1263    // What that buys is the whole of [D-273]: these statements stop scanning
1264    // trunk-sized tables, so archiving a twenty-row lineage stops costing what
1265    // the ledger costs — **22.0 ms to 12.0 ms at an 8,000-edge trunk** — while
1266    // the indexes themselves
1267    // hold only what lineages other than the trunk wrote and cost the write path
1268    // nothing measurable. `the_archive_seeks_the_lineage` in
1269    // `tests/index_plan_tests.rs` is what keeps these five texts and that DDL
1270    // agreeing; delete the predicate from any one of them and it goes red with
1271    // the scan in the message.
1272    //
1273    // **The cutoff path above must not have it.** `archive_session` archives
1274    // across every lineage including the trunk, so the same predicate there
1275    // would silently stop archiving most of the ledger. That is the reason this
1276    // is written out five times rather than folded into a shared constant with
1277    // the other archive's clauses.
1278    //
1279    // [D-273]: ../../docs/architecture/s13-decision-register.md#d-273
1280    let links_archived = tx
1281        .execute(
1282            &format!(
1283                "INSERT OR IGNORE INTO cold.links
1284                 (source_id, target_id, edge_type, valid_from, recorded_at,
1285                  valid_to, weight, properties, branch_id)
1286             SELECT source_id, target_id, edge_type, valid_from, recorded_at,
1287                    valid_to, weight, properties, branch_id
1288             FROM links WHERE branch_id = :branch AND branch_id <> '{main}'",
1289                main = crate::schema::ddl::MAIN_BRANCH
1290            ),
1291            libsql::named_params! {":branch": branch},
1292        )
1293        .await? as usize;
1294
1295    collect_archived_keys(
1296        &tx,
1297        &format!(
1298            "branch_id = :branch AND branch_id <> '{main}'",
1299            main = crate::schema::ddl::MAIN_BRANCH
1300        ),
1301        libsql::named_params! {":branch": branch},
1302    )
1303    .await?;
1304
1305    let links_deleted = delete_guarded(
1306        &tx,
1307        conn,
1308        &format!(
1309            "DELETE FROM links WHERE branch_id = :branch AND branch_id <> '{main}'",
1310            main = crate::schema::ddl::MAIN_BRANCH
1311        ),
1312        libsql::named_params! {":branch": branch},
1313        "links",
1314    )
1315    .await?;
1316
1317    // Doctrine VI, and [`archive_session`]'s reasoning verbatim: `links_current`
1318    // is a function of `links`, so it is re-derived rather than described, and
1319    // only when `links` actually changed. It must also happen **before** the
1320    // `branches` row goes: `links_current.branch_id` carries the same foreign
1321    // key its three siblings do, so the projection has to have stopped naming
1322    // the lineage before the lineage can leave.
1323    //
1324    // Keyed since 0.15.3 (D-245), and here the key set is every key the lineage
1325    // held — which is the whole of what it wrote and *not* the whole of
1326    // `links`, so a branch archived out of a large trunk stops paying for the
1327    // trunk.
1328    if links_deleted > 0 {
1329        repair_archived_keys(&tx).await?;
1330    }
1331
1332    // Concepts after links, for [D-128]'s reason turned around: there it was
1333    // that a concept is archivable only once nothing hot names it, so the edges
1334    // must go first. Here the same ordering is a foreign key — `links.source_id`
1335    // and `links.target_id` reference `concepts(id)`, and this lineage's own
1336    // edges are the ones that would refuse the delete.
1337    //
1338    // [D-128]: ../../docs/architecture/s13-decision-register.md#d-128
1339    let concepts_archived = archive_branch_concepts(&tx, conn, branch).await?;
1340
1341    let log_entries_archived = tx
1342        .execute(
1343            &format!(
1344                "INSERT OR IGNORE INTO cold.transaction_log
1345                 (seq_id, table_name, entity_id, operation, payload, recorded_at, branch_id)
1346             SELECT seq_id, table_name, entity_id, operation, payload, recorded_at, branch_id
1347             FROM transaction_log WHERE branch_id = :branch \
1348                 AND branch_id <> '{main}'",
1349                main = crate::schema::ddl::MAIN_BRANCH
1350            ),
1351            libsql::named_params! {":branch": branch},
1352        )
1353        .await? as usize;
1354
1355    delete_guarded(
1356        &tx,
1357        conn,
1358        &format!(
1359            "DELETE FROM transaction_log WHERE branch_id = :branch AND branch_id <> '{main}'",
1360            main = crate::schema::ddl::MAIN_BRANCH
1361        ),
1362        libsql::named_params! {":branch": branch},
1363        "transaction_log",
1364    )
1365    .await?;
1366
1367    // Last, because the other three tables' `branch_id` all reference it. The
1368    // `archived_at` is the session's wall clock, not a ledger fact: nothing was
1369    // asserted or retired here, and Doctrine III would refuse it if it were.
1370    tx.execute(
1371        "INSERT OR IGNORE INTO cold.branches
1372             (branch_id, parent_id, forked_at, created_at, archived_at)
1373         SELECT branch_id, parent_id, forked_at, created_at, ?2
1374         FROM branches WHERE branch_id = ?1",
1375        libsql::params![branch, archived_at],
1376    )
1377    .await?;
1378
1379    delete_guarded(
1380        &tx,
1381        conn,
1382        "DELETE FROM branches WHERE branch_id = :branch",
1383        libsql::named_params! {":branch": branch},
1384        "branches",
1385    )
1386    .await?;
1387
1388    let horizon: Option<i64> = tx
1389        .query("SELECT MIN(seq_id) FROM transaction_log", ())
1390        .await?
1391        .next()
1392        .await?
1393        .and_then(|row| row.get(0).ok());
1394
1395    // --- archive session closes: the guards re-arm before COMMIT ---
1396    tx.execute(&format!("DROP TABLE {ARCHIVE_SESSION_MARKER}"), ())
1397        .await?;
1398
1399    tx.commit().await?;
1400
1401    Ok(ArchiveReport {
1402        links_archived,
1403        concepts_archived,
1404        log_entries_archived,
1405        horizon,
1406    })
1407}
1408
1409/// [`archive_concepts`] with the lineage predicate in place of the cutoff.
1410///
1411/// A separate function rather than a parameter on that one, because the two
1412/// share their *shape* and not their argument: `CONCEPTS_ARCHIVABLE` is a
1413/// standing predicate about retirement and reference counts, and this is a
1414/// lineage. The partition it encodes is the same and is the reason both exist —
1415/// **entity data crosses, derivative data does not** — and the derived rows are
1416/// deleted here for the same two reasons: Doctrine VII makes them recomputable,
1417/// and `concepts`' inbound foreign keys would refuse the delete otherwise.
1418///
1419/// No guard on "is anything still referencing this concept": the caller has
1420/// already refused the branch if another lineage's hot link names one of its
1421/// concepts, and this lineage's own links went cold a few statements ago.
1422async fn archive_branch_concepts(
1423    tx: &libsql::Transaction,
1424    conn: &libsql::Connection,
1425    branch: &str,
1426) -> Result<usize> {
1427    let moved = tx
1428        .execute(
1429            "INSERT OR IGNORE INTO cold.concepts
1430                 (rowid_pk, id, title, content, embedding_model,
1431                  valid_from, valid_to, recorded_at, retired, branch_id)
1432             SELECT rowid_pk, id, title, content, embedding_model,
1433                    valid_from, valid_to, recorded_at, retired, branch_id
1434             FROM concepts WHERE branch_id = :branch",
1435            libsql::named_params! {":branch": branch},
1436        )
1437        .await? as usize;
1438
1439    if moved == 0 {
1440        return Ok(0);
1441    }
1442
1443    let mut derived: Vec<String> = vec!["analytics_annotations".to_string()];
1444    let mut rows = tx
1445        .query(
1446            "SELECT name FROM sqlite_master WHERE type = 'table' \
1447             AND name LIKE 'embeddings\\_%' ESCAPE '\\'",
1448            (),
1449        )
1450        .await?;
1451    while let Some(row) = rows.next().await? {
1452        derived.push(row.get::<String>(0)?);
1453    }
1454    drop(rows);
1455
1456    for table in &derived {
1457        tx.execute(
1458            &format!(
1459                "DELETE FROM {table} WHERE concept_id IN \
1460                 (SELECT id FROM concepts WHERE branch_id = :branch)"
1461            ),
1462            libsql::named_params! {":branch": branch},
1463        )
1464        .await?;
1465    }
1466
1467    let deleted = delete_guarded(
1468        tx,
1469        conn,
1470        "DELETE FROM concepts WHERE branch_id = :branch",
1471        libsql::named_params! {":branch": branch},
1472        "concepts",
1473    )
1474    .await? as usize;
1475
1476    debug_assert_eq!(
1477        moved, deleted,
1478        "the lineage selected a different set for the copy than for the delete"
1479    );
1480
1481    Ok(deleted)
1482}
1483
1484/// One cold concept row, read as part of a chunk (0.15.19, review C-22).
1485///
1486/// A named struct rather than a tuple because the insert below binds ten
1487/// columns in an order the reader has to be able to check against the DDL, and
1488/// `row.4` is not checkable. `Clone` is one row's worth of strings, taken so
1489/// the loop can destructure by value while the map keeps the chunk alive for
1490/// the ids after it.
1491#[derive(Clone)]
1492struct ColdConcept {
1493    old_rowid: i64,
1494    title: String,
1495    content: String,
1496    model: Option<String>,
1497    valid_from: String,
1498    valid_to: String,
1499    recorded_at: String,
1500    retired: i64,
1501    branch_id: String,
1502}
1503
1504/// Outcome of one rehydration (0.9.0, C3).
1505#[derive(Debug, Clone, PartialEq, Eq)]
1506#[non_exhaustive]
1507pub struct RehydrateReport {
1508    /// Concepts moved back into the hot table.
1509    pub concepts_rehydrated: usize,
1510    /// Of those, how many could **not** keep their original `rowid_pk` because
1511    /// something else had claimed it while they were cold, and were reassigned
1512    /// with the FTS index re-pointed to match.
1513    ///
1514    /// Reported rather than hidden because it is the one way a rehydrated
1515    /// concept differs from the row that was archived, and a caller comparing
1516    /// rowids across the boundary should be able to see that it happened.
1517    pub rowids_reassigned: usize,
1518}
1519
1520/// Move concepts back from the cold file into the hot tables (§2.3, C3).
1521///
1522/// # Rehydration is a move back, not a write
1523///
1524/// It mints no transaction-time facts and is invisible to both clocks. The
1525/// concept's log entries were never removed, so the ledger already says
1526/// everything true about it; writing a fresh `'I'` would assert the concept was
1527/// *learned* at rehydration time, and — because the fold takes the highest
1528/// `seq_id` per entity — would additionally outrank any later `'U'` that retired
1529/// it. See [`crate::schema::ddl::CREATE_CONCEPTS_LOG_INSERT`], which is
1530/// marker-gated at v10 for exactly this reason. The whole operation therefore
1531/// runs inside a declared archive session, which is what suppresses the trigger.
1532///
1533/// # `rowid_pk`: reinstate, or reassign and re-point the index
1534///
1535/// The common case has no collision — the rowid was freed by archival and
1536/// nothing has claimed it since — and reinstating is the clean move-back with no
1537/// side effects at all. When something *has* taken it, the fallback is a fresh
1538/// `rowid_pk` plus an FTS correction: `concepts_fts` is external-content keyed
1539/// on `rowid_pk` ([D-119]), so a reassignment without re-pointing leaves the
1540/// index describing the wrong row, silently. Both exits are taken here rather
1541/// than one being assumed, and [`RehydrateReport::rowids_reassigned`] reports
1542/// which was used.
1543///
1544/// # A concept outlives its lineage, and is refused (0.15.11, W15.1, C-3)
1545///
1546/// [`Database::archive_branch`](crate::Database::archive_branch) takes a
1547/// lineage's `branches` row with it, and a cold
1548/// concept keeps the `branch_id` it was minted on. Rehydrating one after the
1549/// other therefore reinstates a row whose lineage no longer exists — which
1550/// `concepts.branch_id REFERENCES branches(branch_id)` refuses, with foreign
1551/// keys on. It is refused *here* instead, as
1552/// [`DbError::BranchArchived`] naming both the concept and the lineage, for
1553/// the reason that variant's own documentation gives: the engine's message
1554/// names neither, and blames the table that was being written rather than the
1555/// one that is missing.
1556///
1557/// Ids whose lineage is intact are unaffected — the refusal names one concept,
1558/// not the batch.
1559///
1560/// # Errors
1561///
1562/// [`DbError::BranchArchived`] when a requested concept's lineage has no row in
1563/// `branches`. Nothing is written: the whole rehydrate runs in one transaction
1564/// and the refusal leaves it uncommitted.
1565pub async fn rehydrate(
1566    conn: &libsql::Connection,
1567    ids: &[&str],
1568    archive_path: &Path,
1569) -> Result<RehydrateReport> {
1570    if ids.is_empty() {
1571        return Ok(RehydrateReport {
1572            concepts_rehydrated: 0,
1573            rowids_reassigned: 0,
1574        });
1575    }
1576
1577    // No archive, nothing to move back — and **no ATTACH**, which is the point
1578    // (0.15.19, review C-13). `ATTACH` creates the file, so asking this question
1579    // by attaching used to leave a 0-byte cold file behind on every ledger that
1580    // had never been archived, which then broke every historical read. Reported
1581    // as "nothing was rehydrated" rather than as an error, because that is
1582    // already this function's answer for an id the cold file does not hold: a
1583    // missing archive holds none of them.
1584    if !archive_present(archive_path) {
1585        return Ok(RehydrateReport {
1586            concepts_rehydrated: 0,
1587            rowids_reassigned: 0,
1588        });
1589    }
1590
1591    crate::temporal::replay::detach_stale_cold(conn).await;
1592    conn.execute(
1593        "ATTACH DATABASE ?1 AS cold",
1594        libsql::params![archive_path.to_string_lossy().as_ref()],
1595    )
1596    .await?;
1597
1598    let result = rehydrate_session(conn, ids).await;
1599
1600    if let Err(e) = conn.execute("DETACH DATABASE cold", ()).await {
1601        tracing::warn!("rehydrate: failed to DETACH cold database: {e}");
1602    }
1603    result
1604}
1605
1606async fn rehydrate_session(conn: &libsql::Connection, ids: &[&str]) -> Result<RehydrateReport> {
1607    let tx = conn
1608        .transaction_with_behavior(TransactionBehavior::Immediate)
1609        .await?;
1610
1611    // The session opens for the same reason the archive's does, plus one more:
1612    // it is what stops `trg_concepts_log_insert` from firing (v10).
1613    tx.execute(&format!("CREATE TABLE {ARCHIVE_SESSION_MARKER} (x)"), ())
1614        .await?;
1615
1616    // Asked once, and never acted on. A cold file that predates v12 is read
1617    // through a literal — `'main'` is what those rows *were*, since they were
1618    // written when only the trunk existed — and left exactly as it was found.
1619    // The archive writer upgrades cold files; the reader must not, because a
1620    // cold file can be read-only media or sit on a share, and a read path that
1621    // mutates one is a new failure class (D-026, §15.2).
1622    let lineage = if cold_has_branch(&tx, "concepts").await? {
1623        "branch_id"
1624    } else {
1625        "'main' AS branch_id"
1626    };
1627
1628    // Every lineage the hot ledger still registers, read once (0.15.11, W15.1,
1629    // C-3). One query for the whole call rather than one per id: the set is
1630    // bounded by how many lineages exist, not by how many concepts are being
1631    // moved, and the loop below is already one query per id without adding a
1632    // second. A cold file that predates the lineage column reads the literal
1633    // `'main'` above, which is in this set on every ledger — `archive_branch`
1634    // refuses the trunk, so the one name the fallback can produce is the one
1635    // name that cannot be missing.
1636    let mut live_lineages = std::collections::HashSet::new();
1637    let mut rows = tx.query("SELECT branch_id FROM branches", ()).await?;
1638    while let Some(row) = rows.next().await? {
1639        live_lineages.insert(row.get::<String>(0)?);
1640    }
1641    drop(rows);
1642
1643    let mut rehydrated = 0usize;
1644    let mut reassigned = 0usize;
1645
1646    // **One `SELECT` per chunk of [`HYDRATE_CHUNK`] ids, not one per id**
1647    // (0.15.19, review C-22). This was a `SELECT`, a `COUNT(*)`, an `INSERT`
1648    // and a `DELETE` for every id — four round trips each, on a path that holds
1649    // `BEGIN IMMEDIATE` for the whole call. Rehydration is rare, so what this
1650    // buys is the length of that hold rather than throughput: the write lock is
1651    // what the rest of the database is waiting on, and two of the four trips
1652    // per id are reads that a chunk answers at once.
1653    //
1654    // What stays per row is what is genuinely conditional — the rowid
1655    // reinstatement and its FTS repair, which depend on whether something has
1656    // claimed the old rowid in the meantime.
1657    //
1658    // **The caller's order is preserved**, and that is not incidental. The
1659    // refusal below names *one* concept, and `IN (…)` returns rows in whatever
1660    // order the engine likes, so reading a chunk and folding it in arrival
1661    // order would make which concept gets named depend on the query plan. The
1662    // chunk is indexed by id and then walked in the order the caller gave, so
1663    // the same call refuses the same concept every time.
1664    for chunk in ids.chunks(HYDRATE_CHUNK) {
1665        let placeholders = std::iter::repeat_n("?", chunk.len())
1666            .collect::<Vec<_>>()
1667            .join(", ");
1668        // Collected rather than passed as a borrowing iterator: an iterator
1669        // that captures `chunk` makes this future's `Send` bound
1670        // higher-ranked, and the actor that spawns it then fails to prove
1671        // `Send` for half a dozen unrelated types. Owned values, one small
1672        // `Vec` per chunk.
1673        let bind: Vec<libsql::Value> = chunk.iter().map(|id| libsql::Value::from(*id)).collect();
1674        let mut rows = tx
1675            .query(
1676                &format!(
1677                    "SELECT rowid_pk, id, title, content, embedding_model, \
1678                     valid_from, valid_to, recorded_at, retired, {lineage} \
1679                     FROM cold.concepts WHERE id IN ({placeholders})"
1680                ),
1681                libsql::params_from_iter(bind),
1682            )
1683            .await?;
1684
1685        let mut found: std::collections::HashMap<String, ColdConcept> =
1686            std::collections::HashMap::with_capacity(chunk.len());
1687        while let Some(row) = rows.next().await? {
1688            let id: String = row.get(1)?;
1689            found.insert(
1690                id,
1691                ColdConcept {
1692                    old_rowid: row.get(0)?,
1693                    title: row.get(2)?,
1694                    content: row.get(3)?,
1695                    model: row.get(4)?,
1696                    valid_from: row.get(5)?,
1697                    valid_to: row.get(6)?,
1698                    recorded_at: row.get(7)?,
1699                    retired: row.get(8)?,
1700                    branch_id: row.get(9)?,
1701                },
1702            );
1703        }
1704        drop(rows);
1705
1706        // The ids of this chunk the cold file actually held, in the caller's
1707        // order, so the one `DELETE` at the end names exactly what was moved.
1708        let mut moved: Vec<&str> = Vec::with_capacity(found.len());
1709
1710        for id in chunk {
1711            let Some(cold) = found.get(*id) else {
1712                continue;
1713            };
1714            let ColdConcept {
1715                old_rowid,
1716                title,
1717                content,
1718                model,
1719                valid_from,
1720                valid_to,
1721                recorded_at,
1722                retired,
1723                branch_id,
1724            } = cold.clone();
1725
1726            // The lineage has to exist before the row that names it can go back
1727            // (0.15.11, W15.1, C-3). Checked against the set read above rather than
1728            // left to `concepts.branch_id REFERENCES branches(branch_id)`, which
1729            // would refuse this same insert as `FOREIGN KEY constraint failed` —
1730            // an engine-kind error naming the concepts table, when what is missing
1731            // is a branch.
1732            //
1733            // Refused here rather than in a pass over every requested id first: the
1734            // transaction has not committed, so the ids ahead of this one are not
1735            // written either way, and hoisting the reads would hold every payload
1736            // in memory to gain what the rollback already gives. What that costs is
1737            // stated rather than hidden — the work done for the earlier ids is
1738            // spent and discarded.
1739            if !live_lineages.contains(&branch_id) {
1740                return Err(DbError::BranchArchived {
1741                    branch: branch_id,
1742                    concept: (*id).to_string(),
1743                });
1744            }
1745
1746            let taken: i64 = tx
1747                .query(
1748                    "SELECT COUNT(*) FROM concepts WHERE rowid_pk = ?1",
1749                    libsql::params![old_rowid],
1750                )
1751                .await?
1752                .next()
1753                .await?
1754                .expect("COUNT(*) always returns a row")
1755                .get(0)?;
1756
1757            if taken == 0 {
1758                // The clean move back: same row, same rowid, no side effects.
1759                tx.execute(
1760                    "INSERT INTO concepts (rowid_pk, id, title, content, embedding_model, \
1761                 valid_from, valid_to, recorded_at, retired, branch_id) \
1762                 VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)",
1763                    libsql::params![
1764                        old_rowid,
1765                        *id,
1766                        title.clone(),
1767                        content.clone(),
1768                        model,
1769                        valid_from,
1770                        valid_to,
1771                        recorded_at,
1772                        retired,
1773                        branch_id
1774                    ],
1775                )
1776                .await?;
1777            } else {
1778                // Something claimed the rowid while this concept was cold. Take a
1779                // fresh one, then correct the index: `concepts_fts` is
1780                // external-content keyed on `rowid_pk`, and its insert trigger will
1781                // have written an entry at the *new* rowid — what has to be undone
1782                // is the stale entry still sitting at the old one, which the archive
1783                // could not remove because the row it described had already gone.
1784                tx.execute(
1785                    "INSERT INTO concepts (id, title, content, embedding_model, \
1786                 valid_from, valid_to, recorded_at, retired, branch_id) \
1787                 VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)",
1788                    libsql::params![
1789                        *id,
1790                        title.clone(),
1791                        content.clone(),
1792                        model,
1793                        valid_from,
1794                        valid_to,
1795                        recorded_at,
1796                        retired,
1797                        branch_id
1798                    ],
1799                )
1800                .await?;
1801                tx.execute(
1802                    "INSERT INTO concepts_fts (concepts_fts, rowid, title, content) \
1803                 VALUES ('delete', ?1, ?2, ?3)",
1804                    libsql::params![old_rowid, title, content],
1805                )
1806                .await?;
1807                reassigned += 1;
1808            }
1809
1810            moved.push(*id);
1811            rehydrated += 1;
1812        }
1813
1814        if !moved.is_empty() {
1815            let placeholders = std::iter::repeat_n("?", moved.len())
1816                .collect::<Vec<_>>()
1817                .join(", ");
1818            let bind: Vec<libsql::Value> =
1819                moved.iter().map(|id| libsql::Value::from(*id)).collect();
1820            tx.execute(
1821                &format!("DELETE FROM cold.concepts WHERE id IN ({placeholders})"),
1822                libsql::params_from_iter(bind),
1823            )
1824            .await?;
1825        }
1826    }
1827
1828    tx.execute(&format!("DROP TABLE {ARCHIVE_SESSION_MARKER}"), ())
1829        .await?;
1830    tx.commit().await?;
1831
1832    Ok(RehydrateReport {
1833        concepts_rehydrated: rehydrated,
1834        rowids_reassigned: reassigned,
1835    })
1836}
1837
1838/// Run one of the archive's `DELETE`s, naming the table if a guard refuses it.
1839///
1840/// **This is what closes defect AC, and the shape of the fix is the point.**
1841/// There used to be a second classifier here — `classify_archive_violation` —
1842/// which was defined, delegated correctly to [`crate::error::abort_kind`], and
1843/// called from nowhere, so `DbError::ArchiveViolation` was unreachable by any
1844/// code path in the crate. It was recorded as defect H, marked Fixed by a commit
1845/// that made the *body* delegate rather than making the function *called*, and
1846/// so survived its own repair. It is deleted rather than wired up, because
1847/// [`crate::error::classify`] with [`WriteOp::Delete`] already did exactly what
1848/// it did: the defect was one classifier too many, not one too few.
1849///
1850/// A guard firing here means the marker table is absent or was dropped early —
1851/// the session's invariant broken from inside. That is worth a typed error
1852/// naming the table rather than a raw engine message naming a trigger.
1853async fn delete_guarded(
1854    tx: &libsql::Transaction,
1855    conn: &libsql::Connection,
1856    sql: &str,
1857    params: impl libsql::params::IntoParams,
1858    table: &str,
1859) -> Result<u64> {
1860    match tx.execute(sql, params).await {
1861        Ok(n) => Ok(n),
1862        Err(e) => Err(crate::error::classify(conn, e, WriteOp::Delete { table }).await),
1863    }
1864}
1865
1866#[cfg(test)]
1867mod tests {
1868    use super::*;
1869
1870    const EPOCH: &str = "1970-01-01T00:00:00.000000Z";
1871    const OPEN: &str = "9999-12-31T23:59:59.999999Z";
1872    const CLOSED: &str = "1970-01-01T00:30:00.000000Z";
1873    const CUTOFF: &str = "1970-01-01T02:00:00.000000Z";
1874
1875    async fn seeded() -> libsql::Connection {
1876        let db = libsql::Builder::new_local(":memory:")
1877            .build()
1878            .await
1879            .unwrap();
1880        let conn = db.connect().unwrap();
1881        crate::schema::run_migrations(&conn).await.unwrap();
1882        for id in ["a", "b", "c", "e"] {
1883            conn.execute(
1884                "INSERT INTO concepts (id, title, valid_from, recorded_at) \
1885                 VALUES (?1, 'n', ?2, ?2)",
1886                libsql::params![id, EPOCH],
1887            )
1888            .await
1889            .unwrap();
1890        }
1891        // `a → b` twice, so the older row is superseded and archivable;
1892        // `a → c` closed before the cutoff, so the second arm takes it;
1893        // `a → e` open and never superseded, so nothing can touch it.
1894        for (target, valid_to, recorded_at) in [
1895            ("b", OPEN, EPOCH),
1896            ("b", OPEN, "1970-01-01T01:00:00.000000Z"),
1897            ("c", CLOSED, EPOCH),
1898            ("e", OPEN, EPOCH),
1899        ] {
1900            conn.execute(
1901                "INSERT INTO links (source_id, target_id, edge_type, valid_from, valid_to, \
1902                 weight, properties, recorded_at) VALUES ('a', ?1, 'LINKS', ?2, ?3, 1.0, '{}', ?4)",
1903                libsql::params![target, EPOCH, valid_to, recorded_at],
1904            )
1905            .await
1906            .unwrap();
1907        }
1908        conn
1909    }
1910
1911    /// **The key set is the keys the delete will disturb, and no others**
1912    /// (0.15.3, [D-245](../../docs/architecture/s13-decision-register.md#d-245)).
1913    ///
1914    /// A key set that is too wide leaves the projection *correct* — it
1915    /// re-derives untouched partitions to the rows they already held — so
1916    /// every equality test in `archive_projection_tests` passes with it, and
1917    /// the whole point of the release does not. This is the assertion those
1918    /// tests cannot make: what the repair is allowed to look at. `a → e` is
1919    /// the row that must not appear, and the count is pinned as well, because
1920    /// a key set that is too *narrow* is a correctness bug the equality tests
1921    /// would catch but this one names.
1922    #[tokio::test]
1923    async fn the_collected_keys_are_only_the_ones_the_delete_disturbs() {
1924        let conn = seeded().await;
1925        let tx = conn
1926            .transaction_with_behavior(TransactionBehavior::Immediate)
1927            .await
1928            .unwrap();
1929        collect_archived_keys(
1930            &tx,
1931            LINKS_ARCHIVABLE,
1932            libsql::named_params! {":cutoff": CUTOFF},
1933        )
1934        .await
1935        .unwrap();
1936
1937        let mut rows = tx
1938            .query(
1939                &format!("SELECT target_id FROM {ARCHIVED_KEYS} ORDER BY target_id"),
1940                (),
1941            )
1942            .await
1943            .unwrap();
1944        let mut targets = Vec::new();
1945        while let Some(row) = rows.next().await.unwrap() {
1946            targets.push(row.get::<String>(0).unwrap());
1947        }
1948
1949        assert_eq!(
1950            targets,
1951            vec!["b".to_string(), "c".to_string()],
1952            "a → e is untouched by this cutoff and the repair has no business \
1953             re-deriving it; a key set this wide is the full rebuild wearing \
1954             the keyed repair's name"
1955        );
1956    }
1957
1958    /// Two rows at one key are one key, and the repair is per key.
1959    ///
1960    /// `DISTINCT` rather than a bare `SELECT`: `a → b` has two archivable-or-
1961    /// not rows and the repair re-derives its partition once. Without it the
1962    /// `IN` subquery still gives the right answer and the key table grows with
1963    /// the *rows* archived rather than the keys, which is the same cost defect
1964    /// one level down.
1965    #[tokio::test]
1966    async fn a_key_asserted_twice_is_collected_once() {
1967        let conn = seeded().await;
1968        let tx = conn
1969            .transaction_with_behavior(TransactionBehavior::Immediate)
1970            .await
1971            .unwrap();
1972        collect_archived_keys(&tx, "1 = 1", ()).await.unwrap();
1973
1974        let n: i64 = tx
1975            .query(&format!("SELECT COUNT(*) FROM {ARCHIVED_KEYS}"), ())
1976            .await
1977            .unwrap()
1978            .next()
1979            .await
1980            .unwrap()
1981            .unwrap()
1982            .get(0)
1983            .unwrap();
1984        assert_eq!(n, 3, "four rows at three keys collected as three keys");
1985    }
1986}