Skip to main content

macrame/
connection.rs

1use std::path::{Path, PathBuf};
2use std::sync::Arc;
3use tokio::sync::{mpsc, oneshot};
4
5use crate::error::{classify, DbError, Result, WriteOp};
6use crate::graph::edge::EdgeAssertion;
7use crate::integrity::{rebuild_current, RebuildReport};
8use crate::schema::migrations;
9use crate::temporal::archive::{archive, ArchiveReport};
10use crate::temporal::interval::Interval;
11use crate::temporal::snapshot::{self, SnapshotCadence};
12use crate::util::clock::{Clock, SystemClock};
13use crate::util::timestamp;
14use crate::vector::ModelName;
15
16/// Rows per chunk on the background write paths (§5.1.5, D-011, D-014, D-058).
17///
18/// The Write Actor holds the sole write connection, so a single large statement
19/// blocks every other writer for its duration. Chunking bounds that stall; the
20/// cost is that a bulk import is *not* atomic across chunks, which is why it is
21/// a separate command from [`HighPriCommand::WriteBulkAtomic`] rather than a
22/// tuning parameter on it.
23///
24/// # Why these are four constants and not one
25///
26/// Through 0.5.5 this was a single `CHUNK_ROWS = 1000` for all four bulk paths.
27/// The golden rule it was meant to serve is a bound on *duration* — a background
28/// chunk must commit fast enough that an interactive write queued behind it is
29/// not made to wait — and one row count cannot express one duration across paths
30/// whose measured per-row costs differ by 60× (D-058). At 1,000 rows the four
31/// paths took 3.5 ms, 24 ms, 89 ms and 143 ms: the same constant, four answers,
32/// three of them far outside the bound.
33///
34/// Each size below is derived from `benches/budgets.rs`'s `chunk_scaling`
35/// sweep against [`CHUNK_BUDGET`], then verified by measuring that size directly.
36/// They are *measurements of this machine*, not universal constants — D-055's
37/// reasoning about reference hardware applies here too, and re-deriving them on
38/// materially different storage is a `cargo bench` away.
39///
40/// # Sized for the tail, not the median
41///
42/// The first derivation solved `f + c·n = 3 ms` exactly and produced sizes whose
43/// *median* commit was 2.93 ms and whose upper estimate was 2.96 — inside the
44/// bound as reported and outside it for any chunk slower than typical. A latency
45/// bound is a statement about the chunk an unlucky interactive write actually
46/// queues behind, so these solve for ≈2.5 ms instead, leaving the remainder as
47/// headroom for the tail. That costs a few percent of throughput on the two
48/// linear paths and nothing on the two superlinear ones.
49///
50/// As measured by `chunk_budget`, each at its own size: edges **2.39 ms**,
51/// concepts **2.35 ms**, annotations **2.36 ms**, embeddings **2.06 ms**, no
52/// upper estimate above 2.42.
53///
54/// # Known limitation: these are empty-database figures
55///
56/// `chunk_budget` seeds concepts and starts with **no links and no vectors**,
57/// and D-059 established that per-row cost on the edge and embedding paths grows
58/// with the size of the structure being written, not with the chunk. The same
59/// 90-edge chunk takes 47.7 ms against an 8,000-edge hub. So the bound is met as
60/// measured and *not* met on a large database, most of that gap being the schema
61/// defect D-059 documents. Re-deriving these against a realistic fixture needs a
62/// decision about what "realistic" is, which is why it has not been done
63/// silently.
64pub mod chunk_rows {
65    /// Edge assertions (`bulk_import`).
66    ///
67    /// Per-row cost on this path rises with the size of `links_current`, not
68    /// with the chunk (D-059) — so cutting the chunk buys latency and costs
69    /// throughput, ~11% for 1,000 edges. An earlier version of this comment
70    /// claimed it was 3.3× *faster*; that came from multiplying eleven copies of
71    /// a chunk measured into an empty database.
72    ///
73    /// **This size does not meet the 3 ms bound on a large database.** 90 edges
74    /// into an 8,000-edge hub take 47.7 ms, because `trg_links_single_open`'s
75    /// `EXISTS` is served by `idx_lc_traversal_cover` with only `source_id`
76    /// bound and therefore scans the whole out-degree. That is a schema defect
77    /// with a proven fix, recorded in D-059 and not applied here.
78    pub const EDGES: usize = 90;
79
80    /// Concept upserts (`write_concepts`).
81    ///
82    /// Linear at ~23 µs per row, so unlike [`EDGES`] this size *is* a genuine
83    /// throughput sacrifice: 1,000-row chunks ran at 23.6 µs per row against
84    /// ~35 µs here. Paid deliberately — a 1,000-row chunk takes 24 ms, eight
85    /// times the bound.
86    pub const CONCEPTS: usize = 70;
87
88    /// Analytics annotations (`write_analytics_annotations`).
89    ///
90    /// The one path where the old constant was nearly right, and the only bulk
91    /// table with no triggers at all: ~2.5 µs per row, linear, so the bound buys
92    /// a large chunk. 1,000 rows would be 3.5 ms — over, but only just.
93    pub const ANNOTATIONS: usize = 600;
94
95    /// Embedding vectors (`upsert_embeddings`).
96    ///
97    /// The smallest by a wide margin, because DiskANN index maintenance makes an
98    /// embedding the most expensive row in the system. That cost grows with the
99    /// **corpus**, not the chunk (D-059): a fixed 30-vector chunk costs 49 µs per
100    /// vector into an empty corpus and 224 µs into an 8,000-vector one. Graph
101    /// insertion getting dearer as the graph grows is what DiskANN is, so unlike
102    /// [`EDGES`] there is nothing here to fix — but it does mean this size buys
103    /// latency at some throughput, not for free.
104    pub const EMBEDDINGS: usize = 30;
105}
106
107/// The latency bound [`chunk_rows`] is derived from (§5.1.5, D-058).
108///
109/// This is the golden rule's actual content. §9 has carried it as a row count
110/// with a duration attached — "chunk commit, 500 rows ≤ 3 ms" — which reads as
111/// two requirements and is one: the duration is the requirement, and the row
112/// count is whatever satisfies it on a given path and machine.
113///
114/// 3 ms is §9's number, kept rather than renegotiated. What it buys, end to end:
115/// an interactive assertion arriving at the worst possible moment waits for the
116/// chunk in flight (≤ 3 ms, because the SQLite write lock is not preemptible —
117/// see [`HighPriCommand`]) and then runs its own write (≤ 5 ms, §9), so ≤ 8 ms
118/// worst case. That fits inside a 60 Hz frame with room, which is the standard
119/// this bound is ultimately answerable to.
120///
121/// # Three operations are exempt, and the exemption is a contract, not an oversight
122///
123/// This was recorded in three separate rustdoc notes and nowhere near the bound
124/// itself, which is where a reader looks for its scope (§8.6). Stated here, with
125/// Wave 3's measurements:
126///
127/// | Path | Bound | Why it cannot be chunked |
128/// |---|---|---|
129/// | [`Database::write_bulk_atomic`] | none — caller-sized `Vec` | D-014: the batch is *one act* under one stamp. Splitting it is the thing the method exists not to do |
130/// | [`Database::archive`] | measured **26.8 ms** for 2,000 archivable edges; see [`Database::archive_windowed`] | D-012: copy-then-delete must be atomic, or a crash between the phases duplicates or loses rows |
131/// | `rebuild_current` | measured **24.6 / 104 / 318 ms** at 4K / 16K / 40K rows in `links` (was "~50 s per 10M edges", which nothing had measured) | D-023: the window between `DELETE` and `INSERT` is the whole of current belief; a reader landing in it sees a graph with no edges and no error |
132///
133/// The `archive` figure is end-to-end through this method, so it **includes**
134/// the re-derivation `archive()` runs inside its transaction — but it does not
135/// attribute it, and until D-077 more than half of that re-derivation was an
136/// audit comparing `links_current` against the query that had just filled it.
137/// Note also which variable that cost scales with: `rebuild_within` reprojects
138/// **all of `links`**, so the archive's repair term grows with the *surviving*
139/// table and not with the batch being archived. A budget stated per "100K closed
140/// intervals" ([§9](../docs/architecture/s6-s10-flows-to-dependencies.md)) is
141/// therefore parameterised on the wrong quantity.
142///
143/// All three are atomic **by contract**, which is why "cap the batch" and "add a
144/// third tier" were both considered and neither was taken: capping breaks the
145/// guarantee the operation exists to provide, and a third tier changes which
146/// caller waits without changing how long the lock is held. What was wrong was
147/// never the exemption — it was that the bound was stated as though it had none.
148///
149/// A caller who needs the latency bound and not the atomicity has
150/// [`Database::bulk_import`], which is the same write chunked at
151/// [`chunk_rows::EDGES`] and explicitly *not* atomic overall (D-011).
152///
153/// # One of the three is no longer unbounded (T1.1, D-080)
154///
155/// `archive` was the worst of them, because its hold is a function of *how long
156/// since the last archive* rather than of anything the caller chose.
157/// [`Database::archive_windowed`] runs the same work as N sessions, each
158/// atomic, each its own actor turn. Measured on an 8,000-key fixture with four
159/// generations of superseded history: the longest single hold falls from
160/// **3.3 s to 0.77 s** at one-hour windows, for total wall time that is flat
161/// within this cycle's noise.
162///
163/// The same measurement at 2,000 keys goes the other way — the hold falls
164/// 260 ms → 117 ms while total time rises 260 ms → 671 ms — so windowing is a
165/// trade and not a free improvement. It pays when the backlog is large, which
166/// is when the unwindowed hold is a problem in the first place. `archive` is
167/// kept, not deprecated, for exactly that reason.
168pub const CHUNK_BUDGET: std::time::Duration = std::time::Duration::from_millis(3);
169
170/// Predicted hold above which [`Database::write_bulk_atomic`] warns (T1.3).
171///
172/// 250 ms is fifteen frames at 60 Hz: not a hitch, a visible freeze. It is well
173/// above [`CHUNK_BUDGET`] on purpose — this path is exempt from that bound by
174/// contract, so warning at 3 ms would fire on batches that are working exactly
175/// as designed and train the reader to filter the message out.
176pub const BULK_ATOMIC_WARN_HOLD: std::time::Duration = std::time::Duration::from_millis(250);
177
178/// Roughly how long [`Database::write_bulk_atomic`] will hold the actor for
179/// this batch (T1.3, D-081).
180///
181/// # Three terms, because the cost is neither linear nor a function of size
182///
183/// T1.3 asks for "rows × measured per-row cost". That model is wrong twice over,
184/// and both corrections came out of measuring it.
185///
186/// First, the cost is not linear. `write_edges_atomic` opens with
187/// `reject_overlaps_within`, which compares **every pair** in the batch before a
188/// row is written. Second — and this is the one that matters — the quadratic
189/// term's constant depends on the batch's *shape*, not its size. The pairwise
190/// loop starts with an early `continue` on mismatched `(source, target,
191/// edge_type)`; pairs that share all three fall through to `Interval::new` and
192/// `overlaps`, which is **sixteen times** dearer per pair.
193///
194/// ```text
195/// hold ≈ 73 µs · rows  +  5.5 ns · mismatched pairs  +  86 ns · matching pairs
196/// ```
197///
198/// Two batches of 20,000 edges, measured on the same machine: one fanning out to
199/// distinct targets holds the actor for **2.5 s**, and one asserting 20,000
200/// corrections to a single relationship's history holds it for **18.6 s**. A
201/// size-only model is off by 7× between those two, in the direction that
202/// matters — it under-predicts the bad case. So this counts the matching pairs
203/// rather than guessing, with one `HashMap` pass over the batch. That pass is
204/// O(rows) against an operation about to spend milliseconds per row.
205///
206/// # What this is calibrated against, and where it will be wrong
207///
208/// libSQL 0.9.30, one machine, best of three, over 100–20,000 rows in both
209/// shapes; within 5% across that range except below ~500 rows, where fixed costs
210/// dominate and it over-predicts by 3× — harmless, since nothing that small can
211/// approach [`BULK_ATOMIC_WARN_HOLD`].
212///
213/// It is machine-specific and says nothing about disk. It exists to turn
214/// "uncapped" into an order of magnitude a caller can act on — the difference
215/// between 30 ms and 18 s — and should not be read more precisely than that.
216/// `examples/bulk_atomic_diag.rs` prints predicted against measured, so the
217/// model's drift is visible rather than assumed.
218pub fn estimated_bulk_hold(edges: &[EdgeAssertion]) -> std::time::Duration {
219    let rows = edges.len() as u64;
220    let all_pairs = rows.saturating_mul(rows.saturating_sub(1)) / 2;
221
222    // Pairs sharing all three key columns, which is exactly the set that reaches
223    // the guard's expensive path. Grouped rather than sorted: the batch is
224    // borrowed, and sorting would either clone it or reorder the caller's data.
225    let mut groups: std::collections::HashMap<(&str, &str, &str), u64> =
226        std::collections::HashMap::new();
227    for e in edges {
228        *groups
229            .entry((&e.source, &e.target, &e.edge_type))
230            .or_insert(0) += 1;
231    }
232    let matching: u64 = groups.values().map(|&g| g * (g - 1) / 2).sum();
233    let mismatched = all_pairs - matching;
234
235    // Nanoseconds throughout, saturating: a caller who passes a batch large
236    // enough to overflow this has a problem the arithmetic cannot express, and
237    // saturating to ~584 years still crosses every threshold above.
238    std::time::Duration::from_nanos(
239        (73_000u64.saturating_mul(rows))
240            .saturating_add(mismatched.saturating_mul(11) / 2)
241            .saturating_add(matching.saturating_mul(86)),
242    )
243}
244
245/// Most sessions [`Database::archive_windowed`] will run for one call (T1.1).
246///
247/// A limit exists because the session count is a function of *transaction-time
248/// span divided by window*, and both come from the caller — a one-second window
249/// over a decade of history is ten million actor turns, each opening a
250/// transaction and writing a horizon row. That is not a slow archive, it is a
251/// caller who meant something else.
252///
253/// 4,096 is chosen against the operation it bounds rather than against a clock:
254/// at the measured 26.8 ms for a session with work in it, a full run of this
255/// many is about two minutes of background writing, and the whole point of
256/// windowing is that those two minutes are interruptible. It is a refusal
257/// rather than a clamp — see [`DbError::ArchiveWindow`] for why.
258pub const MAX_ARCHIVE_SESSIONS: usize = 4_096;
259
260/// A concept assertion: the payload of an upsert.
261#[derive(Debug, Clone, PartialEq)]
262pub struct ConceptUpsert {
263    pub id: String,
264    pub title: String,
265    pub content: String,
266    pub embedding_model: Option<String>,
267    pub valid_from: String,
268    pub valid_to: String,
269    pub retired: bool,
270}
271
272impl ConceptUpsert {
273    pub fn new(id: impl Into<String>, title: impl Into<String>) -> Self {
274        Self {
275            id: id.into(),
276            title: title.into(),
277            content: String::new(),
278            embedding_model: None,
279            valid_from: String::new(),
280            valid_to: timestamp::OPEN_SENTINEL.to_string(),
281            retired: false,
282        }
283    }
284
285    pub fn content(mut self, content: impl Into<String>) -> Self {
286        self.content = content.into();
287        self
288    }
289
290    pub fn embedding_model(mut self, model: impl Into<String>) -> Self {
291        self.embedding_model = Some(model.into());
292        self
293    }
294
295    pub fn valid_from(mut self, ts: impl Into<String>) -> Self {
296        self.valid_from = ts.into();
297        self
298    }
299
300    pub fn valid_to(mut self, ts: impl Into<String>) -> Self {
301        self.valid_to = ts.into();
302        self
303    }
304
305    pub fn retired(mut self, retired: bool) -> Self {
306        self.retired = retired;
307        self
308    }
309
310    /// Put the timestamps in canonical form (D-029) before they cross the channel.
311    pub fn normalized(mut self) -> Result<Self> {
312        crate::util::ids::validate_id(&self.id)?;
313        self.valid_from = timestamp::normalize(&self.valid_from)?;
314        self.valid_to = timestamp::normalize(&self.valid_to)?;
315        Ok(self)
316    }
317}
318
319/// One derived analytics result for one concept (§5.4, D-041).
320///
321/// Not a `ConceptUpsert`. The distinction is the whole of D-041: a concept
322/// upsert is a statement about the world and belongs in the ledger, while an
323/// annotation is a function of an algorithm applied to a graph and belongs in
324/// `analytics_annotations`, which carries no log trigger. Writing one as the
325/// other overwrote the concept's `content` with the label and recorded every
326/// analytics rerun as a fresh version of the world.
327#[derive(Debug, Clone, PartialEq, Eq)]
328pub struct Annotation {
329    pub concept_id: String,
330    /// Namespaced by convention, e.g. `louvain.community`, `kcore.shell`.
331    pub label: String,
332    /// JSON-encoded payload. Opaque to this crate.
333    pub value: String,
334}
335
336impl Annotation {
337    pub fn new(
338        concept_id: impl Into<String>,
339        label: impl Into<String>,
340        value: impl Into<String>,
341    ) -> Self {
342        Self {
343            concept_id: concept_id.into(),
344            label: label.into(),
345            value: value.into(),
346        }
347    }
348}
349
350/// Commands sent to the Write Actor on the high-priority channel (UI-driven work).
351pub enum HighPriCommand {
352    AssertEdge {
353        edge: EdgeAssertion,
354        responder: oneshot::Sender<Result<()>>,
355    },
356    RetireEdge {
357        source: String,
358        target: String,
359        edge_type: String,
360        valid_from: String,
361        valid_to: String,
362        responder: oneshot::Sender<Result<()>>,
363    },
364    UpsertConcept {
365        concept: ConceptUpsert,
366        responder: oneshot::Sender<Result<()>>,
367    },
368    WriteBulkAtomic {
369        edges: Vec<EdgeAssertion>,
370        responder: oneshot::Sender<Result<usize>>,
371    },
372    RebuildCurrent {
373        responder: oneshot::Sender<Result<RebuildReport>>,
374    },
375    /// Create a model's embedding table and its DiskANN index (D-037, D-048).
376    ///
377    /// High priority despite being setup work: it is one small transaction, and
378    /// every embedding write for the model blocks on it, so queueing it behind a
379    /// bulk job would stall the thing it gates.
380    RegisterModel {
381        model: ModelName,
382        dim: usize,
383        responder: oneshot::Sender<Result<()>>,
384    },
385    Shutdown {
386        responder: oneshot::Sender<Result<()>>,
387    },
388}
389
390/// Commands sent to the Write Actor on the low-priority channel (background work).
391pub enum LowPriCommand {
392    /// One chunk of **concepts** — a ledger write, logged and versioned.
393    WriteConceptsChunk {
394        chunk: Vec<ConceptUpsert>,
395        responder: oneshot::Sender<Result<usize>>,
396    },
397    /// One chunk of **derived annotations** — off-ledger, no log trigger (D-041).
398    ///
399    /// The pair is named apart deliberately: this variant was `WriteAnalyticsChunk`
400    /// beside a `WriteAnnotationsChunk` that carried concepts, which is the
401    /// crossing D-075 undid.
402    WriteAnalyticsChunk {
403        chunk: Vec<Annotation>,
404        responder: oneshot::Sender<Result<usize>>,
405    },
406    /// One chunk of vectors for one model (§5.9, D-048).
407    ///
408    /// Low priority: embedding is bulk derived work and must never preempt an
409    /// interactive assertion.
410    UpsertEmbeddingChunk {
411        model: ModelName,
412        chunk: Vec<(String, Vec<f32>)>,
413        responder: oneshot::Sender<Result<usize>>,
414    },
415    BulkImportChunk {
416        chunk: Vec<EdgeAssertion>,
417        responder: oneshot::Sender<Result<usize>>,
418    },
419    Archive {
420        cutoff: String,
421        archive_path: PathBuf,
422        responder: oneshot::Sender<Result<ArchiveReport>>,
423    },
424    /// Reconstruct the FTS index from `concepts` (§5.9, D-036, D-051).
425    ///
426    /// Low priority: it is maintenance on a derivative table, and a search index
427    /// that is a few seconds stale is a smaller cost than an interactive write
428    /// that waits behind a full reindex.
429    RebuildFts {
430        responder: oneshot::Sender<Result<()>>,
431    },
432    /// One step of a chunked shadow rebuild (§5.8, T1.2, D-082).
433    ///
434    /// Low priority, and one command per step rather than one per rebuild: the
435    /// whole value of building beside the live table is that the actor returns
436    /// here between chunks. See [`Database::rebuild_current_chunked`].
437    ShadowRebuild {
438        step: crate::integrity::ShadowStep,
439        responder: oneshot::Sender<Result<crate::integrity::ShadowOutcome>>,
440    },
441}
442
443enum LoopCtl {
444    Continue,
445    Break,
446}
447
448/// Primary database handle for Macrame bitemporal ledger.
449pub struct Database {
450    db: libsql::Database,
451    /// The file this handle opened, kept so [`Database::diagnostic_conn`] can
452    /// open it again under different flags (T5.1, D-091). `archive_path` and
453    /// `snapshots_dir` are derived from it and were previously the only trace
454    /// of it on the struct.
455    path: PathBuf,
456    read_conn: libsql::Connection,
457    highpri_tx: mpsc::Sender<HighPriCommand>,
458    lowpri_tx: mpsc::Sender<LowPriCommand>,
459    clock: Arc<dyn Clock>,
460    archive_path: PathBuf,
461    snapshots_dir: PathBuf,
462    schema_version: u32,
463    writer: Option<tokio::task::JoinHandle<Result<()>>>,
464    /// Stops the snapshot cadence. Dropping it stops the task too, which is what
465    /// keeps a `Database` that is dropped rather than closed from leaving a task
466    /// running against a connection whose database is going away.
467    cadence_stop: Option<tokio::sync::watch::Sender<bool>>,
468    cadence: Option<tokio::task::JoinHandle<()>>,
469    /// Set by [`Database::close`]. Read only by [`Drop`], which warns when it is
470    /// still false — see that impl for why the omission is worth a warning.
471    closed: bool,
472    /// Shared with the actor (T1.4, T1.2). Held here rather than behind
473    /// `#[cfg(feature = "metrics")]` so `open_inner` has one shape; with the
474    /// feature off the metrics half is a zero-sized type and only
475    /// [`Database::metrics`] is gated — which is also why the field is unread in
476    /// the default build: the actor holds the other `Arc` and does the writing.
477    #[cfg_attr(not(feature = "metrics"), allow(dead_code))]
478    shared: Arc<ActorShared>,
479}
480
481impl Database {
482    /// Open a database file at `path`, configuring pragmas, running migrations, and spawning the Write Actor.
483    ///
484    /// The snapshot cadence runs with [`SnapshotCadence::default`]. Use
485    /// [`Database::open_with_cadence`] to tune or disable it.
486    pub async fn open(path: impl AsRef<Path>) -> Result<Self> {
487        Self::open_with_cadence(path, Some(SnapshotCadence::default())).await
488    }
489
490    /// Open with an explicit snapshot cadence, or `None` to run without one
491    /// (§5.5, D-053).
492    ///
493    /// `None` restores the pre-0.5.5 behaviour, where `close()` is the only
494    /// thing that ever writes an anchor. That is the right setting for a
495    /// short-lived process that will not accumulate a delta worth bounding, and
496    /// for tests that assert on the contents of the snapshot directory.
497    pub async fn open_with_cadence(
498        path: impl AsRef<Path>,
499        cadence: Option<SnapshotCadence>,
500    ) -> Result<Self> {
501        Self::open_inner(path.as_ref(), cadence, None).await
502    }
503
504    /// Open with an injected clock (§5.1.2, **defect K**, D-062).
505    ///
506    /// The reason this exists is testing: `recorded_at` is the transaction-time
507    /// axis, and until now every test that wanted to assert on one had to either
508    /// avoid it or drive a raw connection, because `open()` hardcoded
509    /// [`SystemClock`]. `FakeClock` has been public and constructed in the test
510    /// harness since 0.5.2 with nothing to inject it into — the compiler warned
511    /// about the dead field on every build for three releases.
512    ///
513    /// **The clock is floored against the database before the actor starts.**
514    /// [`Clock::raise_floor`] is called with the newest `recorded_at` in the
515    /// ledger, so an injected clock cannot issue a stamp below what is already
516    /// stored — which would abort the next concept write on
517    /// `trg_concepts_monotonic_ra` rather than merely being odd. This is the
518    /// step whose absence kept the defect open: the obvious implementation
519    /// (take an `Arc<dyn Clock>`, use it) produces a `Database` that fails on
520    /// its first write against any non-empty file.
521    ///
522    /// On a fresh database there is no floor, so an injected `FakeClock` issues
523    /// exactly the stamps it was given.
524    pub async fn open_with_clock(
525        path: impl AsRef<Path>,
526        cadence: Option<SnapshotCadence>,
527        clock: Arc<dyn Clock>,
528    ) -> Result<Self> {
529        Self::open_inner(path.as_ref(), cadence, Some(clock)).await
530    }
531
532    async fn open_inner(
533        path: &Path,
534        cadence: Option<SnapshotCadence>,
535        injected: Option<Arc<dyn Clock>>,
536    ) -> Result<Self> {
537        let db = libsql::Builder::new_local(path).build().await?;
538        let write_conn = configure(db.connect()?).await?;
539        let read_conn = configure(db.connect()?).await?;
540
541        // PRAGMA query_only = ON on reader connection (§5.1.2)
542        read_conn.execute("PRAGMA query_only = ON", ()).await?;
543
544        let migration = migrations::run(&write_conn).await?;
545
546        let (highpri_tx, highpri_rx) = mpsc::channel(256);
547        let (lowpri_tx, lowpri_rx) = mpsc::channel(64);
548
549        // Floored after `migrations::run`, so the tables the floor is read from
550        // are guaranteed to exist.
551        let clock: Arc<dyn Clock> = match injected {
552            Some(clock) => {
553                if let Some(floor) = crate::util::clock::recorded_at_floor(&read_conn).await? {
554                    clock.raise_floor(floor);
555                }
556                clock
557            }
558            None => Arc::new(SystemClock::new(&read_conn).await?),
559        };
560        let shared = Arc::new(ActorShared::default());
561        let writer = tokio::spawn(run_writer_actor(
562            write_conn,
563            Arc::clone(&clock),
564            highpri_rx,
565            lowpri_rx,
566            Arc::clone(&shared),
567        ));
568
569        let archive_path = derive_archive_path(path);
570        let snapshots_dir = derive_snapshots_dir(path);
571
572        // **The cadence gets its own connection (Wave 4.1).** It used to share
573        // `read_conn`, on the reasoning that `libsql::Connection` is an
574        // Arc-backed handle and R15 makes every extra local connection a cost worth
575        // not paying for nothing. The cost it was not paying for turned out to be
576        // real: `reconstruct` brackets a fold with `ATTACH cold … DETACH cold`,
577        // that region is per-connection state, and it is not synchronised. Two
578        // folds on one connection can therefore interleave so that one DETACHes
579        // the handle the other is mid-fold on.
580        //
581        // Recorded in §8.5 as a hazard rather than a defect because it **did not
582        // reproduce**: 200 concurrent reconstructions against a 1 ms cadence with
583        // an archive present produced zero errors, since the cadence anchors at
584        // `MAX(recorded_at)` and so almost always takes the hot path. Narrow, and
585        // real — a write landing between `log_head` and the fold opens it.
586        //
587        // Separate connections remove the interleaving rather than ordering it,
588        // which is why this is preferred to a mutex around the region: there is
589        // no shared state left to race on, and nothing to remember to hold. The
590        // R15 objection does not apply — that fault is about *concurrent* opens,
591        // and this is one more sequential open during `open()`.
592        let (cadence_stop, cadence) = match cadence {
593            Some(cadence) => {
594                let cadence_conn = configure(db.connect()?).await?;
595                cadence_conn.execute("PRAGMA query_only = ON", ()).await?;
596                let (tx, rx) = tokio::sync::watch::channel(false);
597                let handle = tokio::spawn(snapshot::run_cadence(
598                    cadence_conn,
599                    snapshots_dir.clone(),
600                    archive_path.clone(),
601                    cadence,
602                    rx,
603                ));
604                (Some(tx), Some(handle))
605            }
606            None => (None, None),
607        };
608
609        let handle = Self {
610            db,
611            path: path.to_path_buf(),
612            read_conn,
613            highpri_tx,
614            lowpri_tx,
615            clock,
616            archive_path,
617            snapshots_dir,
618            schema_version: migrations::current_version(),
619            writer: Some(writer),
620            cadence_stop,
621            cadence,
622            closed: false,
623            shared,
624        };
625
626        // **Re-anchor after a migration (Wave 4.4).**
627        //
628        // D-043 makes a `SCHEMA_VERSION` bump invalidate every snapshot on disk,
629        // which is correct — a snapshot is a serialised `MaterializedState` and a
630        // schema change can change what that means. What was missing is the other
631        // half: nothing wrote a replacement, so the first `reconstruct` after an
632        // upgrade skipped every file as incompatible and folded from genesis. On
633        // a database with a large log that is the difference between reading one
634        // snapshot and folding the whole history, and the only trace was a
635        // `warn!` per skipped file.
636        //
637        // Written here rather than left to the cadence because the cadence fires
638        // on log *growth* (D-053): an upgraded database that is then read but not
639        // written would never re-anchor at all.
640        //
641        // Failure is logged, not returned. A missing anchor costs time and no
642        // information — snapshots are derivative under Doctrine VI — so refusing
643        // to open a database because its optimisation could not be rebuilt would
644        // trade a real capability for a performance one.
645        //
646        // Gated on the cadence being enabled, as well as on an actual upgrade:
647        // `open_with_cadence(None)` means *this handle writes no snapshots except
648        // at close()*, and a one-off write at open would contradict that for a
649        // caller who asked for the quiet mode precisely to control when files
650        // appear. They still get an anchor from `close()`.
651        if migration.upgraded() && handle.cadence.is_some() {
652            let ts = handle.clock.now();
653            let archive = handle
654                .archive_path
655                .exists()
656                .then_some(handle.archive_path.as_path());
657            match snapshot::write_final(&handle.read_conn, &handle.snapshots_dir, &ts, archive)
658                .await
659            {
660                Ok(path) => tracing::info!(
661                    "schema moved v{} -> v{}; re-anchored snapshots at {:?}",
662                    migration.from,
663                    migration.to,
664                    path
665                ),
666                Err(e) => tracing::warn!(
667                    "schema moved v{} -> v{} but the re-anchor failed: {e}. \
668                     Reconstruction stays correct and folds from genesis until the \
669                     cadence writes one.",
670                    migration.from,
671                    migration.to
672                ),
673            }
674        }
675
676        Ok(handle)
677    }
678
679    /// Read connection handle for queries, traversals, and folds.
680    pub fn read_conn(&self) -> &libsql::Connection {
681        &self.read_conn
682    }
683
684    /// The file this handle opened.
685    pub fn path(&self) -> &Path {
686        &self.path
687    }
688
689    /// A **new, independently owned, OS-level read-only** connection to this
690    /// database, for diagnostics (§4.7, T5.1, D-091).
691    ///
692    /// # Why this exists when `read_conn()` already does
693    ///
694    /// Two different things, and the difference is the point:
695    ///
696    /// * `read_conn()` returns a shared `&Connection` carrying
697    ///   `PRAGMA query_only = ON`. That pragma is **per-connection and
698    ///   reversible by its holder in one statement**, so it is a guardrail
699    ///   against accident, not a capability boundary. And because the reference
700    ///   is shared, a caller who runs a long reporting query on it is competing
701    ///   with every traversal and fold in the process.
702    /// * This returns a connection opened with `SQLITE_OPEN_READ_ONLY`, which is
703    ///   enforced by the engine below the pragma layer, and it is the caller's
704    ///   own.
705    ///
706    /// **Measured on libSQL 0.9.30 rather than assumed**
707    /// (`examples/readonly_open_probe.rs`), against a live WAL database with the
708    /// write actor running:
709    ///
710    /// | | `read_conn()` | `diagnostic_conn()` |
711    /// |---|---|---|
712    /// | `SELECT`, `EXPLAIN QUERY PLAN` | allowed | allowed |
713    /// | `INSERT` | refused | refused |
714    /// | `PRAGMA query_only = OFF` | **allowed** | allowed |
715    /// | `INSERT` after that | **allowed** | **refused** |
716    ///
717    /// The third and fourth rows are the whole difference: turning the pragma
718    /// off restores writes on `read_conn()` and does not here. That is what
719    /// "boundary rather than guardrail" means, and it is now a number rather
720    /// than a claim.
721    ///
722    /// # One way this is *more* permissive, which is worth knowing
723    ///
724    /// `CREATE TEMP TABLE` **succeeds** here and is refused by `read_conn()`.
725    /// Temp tables live in a separate temporary database that is writable
726    /// regardless of how the main one was opened, whereas `query_only` refuses
727    /// them outright — which is the mechanism [D-050] measured when it removed
728    /// `TwoPhaseTempTable` for returning `SQLITE_READONLY (8)` on the read
729    /// connection. So the stronger boundary is not uniformly stronger, and a
730    /// strategy that needs a temp table has a connection it could run on. That
731    /// is recorded, not acted on: D-050 removed the strategy for two reasons and
732    /// this addresses one of them.
733    ///
734    /// # Errors
735    ///
736    /// The file must already exist. `SQLITE_OPEN_READ_ONLY` drops
737    /// `SQLITE_OPEN_CREATE` with it, so a missing file is `SQLITE_CANTOPEN`
738    /// rather than a fresh empty database — which is the right failure, and is
739    /// surfaced as a typed error rather than as libSQL's error 14.
740    pub async fn diagnostic_conn(&self) -> Result<libsql::Connection> {
741        let fail = |reason: String| DbError::DiagnosticConn {
742            path: self.path.display().to_string(),
743            reason,
744        };
745        if !self.path.exists() {
746            return Err(fail(
747                "the file does not exist, and a read-only open cannot create it".to_string(),
748            ));
749        }
750        let db = libsql::Builder::new_local(&self.path)
751            .flags(libsql::OpenFlags::SQLITE_OPEN_READ_ONLY)
752            .build()
753            .await
754            .map_err(|e| fail(e.to_string()))?;
755        db.connect().map_err(|e| fail(e.to_string()))
756    }
757
758    /// Cross-check the snapshot chain against a fold from genesis (§5.5, T5.3,
759    /// D-092).
760    ///
761    /// `write_final` composes onto the previous snapshot, so snapshot *n* is
762    /// derived from snapshot *n−1* and nothing in the chain ever folds the whole
763    /// log. An error at any link propagates forward forever and every read
764    /// agrees with it, because every read descends from it. This is the check
765    /// that would notice.
766    ///
767    /// # When to run it
768    ///
769    /// **Not on a schedule this crate chooses.** A genesis fold is precisely the
770    /// cost snapshots exist to avoid, so running it periodically by default
771    /// would give every application the bill snapshots were bought to remove —
772    /// on a database whose log is large enough for snapshots to matter, which is
773    /// the only kind where this is worth doing. The plan calls it a scheduling
774    /// problem and it is the caller's schedule: an idle period, a nightly job,
775    /// or once per *N* anchors, chosen against a log size this crate cannot see.
776    ///
777    /// The cadence is deliberately left alone for the same reason — it runs on a
778    /// connection shared with nothing and a fold there would compete with
779    /// interactive reads at a moment nobody chose.
780    ///
781    /// # It reports; it does not repair
782    ///
783    /// A divergence means the snapshots are a wrong **cache**, not that the
784    /// ledger is corrupt: [Doctrine VI] makes them disposable, so deleting
785    /// [`Self::snapshots_dir`] restores correctness and costs only speed.
786    /// Rewriting the file here would destroy the evidence that composition has a
787    /// defect, which is the only thing this can tell you that you did not
788    /// already know.
789    ///
790    /// Pair it with the actor counters ([`Self::metrics`], D-079) so a
791    /// divergence found by a scheduled run is visible beside the write latency
792    /// of the period that produced it.
793    ///
794    /// [Doctrine VI]: ../../docs/architecture/s0-s3-foundations.md#doctrine-vi
795    pub async fn verify_snapshot_chain(&self, ts: &str) -> Result<crate::temporal::ChainCheck> {
796        let archive = self
797            .archive_path
798            .exists()
799            .then_some(self.archive_path.as_path());
800        crate::temporal::verify_snapshot_chain(&self.read_conn, ts, archive, &self.snapshots_dir)
801            .await
802    }
803
804    /// The clock every write is stamped with (§5.1.1).
805    pub fn clock(&self) -> &Arc<dyn Clock> {
806        &self.clock
807    }
808
809    /// Schema version this handle opened against.
810    pub fn schema_version(&self) -> u32 {
811        self.schema_version
812    }
813
814    /// Cold database path, derived by convention from the main file.
815    pub fn archive_path(&self) -> &Path {
816        &self.archive_path
817    }
818
819    /// Snapshot directory, derived by convention from the main file.
820    pub fn snapshots_dir(&self) -> &Path {
821        &self.snapshots_dir
822    }
823
824    /// What the write actor has done since this handle was opened (T1.4, D-079).
825    ///
826    /// Requires the `metrics` feature. The counters are per-handle and start at
827    /// zero on `open()` — they are not read from the database, because the thing
828    /// being measured is *this process's* actor and merging two processes'
829    /// histograms would produce a number about neither.
830    ///
831    /// The intended first question is [`crate::metrics::MetricsSnapshot::budget_violations`]:
832    ///
833    /// ```no_run
834    /// # async fn f(db: &macrame::Database) {
835    /// # #[cfg(feature = "metrics")] {
836    /// for k in db.metrics().budget_violations() {
837    ///     eprintln!("{} broke the 3 ms bound {} times", k.kind, k.over_budget);
838    /// }
839    /// # }
840    /// # }
841    /// ```
842    ///
843    /// Reading this does not stop the actor — see
844    /// [`crate::metrics::ActorMetrics::snapshot`] for what that costs in
845    /// consistency, and why the trade goes that way.
846    #[cfg(feature = "metrics")]
847    pub fn metrics(&self) -> crate::metrics::MetricsSnapshot {
848        self.shared.metrics.snapshot()
849    }
850
851    /// The underlying libSQL database, for callers that need their own connection.
852    ///
853    /// # Actor containment is a convention above this line, not a guarantee
854    ///
855    /// **Kept public, and the honest statement of what that costs (Wave 4.3).**
856    /// §5.1 says the write actor is the sole writer, and two mechanisms make that
857    /// true of the handle: every write method goes through a channel, and
858    /// [`Self::read_conn`] carries `PRAGMA query_only = ON`. **Nothing protects a
859    /// connection obtained from here.** A caller can open one, write to `links`
860    /// directly, and the actor will not know — the triggers still fire and the
861    /// ledger stays internally consistent, but the single-writer property that
862    /// [`crate::CHUNK_BUDGET`]'s latency argument rests on is gone, and so is the
863    /// serialisation the overlap guard (D-060) relies on.
864    ///
865    /// This is the same shape as the limit stated in §4.2 for that guard, and it
866    /// is one fact rather than two: **the storage layer permits what this API
867    /// refuses.** Making it private would not change that — the database file is
868    /// reachable by any SQLite client on the machine — it would only remove the
869    /// supported way to do the thing, which is how escape hatches become
870    /// `unsafe`-adjacent folklore.
871    ///
872    /// The free functions [`crate::register_model`] and
873    /// [`crate::upsert_embedding`] take a bare connection for the same reason and
874    /// carry the same caveat; prefer [`Self::register_model`] and
875    /// [`Self::upsert_embeddings`], which go through the actor.
876    ///
877    /// # The legitimate-use list is now one item long (T5.1, D-091)
878    ///
879    /// It used to read: `EXPLAIN QUERY PLAN` and other diagnostics, read-only
880    /// reporting queries wanting their own connection rather than sharing the
881    /// reader, and provoking a guard in a test. The first two are exactly what
882    /// [`Self::diagnostic_conn`] now does, and it does them behind an OS-level
883    /// read-only open rather than on a handle that can write. **Use that.**
884    ///
885    /// What is left is the one use that genuinely requires write access through
886    /// a connection the actor does not own: *provoking a guard* — writing the
887    /// state §4.7 says the storage layer permits and this API refuses, so a test
888    /// can assert the gap is still where the document says it is. That is the
889    /// only thing this crate's own suite uses it for.
890    ///
891    /// # Why `#[doc(hidden)]` and not a `raw-access` feature
892    ///
893    /// T5.1 offers either. The feature is the stronger declaration — it shows up
894    /// in the consumer's `Cargo.toml`, where a reviewer sees it — and it was
895    /// **not** taken, for a reason specific to what uses this:
896    ///
897    /// Cargo features are additive and cannot be *required* by a test target
898    /// except through `required-features`, which makes a plain `cargo test`
899    /// **skip** that binary silently. The binaries that call this are
900    /// `storage_boundary_tests` and `wave1_regression_tests` — the §4.7
901    /// tripwires, whose entire job is to fail when a documented gap moves. Gating
902    /// them behind a feature would mean the ordinary `cargo test` stopped running
903    /// the tests that enforce the section this item is about, to make a
904    /// declaration about a hatch. That trade is the wrong way round, and it is
905    /// the same failure the project already names: a suite that quietly does less
906    /// than it appears to.
907    ///
908    /// So the hatch stays reachable and stops being *discoverable*: it is absent
909    /// from the docs, and the documented path for every non-write use is
910    /// [`Self::diagnostic_conn`]. [D-068] is unchanged — removing it would buy
911    /// the appearance of a guarantee, since the file is reachable by any SQLite
912    /// client on the machine.
913    ///
914    /// [D-068]: ../../docs/architecture/s13-decision-register.md#d-068
915    #[doc(hidden)]
916    pub fn raw(&self) -> &libsql::Database {
917        &self.db
918    }
919
920    // -- write surface (§5.1, Appendix A) --
921    //
922    // Every method here validates and canonicalises before the value crosses the
923    // channel, so a bad edge type or a second-precision timestamp is a typed
924    // error at the call site rather than an engine `CHECK` failure surfacing
925    // from the far side of an actor with no context attached.
926    //
927    // NOTE (§5.1.8, D-028): awaiting one of these waits on a Rust channel, not
928    // in SQLite, so `busy_timeout` does not bound it. During an in-flight
929    // `rebuild_current` or `archive` the caller stalls for that transaction's
930    // duration. Wrap in `tokio::time::timeout` if you need a bound — but a
931    // timeout is not a cancellation: the command stays queued and commits when
932    // the actor reaches it.
933
934    /// Assert an edge (Doctrine III: a new row, never an update).
935    pub async fn assert_edge(&self, edge: EdgeAssertion) -> Result<()> {
936        let edge = edge.normalized()?;
937        self.high(|responder| HighPriCommand::AssertEdge { edge, responder })
938            .await
939    }
940
941    /// Close an open interval by asserting its replacement (Doctrine III).
942    pub async fn retire_edge(
943        &self,
944        source: impl Into<String>,
945        target: impl Into<String>,
946        edge_type: impl Into<String>,
947        valid_from: &str,
948        valid_to: &str,
949    ) -> Result<()> {
950        let edge_type = edge_type.into();
951        crate::graph::edge::validate_edge_type(&edge_type)?;
952        let valid_from = timestamp::normalize(valid_from)?;
953        let valid_to = timestamp::normalize(valid_to)?;
954        let (source, target) = (source.into(), target.into());
955
956        self.high(|responder| HighPriCommand::RetireEdge {
957            source,
958            target,
959            edge_type,
960            valid_from,
961            valid_to,
962            responder,
963        })
964        .await
965    }
966
967    /// Insert or update a concept.
968    pub async fn upsert_concept(&self, concept: ConceptUpsert) -> Result<()> {
969        let concept = concept.normalized()?;
970        self.high(|responder| HighPriCommand::UpsertConcept { concept, responder })
971            .await
972    }
973
974    /// Assert many edges in one transaction under one stamp (D-014).
975    ///
976    /// # This is the one write with no latency bound, and here is what it costs
977    ///
978    /// The batch is one act under one `recorded_at`, so it cannot be chunked —
979    /// splitting it is the thing this method exists not to do. That makes the
980    /// actor's hold a function of `edges.len()`, and until now the only
981    /// statement of that anywhere was the prose "uncapped" in
982    /// [`CHUNK_BUDGET`]'s table. A caller who stalls every other writer for
983    /// eight seconds should have been able to predict it from the signature.
984    ///
985    /// Measured on libSQL 0.9.30 (T1.3, D-081), holding the actor for:
986    ///
987    /// | rows | hold |
988    /// |---|---|
989    /// | 500 | ~34 ms |
990    /// | 2,000 | ~155 ms |
991    /// | 10,000 | ~1.0 s |
992    /// | 20,000 | ~2.6 s |
993    ///
994    /// [`estimated_bulk_hold`] is that curve as a function, and this method
995    /// emits a `tracing::warn!` when it predicts more than
996    /// [`BULK_ATOMIC_WARN_HOLD`]. **The estimate is a shape, not a promise** —
997    /// see [`estimated_bulk_hold`] for what it is calibrated against and where
998    /// it will be wrong.
999    ///
1000    /// A caller who needs the latency bound and not the atomicity wants
1001    /// [`Self::bulk_import`], which is the same write chunked and explicitly not
1002    /// atomic overall (D-011).
1003    pub async fn write_bulk_atomic(&self, edges: Vec<EdgeAssertion>) -> Result<usize> {
1004        let estimate = estimated_bulk_hold(&edges);
1005        if estimate > BULK_ATOMIC_WARN_HOLD {
1006            // Warned here rather than in the actor, and before the send: this is
1007            // the caller's own task, so the log line lands with their span
1008            // attached and names the call site that chose the batch size. By the
1009            // time the actor has it, the only context left is "a large batch".
1010            tracing::warn!(
1011                rows = edges.len(),
1012                estimated_hold_ms = estimate.as_millis() as u64,
1013                "write_bulk_atomic will hold the write actor for roughly \
1014                 {estimate:?} — it is atomic by contract (D-014) and cannot be \
1015                 chunked. Every other writer waits that long. Use bulk_import \
1016                 if the batch does not need to be all-or-nothing."
1017            );
1018        }
1019
1020        let edges = normalize_all(edges)?;
1021        self.high(|responder| HighPriCommand::WriteBulkAtomic { edges, responder })
1022            .await
1023    }
1024
1025    /// Rebuild `links_current` from `links` and verify zero drift (§5.8).
1026    pub async fn rebuild_current(&self) -> Result<RebuildReport> {
1027        self.high(|responder| HighPriCommand::RebuildCurrent { responder })
1028            .await
1029    }
1030
1031    /// Rebuild `links_current` beside itself, in chunks (§5.8, T1.2, D-082).
1032    ///
1033    /// Same result as [`Self::rebuild_current`], different latency profile.
1034    /// `rebuild_current` is one transaction holding the write lock for its whole
1035    /// duration, because D-023 will not let the `DELETE` and the `INSERT` be
1036    /// split: a reader landing between them sees a graph with no edges and no
1037    /// error. This builds the replacement in a shadow table instead — the live
1038    /// table stays live and trigger-maintained throughout — and swaps it in at
1039    /// the end.
1040    ///
1041    /// Each step is its own actor turn, so an interactive assertion can jump the
1042    /// queue between chunks. That is the whole of the improvement, and it is why
1043    /// the loop is here rather than inside the actor's arm (the same reasoning
1044    /// as [`Self::archive_windowed`] and [`Self::bulk_import`]).
1045    ///
1046    /// # What the swap still costs
1047    ///
1048    /// Not microseconds. Index names are global and SQLite has no `ALTER INDEX
1049    /// … RENAME`, so the shadow cannot be built carrying `links_current`'s index
1050    /// names while `links_current` still holds them — and building it under
1051    /// other names would leave the table permanently indexed under names absent
1052    /// from [`CREATE_INDICES`](crate::schema::ddl::CREATE_INDICES), so the next
1053    /// migration would create a second copy of each.
1054    /// `DROP TABLE` frees the names, so the swap transaction is where
1055    /// the three indexes get built. What the chunking moves off the lock is the
1056    /// **projection** — the window function over all of `links` — which is the
1057    /// O(E log E) term.
1058    ///
1059    /// # When this returns an error rather than a repair
1060    ///
1061    /// [`DbError::RebuildInterrupted`] means an archive committed while the
1062    /// shadow was being built. Its deletions are invisible to a catch-up pass
1063    /// keyed on `recorded_at` — a deleted row has no `recorded_at` left to find
1064    /// it by — so the work is discarded rather than swapped in. `links_current`
1065    /// is untouched and the call can simply be retried.
1066    ///
1067    /// Use [`Self::rebuild_current`] when the repair must be one atomic act, or
1068    /// when nothing else is contending for the actor and the extra turns are
1069    /// pure overhead.
1070    pub async fn rebuild_current_chunked(&self) -> Result<RebuildReport> {
1071        use crate::integrity::{ShadowOutcome, ShadowStep};
1072
1073        // Each `else` arm is unreachable: the actor maps each step to its own
1074        // outcome variant. Written as a refutable pattern rather than an
1075        // `unwrap` so that adding a step cannot turn a mismatch into a panic on
1076        // the write path — and `WriterDroppedResponder` is the honest name for
1077        // "the actor answered with something this cannot use".
1078        let ShadowOutcome::Started { build_start, epoch } =
1079            self.shadow_step(ShadowStep::Begin).await?
1080        else {
1081            return Err(DbError::WriterDroppedResponder);
1082        };
1083
1084        let mut after: Option<String> = None;
1085        loop {
1086            let ShadowOutcome::Filled { last } = self
1087                .shadow_step(ShadowStep::Fill {
1088                    after: after.take(),
1089                })
1090                .await?
1091            else {
1092                return Err(DbError::WriterDroppedResponder);
1093            };
1094            match last {
1095                Some(last) => after = Some(last),
1096                None => break,
1097            }
1098        }
1099
1100        let ShadowOutcome::Swapped { rows } = self
1101            .shadow_step(ShadowStep::Swap { build_start, epoch })
1102            .await?
1103        else {
1104            return Err(DbError::WriterDroppedResponder);
1105        };
1106
1107        Ok(RebuildReport {
1108            rows_rebuilt: rows,
1109            // Not audited. The chunked path's whole argument is that the
1110            // expensive work happens off the lock, and `audit_current` is two
1111            // `EXCEPT` passes over the projection — the cost D-077 removed from
1112            // the archive for the same reason. A caller who wants the check has
1113            // `audit_current` on the read connection, where it costs nobody the
1114            // write lock.
1115            drift_after: 0,
1116        })
1117    }
1118
1119    /// Run one step of a chunked rebuild, for a caller doing its own scheduling.
1120    ///
1121    /// [`Self::rebuild_current_chunked`] is this in a loop and is what almost
1122    /// everyone wants. This exists because that loop offers no seam: it drives
1123    /// `Begin`, then `Fill` to exhaustion, then `Swap`, and a caller who needs to
1124    /// do something *between* steps — pace them against a frame budget, abandon
1125    /// a rebuild that has run long enough, or provoke the archive interlock in a
1126    /// test — cannot get in.
1127    ///
1128    /// The obligation that comes with it: `epoch` from
1129    /// [`ShadowOutcome::Started`](crate::integrity::ShadowOutcome) must be handed
1130    /// back to [`ShadowStep::Swap`](crate::integrity::ShadowStep), or the
1131    /// archive interlock is defeated and a stale projection can be swapped in.
1132    /// The looping version cannot get that wrong; this one can.
1133    pub async fn shadow_step(
1134        &self,
1135        step: crate::integrity::ShadowStep,
1136    ) -> Result<crate::integrity::ShadowOutcome> {
1137        self.low(|responder| LowPriCommand::ShadowRebuild { step, responder })
1138            .await
1139    }
1140
1141    /// Import edges on the background channel, chunked (D-011).
1142    ///
1143    /// Atomic *per chunk*, not overall: a failure partway leaves earlier chunks
1144    /// committed. That is the tradeoff [`chunk_rows`] documents — use
1145    /// [`Database::write_bulk_atomic`] when the batch must be all-or-nothing.
1146    ///
1147    /// Chunked at [`chunk_rows::EDGES`], which is also faster in total than the
1148    /// larger chunks this used through 0.5.5 (D-058).
1149    pub async fn bulk_import(&self, edges: Vec<EdgeAssertion>) -> Result<usize> {
1150        let edges = normalize_all(edges)?;
1151        let chunks: Vec<_> = edges.chunks(chunk_rows::EDGES).map(<[_]>::to_vec).collect();
1152        self.low_chunked(chunks, |chunk, responder| LowPriCommand::BulkImportChunk {
1153            chunk,
1154            responder,
1155        })
1156        .await
1157    }
1158
1159    /// Upsert many **concepts** on the background channel, chunked (D-011).
1160    ///
1161    /// This is the bulk concept path, and every row it writes is a ledger write:
1162    /// it versions the concept and lands in `transaction_log`. Derived analytics
1163    /// output does not belong here — see
1164    /// [`Database::write_analytics_annotations`] and D-041.
1165    ///
1166    /// Called `write_annotations` through 0.5.6, from when the two writes were
1167    /// one call. D-041 split them and the name stayed on the wrong one for three
1168    /// releases, so the crate had a `write_annotations` that wrote concepts
1169    /// sitting beside a `write_analytics_annotations` that wrote annotations
1170    /// (D-075).
1171    pub async fn write_concepts(&self, concepts: Vec<ConceptUpsert>) -> Result<usize> {
1172        let concepts: Vec<ConceptUpsert> = concepts
1173            .into_iter()
1174            .map(ConceptUpsert::normalized)
1175            .collect::<Result<_>>()?;
1176        let chunks: Vec<_> = concepts
1177            .chunks(chunk_rows::CONCEPTS)
1178            .map(<[_]>::to_vec)
1179            .collect();
1180        self.low_chunked(chunks, |chunk, responder| {
1181            LowPriCommand::WriteConceptsChunk { chunk, responder }
1182        })
1183        .await
1184    }
1185
1186    /// State as believed at `ts` (§5.5, D-026, D-049).
1187    ///
1188    /// A read: it runs on `read_conn` and never touches the Write Actor, so a
1189    /// reconstruction and a full-speed write-back do not slow each other.
1190    ///
1191    /// Prefer this to calling [`crate::temporal::reconstruct`] directly. The
1192    /// free function takes the archive path and the snapshot directory as
1193    /// arguments, and a caller who passes `None` for the second gets a correct
1194    /// answer that folds the whole log every time — the composition is opt-in
1195    /// at that layer and easy to leave off by accident. Here both come from the
1196    /// handle, so the fast path is the default one.
1197    pub async fn reconstruct(&self, ts: &str) -> Result<crate::temporal::MaterializedState> {
1198        let ts = timestamp::normalize(ts)?;
1199        crate::temporal::reconstruct(
1200            &self.read_conn,
1201            &ts,
1202            Some(&self.archive_path),
1203            Some(&self.snapshots_dir),
1204        )
1205        .await
1206    }
1207
1208    /// Create a model's embedding table and DiskANN index (§5.9, D-048).
1209    ///
1210    /// Idempotent: registering a model that already exists at the same
1211    /// dimension succeeds, and at a different dimension fails with
1212    /// [`DbError::DimMismatch`] naming both, rather than no-opping through
1213    /// `IF NOT EXISTS` and leaving the caller believing the dimension they
1214    /// asked for is the one in force.
1215    ///
1216    /// This issues DDL, which everywhere else in the crate is the migration
1217    /// runner's exclusive business (D-032). The exception is bounded and
1218    /// deliberate: a model's table is created once, by an explicit call, and
1219    /// the alternative — a caller-supplied write connection — is the very thing
1220    /// the Write Actor exists to make impossible.
1221    ///
1222    /// # Latency
1223    ///
1224    /// One small transaction, but it queues like any other write: see §5.1.8.
1225    pub async fn register_model(&self, model: &ModelName, dim: usize) -> Result<()> {
1226        let model = model.clone();
1227        self.high(|responder| HighPriCommand::RegisterModel {
1228            model,
1229            dim,
1230            responder,
1231        })
1232        .await
1233    }
1234
1235    /// Store or replace vectors for `model`, chunked (§5.9, D-011, D-048).
1236    ///
1237    /// The write path for embeddings. Before 0.5.4 there was none:
1238    /// [`crate::vector::upsert_embedding`] takes a raw connection, `read_conn`
1239    /// is `query_only`, and the write connection lives inside the actor — so an
1240    /// application could search vectors it had no way to store.
1241    ///
1242    /// Low priority and chunked at [`chunk_rows::EMBEDDINGS`], because embedding
1243    /// is bulk derived work: a 50,000-vector backfill must yield to an
1244    /// interactive assertion at every chunk boundary. That constant is the
1245    /// smallest of the four by a wide margin — DiskANN index maintenance makes an
1246    /// embedding the most expensive row in the system (D-058). Atomic per chunk, not overall, which
1247    /// is the same trade [`Database::bulk_import`] makes and is safer here than
1248    /// there — an embedding is derived (Doctrine VII), so a partially written
1249    /// batch is recoverable by re-embedding.
1250    ///
1251    /// Fails with [`DbError::ModelNotRegistered`] if `model` has no table, and
1252    /// [`DbError::DimMismatch`] if a vector's length is not the declared
1253    /// dimension. The dimension is read from the schema once per chunk (D-037):
1254    /// the crate keeps no registry of its own to fall out of date.
1255    pub async fn upsert_embeddings(
1256        &self,
1257        model: &ModelName,
1258        rows: Vec<(String, Vec<f32>)>,
1259    ) -> Result<usize> {
1260        let chunks: Vec<_> = rows
1261            .chunks(chunk_rows::EMBEDDINGS)
1262            .map(<[_]>::to_vec)
1263            .collect();
1264        self.low_chunked(chunks, |chunk, responder| {
1265            LowPriCommand::UpsertEmbeddingChunk {
1266                model: model.clone(),
1267                chunk,
1268                responder,
1269            }
1270        })
1271        .await
1272    }
1273
1274    /// Reconstruct the concept-text search index from the ledger (§5.9, D-036).
1275    ///
1276    /// The FTS index is derivative: D-036 promises every derivative table can be
1277    /// rebuilt from the ledger tables, and this is that promise made callable
1278    /// for `concepts_fts`. Needed after a restore that skipped the shadow
1279    /// tables, or if the index is ever suspected of drifting from the text —
1280    /// and, as a matter of policy, cheaper to run than to reason about.
1281    ///
1282    /// The work is `INSERT INTO concepts_fts(concepts_fts) VALUES('rebuild')`,
1283    /// which is FTS5's own operation over the content table, so this is not a
1284    /// second implementation of the sync triggers that could disagree with them.
1285    pub async fn rebuild_fts(&self) -> Result<()> {
1286        self.low(|responder| LowPriCommand::RebuildFts { responder })
1287            .await
1288    }
1289
1290    // **There is deliberately no `verify_fts()` (§5.9, D-071).**
1291    //
1292    // `rebuild_fts` is the repair with no way to ask whether it is needed, and
1293    // Wave 5 set out to add the missing half. FTS5 offers `'integrity-check'`,
1294    // which looked like exactly the engine-provided answer this crate prefers.
1295    // It is not: on libSQL 0.9.30 it verifies the index's *internal* consistency
1296    // and not its agreement with the content table. Measured — after
1297    // `'delete-all'` the index matches nothing where it matched ten rows, and
1298    // both `'integrity-check'` and `'integrity-check', 0` still report success.
1299    //
1300    // A `verify_fts()` on that footing would answer "healthy" for an empty
1301    // index, which is worse than having no method at all: it is the shape of
1302    // defect AC, a function that looks like it checks something and does not.
1303    // `an_emptied_fts_index_still_passes_integrity_check` pins the limitation so
1304    // that if a later libSQL fixes it, the test fails and says so.
1305
1306    /// Write derived analytics results on the background channel, chunked
1307    /// (§5.4, D-041).
1308    ///
1309    /// Rows go to `analytics_annotations`, which has no log trigger, so nothing
1310    /// written here reaches `transaction_log` and nothing here versions a
1311    /// concept. Rerunning an algorithm replaces the previous pass rather than
1312    /// recording that the world changed.
1313    ///
1314    /// Low priority and chunked at [`chunk_rows::ANNOTATIONS`] — the largest of
1315    /// the four, because this is the only bulk table carrying no triggers at all
1316    /// and its rows are correspondingly cheap (D-058) — so a 50,000-label Louvain
1317    /// save yields to interactive writes at every chunk boundary and carries the
1318    /// per-chunk fidelity boundary of §5.1.6 — a partially written pass is
1319    /// recoverable by rerunning, which is the property that makes derived state
1320    /// safe to write this way and assertions not.
1321    pub async fn write_analytics_annotations(&self, annotations: Vec<Annotation>) -> Result<usize> {
1322        let chunks: Vec<_> = annotations
1323            .chunks(chunk_rows::ANNOTATIONS)
1324            .map(<[_]>::to_vec)
1325            .collect();
1326        self.low_chunked(chunks, |chunk, responder| {
1327            LowPriCommand::WriteAnalyticsChunk { chunk, responder }
1328        })
1329        .await
1330    }
1331
1332    /// Move closed intervals and superseded log rows older than `cutoff` to the
1333    /// cold database (§5.7, D-012).
1334    pub async fn archive(&self, cutoff: &str) -> Result<ArchiveReport> {
1335        let cutoff = timestamp::normalize(cutoff)?;
1336        let archive_path = self.archive_path.clone();
1337        self.low(|responder| LowPriCommand::Archive {
1338            cutoff,
1339            archive_path,
1340            responder,
1341        })
1342        .await
1343    }
1344
1345    /// Archive up to `cutoff` as a sequence of sessions, each covering at most
1346    /// `window` of **transaction** time (T1.1, D-080).
1347    ///
1348    /// `archive(cutoff)` is one transaction whose size is set by how long it has
1349    /// been since the last one, which makes it the least bounded of the three
1350    /// operations exempt from [`CHUNK_BUDGET`] — its hold is a function of
1351    /// operational history rather than of anything a caller chose. This runs the
1352    /// same work as *N* complete sessions, each with its own marker, horizon row
1353    /// and rebuild, and returns one [`ArchiveReport`] per session in order.
1354    ///
1355    /// # D-012 is satisfied per session, and that is what it requires
1356    ///
1357    /// The atomicity D-012 demands is that copy-then-delete never be split — a
1358    /// crash between the phases duplicates or loses rows. *N* small sessions
1359    /// satisfy that exactly as one large one does. The obligation windowing adds
1360    /// is that a partial run leave a coherent intermediate state, which it does:
1361    /// each session commits a valid horizon, so a failure at window *k* leaves a
1362    /// database archived up to boundary *k−1* and nothing in between. **The
1363    /// sequence is not atomic and does not claim to be** — on error, the reports
1364    /// for the sessions that did commit are lost with it, but their effect is
1365    /// not, and re-running with the same `cutoff` completes the job.
1366    ///
1367    /// # Each session is its own actor turn, and that is the entire point
1368    ///
1369    /// This loop lives here, on the handle, rather than inside the actor's
1370    /// `Archive` arm. Putting it there would have produced *N* small
1371    /// transactions inside **one** hold, which shrinks the transaction and
1372    /// changes the latency not at all: the actor is single-threaded, so nothing
1373    /// else writes until its turn returns regardless of how many `COMMIT`s the
1374    /// turn contains. Sending *N* commands returns the actor to its `select!`
1375    /// between sessions, which is where an interactive assertion gets to jump
1376    /// the queue — and it is high-priority, so it does.
1377    ///
1378    /// The same reasoning is why [`Self::bulk_import`] chunks here and not
1379    /// there, and it is the trap T1.2 names for `CREATE TABLE … AS SELECT`.
1380    ///
1381    /// # Choosing a window
1382    ///
1383    /// The bound is on *transaction* time, so the session count is set by how
1384    /// far back the hot file goes, not by how much it holds. A window is
1385    /// rejected rather than clamped if it would need more than
1386    /// [`MAX_ARCHIVE_SESSIONS`] sessions — see [`DbError::ArchiveWindow`].
1387    ///
1388    /// Windows containing nothing archivable are cheap but not free: each still
1389    /// opens a transaction and writes a horizon row. What they no longer do is
1390    /// re-project `links_current`, which `archive_session` now skips when its
1391    /// `DELETE` removed no rows — without that, windowing costs *more* in total
1392    /// than not windowing, because the repair term scales with the surviving
1393    /// table and not with the batch (D-077).
1394    pub async fn archive_windowed(
1395        &self,
1396        cutoff: &str,
1397        window: std::time::Duration,
1398    ) -> Result<Vec<ArchiveReport>> {
1399        let cutoff = timestamp::normalize(cutoff)?;
1400        let boundaries = self.archive_boundaries(&cutoff, window).await?;
1401
1402        let mut reports = Vec::with_capacity(boundaries.len());
1403        for boundary in boundaries {
1404            let archive_path = self.archive_path.clone();
1405            reports.push(
1406                self.low(|responder| LowPriCommand::Archive {
1407                    cutoff: boundary,
1408                    archive_path,
1409                    responder,
1410                })
1411                .await?,
1412            );
1413        }
1414        Ok(reports)
1415    }
1416
1417    /// The cutoffs [`Self::archive_windowed`] will run, ascending, ending at
1418    /// `cutoff` exactly.
1419    ///
1420    /// Read on `read_conn`, not on the actor: this is two `MIN`s and the actor
1421    /// has no reason to hold its lock for them.
1422    ///
1423    /// The lower end comes from the data rather than from the clock. Stepping
1424    /// from some fixed epoch would make the session count a function of the
1425    /// calendar — a database opened yesterday would still be asked to archive
1426    /// 1970 — whereas the oldest `recorded_at` actually present is the earliest
1427    /// boundary that can contain anything.
1428    async fn archive_boundaries(
1429        &self,
1430        cutoff: &str,
1431        window: std::time::Duration,
1432    ) -> Result<Vec<String>> {
1433        // A single session at `cutoff` is exactly `archive(cutoff)`, and it is
1434        // the right answer for an empty hot file: it still writes the horizon
1435        // row, so windowed and unwindowed runs leave the same observable state.
1436        let Some(oldest) = self.oldest_hot_stamp(cutoff).await? else {
1437            return Ok(vec![cutoff.to_string()]);
1438        };
1439
1440        let start = timestamp::parse(&oldest)?;
1441        let end = timestamp::parse(cutoff)?;
1442        let Ok(span) = end.duration_since(start) else {
1443            // Everything in the hot file is at or after the cutoff, so there is
1444            // nothing in range to divide.
1445            return Ok(vec![cutoff.to_string()]);
1446        };
1447
1448        if window.is_zero() {
1449            return Err(DbError::ArchiveWindow {
1450                window,
1451                reason: "a zero-length window never advances past the first boundary".into(),
1452            });
1453        }
1454
1455        // `div_ceil` on nanos: a span of 90 minutes in 60-minute windows is two
1456        // sessions, not one. `as_nanos` is u128, so neither the division nor the
1457        // span can overflow for any timestamp this crate can store.
1458        let sessions = span.as_nanos().div_ceil(window.as_nanos());
1459        if sessions > MAX_ARCHIVE_SESSIONS as u128 {
1460            return Err(DbError::ArchiveWindow {
1461                window,
1462                reason: format!(
1463                    "a span of {span:?} would need {sessions} sessions (limit \
1464                     {MAX_ARCHIVE_SESSIONS}); widen the window"
1465                ),
1466            });
1467        }
1468
1469        let mut boundaries = Vec::with_capacity(sessions as usize);
1470        for k in 1..sessions {
1471            boundaries.push(timestamp::format(start + window * k as u32));
1472        }
1473        // The last boundary is `cutoff` itself and not `start + n*window`, which
1474        // would overshoot and archive rows the caller excluded.
1475        boundaries.push(cutoff.to_string());
1476        Ok(boundaries)
1477    }
1478
1479    /// Oldest `recorded_at` below `cutoff` in either hot table, or `None`.
1480    async fn oldest_hot_stamp(&self, cutoff: &str) -> Result<Option<String>> {
1481        let mut oldest: Option<String> = None;
1482        for table in ["links", "transaction_log"] {
1483            let found: Option<String> = self
1484                .read_conn
1485                .query(
1486                    &format!("SELECT MIN(recorded_at) FROM {table} WHERE recorded_at < ?1"),
1487                    libsql::params![cutoff],
1488                )
1489                .await?
1490                .next()
1491                .await?
1492                .and_then(|row| row.get(0).ok());
1493            if let Some(found) = found {
1494                if oldest.as_ref().is_none_or(|o| found < *o) {
1495                    oldest = Some(found);
1496                }
1497            }
1498        }
1499        Ok(oldest)
1500    }
1501
1502    /// Send a high-priority command and wait for its answer.
1503    ///
1504    /// The two error mappings here are the whole reason this helper exists.
1505    /// `send` failing means the actor is gone — `WriterUnavailable`. The
1506    /// responder being dropped without an answer means the actor took the
1507    /// command and never replied — `WriterDroppedResponder`, which is a bug in
1508    /// the actor rather than a condition the caller can retry. Both variants
1509    /// existed in `error.rs` from 0.4.5 and neither was ever constructed, so a
1510    /// dead actor and a hung one were both just a caller waiting forever.
1511    async fn high<T>(
1512        &self,
1513        make: impl FnOnce(oneshot::Sender<Result<T>>) -> HighPriCommand,
1514    ) -> Result<T> {
1515        let (tx, rx) = oneshot::channel();
1516        self.highpri_tx
1517            .send(make(tx))
1518            .await
1519            .map_err(|_| DbError::WriterUnavailable)?;
1520        rx.await.map_err(|_| DbError::WriterDroppedResponder)?
1521    }
1522
1523    /// Send each chunk in turn and sum the counts — the shape all four bulk
1524    /// paths share (T3.4, D-086).
1525    ///
1526    /// # This is sequential on purpose, and the purpose is a measurement
1527    ///
1528    /// T3.4 proposed pipelining: send *k* chunks ahead so the actor never finds
1529    /// an empty queue. The reasoning is that awaiting each chunk before building
1530    /// the next leaves the actor idle for a channel round trip every time, which
1531    /// on a 1M-edge import is ~11,000 idle gaps.
1532    ///
1533    /// Both halves of that are true and the conclusion does not follow. The gaps
1534    /// are real; they are also **four orders of magnitude smaller than the work
1535    /// they interrupt**. A tokio mpsc hop is sub-microsecond and a chunk takes
1536    /// 13–21 ms. Implemented and swept at depths 1, 2, 4, 8 and 16 over 20K and
1537    /// 100K edges: every cell landed within 1% of sequential, in both directions
1538    /// — see `examples/pipeline_diag.rs`, which is kept precisely so this is not
1539    /// re-proposed from the same reasoning.
1540    ///
1541    /// So the pipelining was removed and the deduplication kept. It was not free
1542    /// to hold: with chunks in flight, a failure at chunk `i` no longer leaves a
1543    /// **prefix** committed, because `i+1 ..= i+k-1` were already sent and commit
1544    /// anyway. D-011 promises "earlier chunks committed", and paying for that
1545    /// with a weaker recovery story in exchange for nothing measurable is the
1546    /// wrong trade.
1547    ///
1548    /// Sending stops at the first error, so what commits is exactly the prefix
1549    /// before the failure.
1550    async fn low_chunked<C>(
1551        &self,
1552        chunks: Vec<C>,
1553        make: impl Fn(C, oneshot::Sender<Result<usize>>) -> LowPriCommand,
1554    ) -> Result<usize> {
1555        let mut written = 0usize;
1556        for chunk in chunks {
1557            let (tx, rx) = oneshot::channel();
1558            self.lowpri_tx
1559                .send(make(chunk, tx))
1560                .await
1561                .map_err(|_| DbError::WriterUnavailable)?;
1562            written += rx.await.map_err(|_| DbError::WriterDroppedResponder)??;
1563        }
1564        Ok(written)
1565    }
1566
1567    async fn low<T>(
1568        &self,
1569        make: impl FnOnce(oneshot::Sender<Result<T>>) -> LowPriCommand,
1570    ) -> Result<T> {
1571        let (tx, rx) = oneshot::channel();
1572        self.lowpri_tx
1573            .send(make(tx))
1574            .await
1575            .map_err(|_| DbError::WriterUnavailable)?;
1576        rx.await.map_err(|_| DbError::WriterDroppedResponder)?
1577    }
1578
1579    /// Clean shutdown: stop the Write Actor, then write the final snapshot (§5.1.7).
1580    ///
1581    /// Order matters. The snapshot is taken *after* the actor has stopped and
1582    /// been joined, so no write can land between the fold and the file — the
1583    /// anchor it records is the last thing that happened, not the last thing
1584    /// that happened to be visible.
1585    ///
1586    /// A failed snapshot is reported rather than swallowed. It is not a
1587    /// durability loss — the ledger is in the WAL and the log replays without
1588    /// it — but it means the next open starts from an older anchor, and a caller
1589    /// that never hears about it cannot know why startup got slower.
1590    ///
1591    /// **The cadence stops first (§5.5, D-053).** Both it and `write_final` end
1592    /// by running retention over the snapshot directory, and retention deletes
1593    /// files. Letting them overlap would mean one pass enumerating the directory
1594    /// while the other removes from it — not a correctness problem for the
1595    /// ledger, which is why the ordering is stated rather than locked, but a
1596    /// source of spurious warnings and of a final anchor that could be deleted
1597    /// by a cleanup that started before it existed. Stopping the cadence, then
1598    /// the actor, then taking the snapshot leaves exactly one writer at each
1599    /// step.
1600    pub async fn close(mut self) -> Result<()> {
1601        if let Some(stop) = self.cadence_stop.take() {
1602            let _ = stop.send(true);
1603        }
1604        if let Some(handle) = self.cadence.take() {
1605            let _ = handle.await;
1606        }
1607
1608        let (tx, rx) = oneshot::channel();
1609        let _ = self
1610            .highpri_tx
1611            .send(HighPriCommand::Shutdown { responder: tx })
1612            .await;
1613        let _ = rx.await;
1614
1615        // **The writer's `Result` is propagated, not discarded (Wave 4.2).**
1616        // It used to be `let _ = handle.await`, so an actor that had panicked or
1617        // returned an error closed "successfully" and the caller's last chance to
1618        // learn that the write path had died was spent silently. A `JoinError`
1619        // here means the actor panicked; the inner `Result` is whatever it
1620        // returned.
1621        //
1622        // Ordered before the final snapshot on purpose: a snapshot written after
1623        // a failed writer records a state the caller has no reason to trust, and
1624        // returning the writer's error while also having written that file is
1625        // worse than not writing it.
1626        if let Some(handle) = self.writer.take() {
1627            match handle.await {
1628                Ok(res) => res?,
1629                Err(e) => {
1630                    return Err(DbError::WriterStopped(format!(
1631                        "the write actor did not exit cleanly: {e}"
1632                    )))
1633                }
1634            }
1635        }
1636
1637        let ts = self.clock.now();
1638        let archive = self
1639            .archive_path
1640            .exists()
1641            .then_some(self.archive_path.as_path());
1642        snapshot::write_final(&self.read_conn, &self.snapshots_dir, &ts, archive).await?;
1643
1644        // Marks the handle closed so `Drop` knows not to complain.
1645        self.closed = true;
1646        Ok(())
1647    }
1648}
1649
1650/// Notes a missed `close()` at `warn!`, and deliberately does **not** assert.
1651///
1652/// **§7.3 offered option B — document `close()` as mandatory and `debug_assert`
1653/// in `Drop` — and Wave 4.2 implemented it, measured the consequence, and
1654/// reduced it to a warning.** The assert fired on roughly thirty tests on its
1655/// first run. That is the signal it was built to produce, and the right reading
1656/// of it was not "thirty tests are wrong".
1657///
1658/// What dropping actually costs is one final snapshot. Nothing else: every
1659/// public write method awaits its responder, so by the time a caller *can* drop
1660/// the handle, every write it issued has already committed; and the cadence stops
1661/// on its own, because `cadence_stop` is a `watch::Sender` whose drop signals the
1662/// task. A snapshot is derivative state under Doctrine VI — disposable,
1663/// reconstructible, and never the only copy of anything. Losing one makes the
1664/// next `reconstruct` fold from an older anchor, which is **slower, not wrong**.
1665///
1666/// A `debug_assert` aborts a test run. Spending that on a performance loss, in a
1667/// project whose own notes say a suite that fails for reasons unrelated to the
1668/// code under test trains people to ignore red, is the wrong trade — and paying
1669/// it in thirty places would have made `close()` look mandatory by ceremony
1670/// rather than by consequence. `close()` remains the right thing to call, and
1671/// the two reasons to call it are now stated where they can be acted on: the
1672/// snapshot, and the writer's `Result`, which only `close()` can return.
1673///
1674/// Option A ("abort the actor and log") stays rejected, for the reason it was
1675/// rejected twice before: `Drop` cannot await, so it cannot drain, and cleanup
1676/// that cannot clean up is worse than none — it looks like cleanup.
1677impl Drop for Database {
1678    fn drop(&mut self) {
1679        if !self.closed {
1680            tracing::warn!(
1681                "Database dropped without close(): the final snapshot was not written, \
1682                 so the next reconstruct folds from an older anchor, and the write \
1683                 actor's exit status was not checked. Prefer close().await."
1684            );
1685        }
1686    }
1687}
1688
1689fn normalize_all(edges: Vec<EdgeAssertion>) -> Result<Vec<EdgeAssertion>> {
1690    edges.into_iter().map(EdgeAssertion::normalized).collect()
1691}
1692
1693/// Identical pragma configuration on every connection.
1694async fn configure(conn: libsql::Connection) -> Result<libsql::Connection> {
1695    // NOTE: `journal_mode` and `busy_timeout` return their resulting value as a
1696    // row, and libsql's `execute()` rejects any statement that yields rows
1697    // ("Execute returned rows"). They must be issued through `query()`.
1698    let _ = conn.query("PRAGMA journal_mode = WAL", ()).await?;
1699    let _ = conn.query("PRAGMA busy_timeout = 5000", ()).await?;
1700    conn.execute("PRAGMA synchronous = NORMAL", ()).await?;
1701    conn.execute("PRAGMA foreign_keys = ON", ()).await?;
1702    conn.execute("PRAGMA recursive_triggers = OFF", ()).await?;
1703    Ok(conn)
1704}
1705
1706/// Helper to derive the snapshot directory by convention: foo.db -> foo_snapshots/
1707fn derive_snapshots_dir(path: &Path) -> PathBuf {
1708    let mut dir = path.to_path_buf();
1709    let stem = path
1710        .file_stem()
1711        .and_then(|s| s.to_str())
1712        .unwrap_or("macrame");
1713    dir.set_file_name(format!("{stem}_snapshots"));
1714    dir
1715}
1716
1717/// Helper to derive archive database path by convention: foo.db -> foo_archive.db
1718fn derive_archive_path(path: &Path) -> PathBuf {
1719    let mut archive = path.to_path_buf();
1720    if let Some(stem) = path.file_stem().and_then(|s| s.to_str()) {
1721        let ext = path.extension().and_then(|e| e.to_str()).unwrap_or("db");
1722        archive.set_file_name(format!("{stem}_archive.{ext}"));
1723    } else {
1724        archive.set_extension("archive.db");
1725    }
1726    archive
1727}
1728
1729/// Dedicated Write Actor event loop prioritizing high-priority UI requests over low-priority background work.
1730///
1731/// # The turn is the unit, not the statement (T1.4)
1732///
1733/// One iteration of this loop is one *hold*: the actor is single-threaded and
1734/// the SQLite write lock is not preemptible, so from the moment a command starts
1735/// executing until it returns, nothing else writes. That is the quantity
1736/// [`CHUNK_BUDGET`] bounds, and so it is the quantity
1737/// [`crate::metrics::ActorMetrics`] measures — deliberately around the whole
1738/// `execute` call rather than inside it. Timing the SQL alone would have
1739/// reported a bound that held while callers waited.
1740///
1741/// Queue depth is sampled *before* the `select!`, so it is the backlog the turn
1742/// found on arrival rather than the one it left behind.
1743async fn run_writer_actor(
1744    conn: libsql::Connection,
1745    clock: Arc<dyn Clock>,
1746    mut highpri_rx: mpsc::Receiver<HighPriCommand>,
1747    mut lowpri_rx: mpsc::Receiver<LowPriCommand>,
1748    shared: Arc<ActorShared>,
1749) -> Result<()> {
1750    loop {
1751        shared
1752            .metrics
1753            .record_turn(highpri_rx.len(), lowpri_rx.len());
1754
1755        let ctl = tokio::select! {
1756            biased;
1757            Some(cmd) = highpri_rx.recv() => {
1758                let turn = Turn::start(cmd.kind(), &shared);
1759                cmd.execute(&conn, &*clock, &turn).await
1760            }
1761            Some(cmd) = lowpri_rx.recv() => {
1762                let turn = Turn::start(cmd.kind(), &shared);
1763                cmd.execute(&conn, &*clock, &turn).await
1764            }
1765            else => LoopCtl::Break,
1766        };
1767        if matches!(ctl, LoopCtl::Break) {
1768            break;
1769        }
1770    }
1771    Ok(())
1772}
1773
1774/// One command's hold: the timer, its label, and the counters it reports to.
1775///
1776/// # The hold is recorded *before* the caller is answered, and it has to be
1777///
1778/// The obvious placement — time the whole `execute` call from the loop — is
1779/// wrong in a way that only shows up under test. Every arm of `execute` ends by
1780/// sending on a `oneshot`, which wakes the waiting caller; the actor then
1781/// returns to the loop and records. Those are two tasks, so a caller that awaits
1782/// its own write and immediately reads [`Database::metrics`] can be scheduled
1783/// first and see a turn count that does not include the write it just did.
1784///
1785/// Not a correctness bug in the ledger, and it would never have been noticed in
1786/// production — a dashboard sampling every few seconds cannot see the window.
1787/// It makes every test and diagnostic of the counters flaky, which is worse: the
1788/// instrumentation would have been *believed* while being wrong exactly when
1789/// someone tried to check it. `examples/bulk_atomic_diag.rs` was the thing that
1790/// caught it, reporting a 20,000-row batch as a 0 ms hold.
1791///
1792/// So `answer` records and then sends, in that order, and the ordering is the
1793/// method's whole reason to exist. What it costs is that the `oneshot::send`
1794/// itself falls outside the measurement, which is a few nanoseconds against a
1795/// turn measured in microseconds at best.
1796struct Turn<'a> {
1797    kind: crate::metrics::CommandKind,
1798    timer: crate::metrics::HoldTimer,
1799    shared: &'a ActorShared,
1800}
1801
1802/// State the actor owns and a `Turn` needs to reach.
1803///
1804/// `archive_epoch` is here rather than in [`crate::metrics::ActorMetrics`]
1805/// because it is **not** a metric: T1.2's shadow rebuild reads it to decide
1806/// whether its work is still valid, so it has to be present in every build, not
1807/// only under the `metrics` feature. Counting archives happens to be what both
1808/// want; only one of them is allowed to be compiled out.
1809#[derive(Default)]
1810struct ActorShared {
1811    metrics: crate::metrics::ActorMetrics,
1812    archive_epoch: std::sync::atomic::AtomicU64,
1813}
1814
1815impl<'a> Turn<'a> {
1816    fn start(kind: crate::metrics::CommandKind, shared: &'a ActorShared) -> Self {
1817        Self {
1818            kind,
1819            timer: crate::metrics::HoldTimer::start(),
1820            shared,
1821        }
1822    }
1823
1824    fn epoch(&self) -> u64 {
1825        self.shared
1826            .archive_epoch
1827            .load(std::sync::atomic::Ordering::Relaxed)
1828    }
1829
1830    /// Record that an archive session committed.
1831    ///
1832    /// Bumped on **success only**: a failed archive rolls back, so it deletes
1833    /// nothing and invalidates no shadow build.
1834    fn archive_committed(&self) {
1835        self.shared
1836            .archive_epoch
1837            .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
1838    }
1839
1840    /// Close the hold and hand the result back. Never the other way round.
1841    ///
1842    /// The `let _ =` on the send is deliberate and predates this: a caller that
1843    /// dropped its receiver — `tokio::time::timeout` around a write, which
1844    /// [`Database`]'s write surface explicitly documents — is not an actor
1845    /// error, and the command committed regardless.
1846    fn answer<T>(&self, responder: oneshot::Sender<Result<T>>, res: Result<T>) {
1847        self.shared
1848            .metrics
1849            .record_hold(self.kind, self.timer.elapsed());
1850        let _ = responder.send(res);
1851    }
1852}
1853
1854const INSERT_LINK: &str = "INSERT INTO links \
1855     (source_id, target_id, edge_type, valid_from, valid_to, weight, properties, recorded_at) \
1856     VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)";
1857
1858/// Shared by the single-concept write and the chunked one, so the two paths
1859/// cannot drift into upserting different column sets — and so the chunk has a
1860/// statement text it can prepare once (D-056).
1861const UPSERT_CONCEPT: &str = "INSERT INTO concepts \
1862     (id, title, content, embedding_model, valid_from, valid_to, recorded_at, retired) \
1863     VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8) \
1864     ON CONFLICT(id) DO UPDATE SET \
1865         title = excluded.title, \
1866         content = excluded.content, \
1867         embedding_model = excluded.embedding_model, \
1868         valid_from = excluded.valid_from, \
1869         valid_to = excluded.valid_to, \
1870         recorded_at = excluded.recorded_at, \
1871         retired = excluded.retired";
1872
1873/// The parameter row for [`UPSERT_CONCEPT`], in one place for the same reason.
1874fn concept_params<'a>(concept: &'a ConceptUpsert, stamp: &'a str) -> [libsql::Value; 8] {
1875    [
1876        concept.id.as_str().into(),
1877        concept.title.as_str().into(),
1878        concept.content.as_str().into(),
1879        concept
1880            .embedding_model
1881            .as_deref()
1882            .map_or(libsql::Value::Null, Into::into),
1883        concept.valid_from.as_str().into(),
1884        concept.valid_to.as_str().into(),
1885        stamp.into(),
1886        (concept.retired as i64).into(),
1887    ]
1888}
1889
1890impl HighPriCommand {
1891    /// The metrics label for this variant (T1.4).
1892    ///
1893    /// Exhaustive for the same reason `execute` is: a new variant that silently
1894    /// borrowed another's label would attribute its holds to the wrong command,
1895    /// and the one question the counters exist to answer is *which* command
1896    /// broke the budget.
1897    fn kind(&self) -> crate::metrics::CommandKind {
1898        use crate::metrics::CommandKind as K;
1899        match self {
1900            HighPriCommand::AssertEdge { .. } => K::AssertEdge,
1901            HighPriCommand::RetireEdge { .. } => K::RetireEdge,
1902            HighPriCommand::UpsertConcept { .. } => K::UpsertConcept,
1903            HighPriCommand::WriteBulkAtomic { .. } => K::WriteBulkAtomic,
1904            HighPriCommand::RebuildCurrent { .. } => K::RebuildCurrent,
1905            HighPriCommand::RegisterModel { .. } => K::RegisterModel,
1906            HighPriCommand::Shutdown { .. } => K::Shutdown,
1907        }
1908    }
1909
1910    /// Run one command and answer its caller.
1911    ///
1912    /// Deliberately exhaustive — there is no `_` arm. The 0.4.5–0.5.4 actor
1913    /// matched `Shutdown` and `AssertEdge` and sent everything else to
1914    /// `_ => LoopCtl::Continue`, which **dropped the responder**: the caller's
1915    /// `rx.await` resolved to a `RecvError` that no code mapped, so four of six
1916    /// commands were indistinguishable from a hung database. An exhaustive match
1917    /// makes that failure a compile error instead of a runtime silence, which is
1918    /// why adding a variant should break this function.
1919    async fn execute(
1920        self,
1921        conn: &libsql::Connection,
1922        clock: &dyn Clock,
1923        turn: &Turn<'_>,
1924    ) -> LoopCtl {
1925        match self {
1926            HighPriCommand::Shutdown { responder } => {
1927                turn.answer(responder, Ok(()));
1928                return LoopCtl::Break;
1929            }
1930            HighPriCommand::AssertEdge { edge, responder } => {
1931                let stamp = clock.now();
1932                if let Err(e) = reject_overlapping_interval(conn, &edge).await {
1933                    turn.answer(responder, Err(e));
1934                    return LoopCtl::Continue;
1935                }
1936                let res = match conn
1937                    .execute(
1938                        INSERT_LINK,
1939                        libsql::params![
1940                            edge.source.as_str(),
1941                            edge.target.as_str(),
1942                            edge.edge_type.as_str(),
1943                            edge.valid_from.as_str(),
1944                            edge.valid_to.as_str(),
1945                            edge.weight,
1946                            edge.properties.as_str(),
1947                            stamp.as_str()
1948                        ],
1949                    )
1950                    .await
1951                {
1952                    Ok(_) => Ok(()),
1953                    Err(e) => Err(classify(
1954                        conn,
1955                        e,
1956                        WriteOp::Edge {
1957                            source_id: &edge.source,
1958                            target_id: &edge.target,
1959                            edge_type: &edge.edge_type,
1960                        },
1961                    )
1962                    .await),
1963                };
1964                turn.answer(responder, res);
1965            }
1966            HighPriCommand::RetireEdge {
1967                source,
1968                target,
1969                edge_type,
1970                valid_from,
1971                valid_to,
1972                responder,
1973            } => {
1974                let stamp = clock.now();
1975                let res = retire_edge(
1976                    conn,
1977                    &source,
1978                    &target,
1979                    &edge_type,
1980                    &valid_from,
1981                    &valid_to,
1982                    &stamp,
1983                )
1984                .await;
1985                turn.answer(responder, res);
1986            }
1987            HighPriCommand::UpsertConcept { concept, responder } => {
1988                let stamp = clock.now();
1989                let res = upsert_concept(conn, &concept, &stamp).await;
1990                turn.answer(responder, res);
1991            }
1992            HighPriCommand::WriteBulkAtomic { edges, responder } => {
1993                // One stamp for the whole batch (D-014): the rows were asserted
1994                // by one act, and giving them different transaction times would
1995                // invent an ordering the caller never expressed.
1996                let stamp = clock.now();
1997                let res = write_edges_atomic(conn, &edges, &stamp).await;
1998                turn.answer(responder, res);
1999            }
2000            HighPriCommand::RebuildCurrent { responder } => {
2001                turn.answer(responder, rebuild_current(conn).await);
2002            }
2003            HighPriCommand::RegisterModel {
2004                model,
2005                dim,
2006                responder,
2007            } => {
2008                turn.answer(
2009                    responder,
2010                    crate::vector::register_model(conn, &model, dim).await,
2011                );
2012            }
2013        }
2014        LoopCtl::Continue
2015    }
2016}
2017
2018impl LowPriCommand {
2019    /// The metrics label for this variant (T1.4). See [`HighPriCommand::kind`].
2020    fn kind(&self) -> crate::metrics::CommandKind {
2021        use crate::metrics::CommandKind as K;
2022        match self {
2023            LowPriCommand::WriteConceptsChunk { .. } => K::WriteConceptsChunk,
2024            LowPriCommand::WriteAnalyticsChunk { .. } => K::WriteAnalyticsChunk,
2025            LowPriCommand::UpsertEmbeddingChunk { .. } => K::UpsertEmbeddingChunk,
2026            LowPriCommand::BulkImportChunk { .. } => K::BulkImportChunk,
2027            LowPriCommand::Archive { .. } => K::Archive,
2028            LowPriCommand::RebuildFts { .. } => K::RebuildFts,
2029            LowPriCommand::ShadowRebuild { .. } => K::ShadowRebuild,
2030        }
2031    }
2032
2033    /// Run one background command and answer its caller.
2034    ///
2035    /// Also exhaustive. The pre-0.5.4 version was a single `LoopCtl::Continue`
2036    /// for *every* variant — every background write silently discarded, its
2037    /// caller waiting forever.
2038    async fn execute(
2039        self,
2040        conn: &libsql::Connection,
2041        clock: &dyn Clock,
2042        turn: &Turn<'_>,
2043    ) -> LoopCtl {
2044        match self {
2045            LowPriCommand::BulkImportChunk { chunk, responder } => {
2046                // A stamp per chunk, not per batch: the chunks commit
2047                // separately, so a shared stamp would claim a simultaneity the
2048                // storage does not have.
2049                let stamp = clock.now();
2050                turn.answer(responder, write_edges_atomic(conn, &chunk, &stamp).await);
2051            }
2052            LowPriCommand::WriteConceptsChunk { chunk, responder } => {
2053                let stamp = clock.now();
2054                turn.answer(responder, write_concepts_atomic(conn, &chunk, &stamp).await);
2055            }
2056            LowPriCommand::WriteAnalyticsChunk { chunk, responder } => {
2057                let stamp = clock.now();
2058                turn.answer(
2059                    responder,
2060                    write_annotations_atomic(conn, &chunk, &stamp).await,
2061                );
2062            }
2063            LowPriCommand::UpsertEmbeddingChunk {
2064                model,
2065                chunk,
2066                responder,
2067            } => {
2068                // No clock reading: an embedding carries no timestamp on either
2069                // axis. It is a derived artifact of a model applied to content
2070                // (Doctrine VII), and the ledger already records when the
2071                // content changed.
2072                turn.answer(
2073                    responder,
2074                    crate::vector::search::upsert_embedding_chunk(conn, &model, &chunk).await,
2075                );
2076            }
2077            LowPriCommand::Archive {
2078                cutoff,
2079                archive_path,
2080                responder,
2081            } => {
2082                // The archive *time*, not the cutoff. `archive_horizon` records
2083                // both and they are different facts — see `archive()` (Wave 4.5).
2084                let archived_at = clock.now();
2085                let res = archive(conn, &cutoff, &archived_at, &archive_path).await;
2086                // Before the answer, so a shadow rebuild that reads the epoch on
2087                // its next turn cannot miss an archive that has already deleted
2088                // rows out from under it (T1.2).
2089                if res.is_ok() {
2090                    turn.archive_committed();
2091                }
2092                turn.answer(responder, res);
2093            }
2094            LowPriCommand::ShadowRebuild { step, responder } => {
2095                use crate::integrity::{shadow, ShadowOutcome, ShadowStep};
2096                let res = match step {
2097                    ShadowStep::Begin => {
2098                        shadow::begin(conn)
2099                            .await
2100                            .map(|build_start| ShadowOutcome::Started {
2101                                build_start,
2102                                epoch: turn.epoch(),
2103                            })
2104                    }
2105                    ShadowStep::Fill { after } => shadow::fill_chunk(conn, after.as_deref())
2106                        .await
2107                        .map(|last| ShadowOutcome::Filled { last }),
2108                    ShadowStep::Swap { build_start, epoch } => {
2109                        shadow::swap(conn, &build_start, epoch, turn.epoch())
2110                            .await
2111                            .map(|rows| ShadowOutcome::Swapped { rows })
2112                    }
2113                };
2114                turn.answer(responder, res);
2115            }
2116            LowPriCommand::RebuildFts { responder } => {
2117                let res = conn
2118                    .execute(crate::schema::ddl::REBUILD_CONCEPTS_FTS, ())
2119                    .await
2120                    .map(|_| ())
2121                    .map_err(Into::into);
2122                turn.answer(responder, res);
2123            }
2124        }
2125        LoopCtl::Continue
2126    }
2127}
2128
2129/// Close an open interval by asserting its successor (Doctrine III).
2130///
2131/// Never an `UPDATE`. The replacement row copies weight and properties from
2132/// current belief and differs only in `valid_to` and `recorded_at`, so the
2133/// original assertion survives intact and `reconstruct` at an earlier instant
2134/// still sees the interval open — which is the entire point of a bitemporal
2135/// ledger.
2136async fn retire_edge(
2137    conn: &libsql::Connection,
2138    source: &str,
2139    target: &str,
2140    edge_type: &str,
2141    valid_from: &str,
2142    valid_to: &str,
2143    stamp: &str,
2144) -> Result<()> {
2145    let affected = conn
2146        .execute(
2147            "INSERT INTO links \
2148                 (source_id, target_id, edge_type, valid_from, valid_to, weight, properties, recorded_at) \
2149             SELECT source_id, target_id, edge_type, valid_from, ?5, weight, properties, ?6 \
2150             FROM links_current \
2151             WHERE source_id = ?1 AND target_id = ?2 AND edge_type = ?3 AND valid_from = ?4",
2152            libsql::params![source, target, edge_type, valid_from, valid_to, stamp],
2153        )
2154        .await
2155        .map_err(DbError::Engine)?;
2156
2157    if affected == 0 {
2158        return Err(DbError::NotFound(format!(
2159            "{source} -> {target} ({edge_type}) at {valid_from}"
2160        )));
2161    }
2162    Ok(())
2163}
2164
2165async fn upsert_concept(
2166    conn: &libsql::Connection,
2167    concept: &ConceptUpsert,
2168    stamp: &str,
2169) -> Result<()> {
2170    let res = conn
2171        .execute(UPSERT_CONCEPT, concept_params(concept, stamp))
2172        .await;
2173
2174    match res {
2175        Ok(_) => Ok(()),
2176        Err(e) => Err(classify(
2177            conn,
2178            e,
2179            WriteOp::Concept {
2180                id: &concept.id,
2181                recorded_at: stamp,
2182            },
2183        )
2184        .await),
2185    }
2186}
2187
2188/// Every recorded interval for one relationship key, for [`Interval::overlaps`]
2189/// to judge.
2190///
2191/// **Three equalities and nothing else, deliberately — and the "and nothing
2192/// else" was measured, not assumed.** The first version added
2193/// `AND valid_from < :new_valid_to`, a provably safe narrowing (overlap requires
2194/// `max(start) < min(end)`, so an interval starting at or after the new one's end
2195/// cannot overlap it). It cost **9.8 ms on a 90-edge chunk into a 2,000-edge
2196/// hub**, because it walked the planner straight into D-059's trap:
2197///
2198/// ```text
2199/// with the range:     SEARCH links_current USING COVERING INDEX
2200///                     idx_lc_traversal_cover (source_id=? AND valid_from<?)
2201/// without it:         SEARCH links_current USING COVERING INDEX
2202///                     idx_lc_open_interval (source_id=? AND target_id=? AND edge_type=?)
2203/// ```
2204///
2205/// `idx_lc_traversal_cover` leads on `(source_id, valid_from, …)` and contains
2206/// every column this query mentions, so with a `valid_from` range available it
2207/// wins as a covering index while binding **one** equality column — and the
2208/// guard scans the source's entire out-degree. That is the same shape as the
2209/// defect D-059 diagnosed in `trg_links_single_open`, reintroduced by an
2210/// optimisation, one wave after it was fixed.
2211///
2212/// Dropping the range makes the query a pure three-column point lookup that
2213/// `idx_lc_open_interval` serves exactly, and the rows it returns are the
2214/// intervals recorded for one `(source, target, edge_type)` — a version count,
2215/// not an out-degree. **A narrowing predicate is not free if it changes the
2216/// plan**, which is the general lesson and the reason this constant carries its
2217/// own `EXPLAIN` output.
2218const OVERLAP_CANDIDATES: &str = "SELECT valid_from, valid_to FROM links_current \
2219     WHERE source_id = ?1 AND target_id = ?2 AND edge_type = ?3 \
2220       AND valid_from <> ?4";
2221
2222/// Whether this pair is the storage layer's case rather than this guard's.
2223///
2224/// Two **open** intervals overlap — they share every instant from the later
2225/// start onwards — so a naive overlap check reports them, and reporting them
2226/// here would leave `DbError::SingleOpenViolation` constructible by nothing.
2227/// That variant is the more specific error, it is enforced by
2228/// `trg_links_single_open` rather than by this function, and its field names
2229/// were ratified in §1.2. Shadowing it with a general one would be defect Q's
2230/// shape reintroduced by a fix: a typed error that no code path can produce.
2231///
2232/// So the two guards partition the space rather than overlapping it. Both open
2233/// belongs to the trigger. Everything else — open against closed, closed against
2234/// closed — is unguarded at the storage layer and belongs here. That the split
2235/// is exactly the trigger's `WHEN` clause is not a coincidence; it is the
2236/// definition of what was missing.
2237fn defer_to_single_open(proposed: &Interval, existing: &Interval) -> bool {
2238    proposed.is_open() && existing.is_open()
2239}
2240
2241/// Refuse an assertion whose valid-time interval overlaps one already recorded
2242/// for the same `(source, target, edge_type)` — **defect AA, D-060**.
2243///
2244/// `trg_links_single_open` fires only `WHEN NEW.valid_to = '9999-…'`, so it
2245/// guards the open sentinel and nothing else. Two *closed* intervals that
2246/// overlap were accepted without complaint, and `query_as_of_edges` at an
2247/// instant inside both returned one relationship as two edges.
2248///
2249/// **This runs in the write actor, which is what makes it sound.** The obvious
2250/// place is `EdgeAssertion::normalized`, and it cannot go there — `normalized`
2251/// is a pure function with no connection, and doing the read at the API boundary
2252/// instead would leave a check-then-write race between the read and the actor's
2253/// insert. Inside the actor there is one writer by construction (D-014), and for
2254/// the batch paths this runs inside the same transaction as the insert, so the
2255/// window does not exist rather than being small.
2256///
2257/// **What it does not cover, and §4.2 now says so:** raw SQL against the same
2258/// file. The storage layer permits what this API refuses, which is the honest
2259/// cost of not putting the check in a trigger. The alternative was a second
2260/// index probe inside `trg_links_single_open` on every insert — on the path
2261/// D-059 has just finished making fast — for a guarantee that only holds against
2262/// callers who were going through the actor anyway.
2263///
2264/// `valid_from <> ?4` excludes the row being re-asserted. Re-assertion at the
2265/// same `valid_from` is Doctrine III's ordinary case — a new belief about the
2266/// same interval — and is settled by the primary key and the single-open
2267/// trigger, not here.
2268/// The single-assertion path prepares one statement for one check, which is what
2269/// `AssertEdge` needs; the batch path prepares once and calls
2270/// [`check_prepared`] per row.
2271async fn reject_overlapping_interval(
2272    conn: &libsql::Connection,
2273    edge: &EdgeAssertion,
2274) -> Result<()> {
2275    let stmt = conn.prepare(OVERLAP_CANDIDATES).await?;
2276    check_prepared(&stmt, edge).await
2277}
2278
2279/// The guard's body, against a statement the caller has already prepared.
2280///
2281/// **Split out because preparing per row was worth 10.4 ms on a 90-edge chunk**
2282/// (§8.8) — the same defect D-056 and D-057 diagnosed and fixed for
2283/// `INSERT_LINK`, reintroduced by the Wave 2 guard that was written beside it.
2284/// Measured with and without the guard, on a 2,000-edge hub: 8.65 ms → 19.25 ms,
2285/// and *identical* with and without `idx_lc_open_interval`, which is what
2286/// identified preparation rather than a scan as the cost. A guard that reads an
2287/// index correctly and prepares its statement 90 times is indistinguishable, at
2288/// the call site, from one that scans.
2289///
2290/// `reset()` between rows is not optional: libsql binds and steps without
2291/// resetting, so a reused statement must be returned to its initial state.
2292async fn check_prepared(stmt: &libsql::Statement, edge: &EdgeAssertion) -> Result<()> {
2293    let proposed = Interval::new(edge.valid_from.clone(), edge.valid_to.clone());
2294
2295    stmt.reset();
2296    let mut rows = stmt
2297        .query(libsql::params![
2298            edge.source.as_str(),
2299            edge.target.as_str(),
2300            edge.edge_type.as_str(),
2301            edge.valid_from.as_str()
2302        ])
2303        .await?;
2304
2305    while let Some(row) = rows.next().await? {
2306        let existing = Interval::new(row.get::<String>(0)?, row.get::<String>(1)?);
2307        if defer_to_single_open(&proposed, &existing) {
2308            continue;
2309        }
2310        if proposed.overlaps(&existing) {
2311            return Err(DbError::OverlappingInterval {
2312                overlap: Box::new(crate::error::Overlap {
2313                    source_id: edge.source.clone(),
2314                    target_id: edge.target.clone(),
2315                    edge_type: edge.edge_type.clone(),
2316                    valid_from: edge.valid_from.clone(),
2317                    valid_to: edge.valid_to.clone(),
2318                    existing_from: existing.valid_from,
2319                    existing_to: existing.valid_to,
2320                }),
2321            });
2322        }
2323    }
2324
2325    Ok(())
2326}
2327
2328/// The same guard applied *within* a batch, before any of it is written.
2329///
2330/// The database check cannot see rows that are not in the database yet, so a
2331/// batch carrying two overlapping intervals for one relationship would pass
2332/// every per-row check and commit the overlap in one transaction. Quadratic in
2333/// the batch, which is affordable because the chunk is bounded at
2334/// [`chunk_rows::EDGES`] = 90 and because the comparison is a pair of string
2335/// compares — and because grouping first means the inner loop only ever runs
2336/// over edges sharing a key, which is normally one.
2337fn reject_overlaps_within(edges: &[EdgeAssertion]) -> Result<()> {
2338    for (i, a) in edges.iter().enumerate() {
2339        let ia = Interval::new(a.valid_from.clone(), a.valid_to.clone());
2340        for b in &edges[i + 1..] {
2341            if a.source != b.source || a.target != b.target || a.edge_type != b.edge_type {
2342                continue;
2343            }
2344            // Identical valid_from is re-assertion within one batch: the last
2345            // writer wins by seq_id, as it does across batches. Not an overlap.
2346            if a.valid_from == b.valid_from {
2347                continue;
2348            }
2349            let ib = Interval::new(b.valid_from.clone(), b.valid_to.clone());
2350            // Both open is the trigger's case; it fires during the insert and
2351            // rolls the batch back with the more specific error.
2352            if defer_to_single_open(&ia, &ib) {
2353                continue;
2354            }
2355            if ia.overlaps(&ib) {
2356                return Err(DbError::OverlappingInterval {
2357                    overlap: Box::new(crate::error::Overlap {
2358                        source_id: a.source.clone(),
2359                        target_id: a.target.clone(),
2360                        edge_type: a.edge_type.clone(),
2361                        valid_from: a.valid_from.clone(),
2362                        valid_to: a.valid_to.clone(),
2363                        existing_from: ib.valid_from,
2364                        existing_to: ib.valid_to,
2365                    }),
2366                });
2367            }
2368        }
2369    }
2370    Ok(())
2371}
2372
2373/// Write every edge or none, under a single stamp.
2374///
2375/// **The statement is prepared once for the whole chunk (§9, D-056).** It used to
2376/// be `tx.execute(INSERT_LINK, …)` per row, which re-prepares on every call — and
2377/// `links` carries two triggers, so each preparation compiles their bodies along
2378/// with the insert.
2379///
2380/// Measured at 500 rows: **≈62 ms → ≈37 ms, a 41% saving.** Preparation was a
2381/// large cost and *not* the dominant one, which the first guess had it as. The
2382/// residual is the triggers themselves: the same 500 rows with
2383/// `trg_links_log_insert` and `trg_links_current_sync` dropped commit in **2.96
2384/// ms**, so trigger amplification is ~92% of what remains. There is no further
2385/// win available here without changing what the ledger records, and Doctrine IV
2386/// is what says it must be recorded. See D-056 for what that implies about §9's
2387/// ≤ 3 ms budget — briefly, 2.96 ms *is* the un-amplified figure, so the budget
2388/// appears to have been set without the amplification its own preamble says is
2389/// included.
2390///
2391/// `reset()` between rows is not optional: libsql's `execute` binds and steps
2392/// without resetting, so a reused statement must be returned to its initial state
2393/// or the second row steps a completed statement.
2394async fn write_edges_atomic(
2395    conn: &libsql::Connection,
2396    edges: &[EdgeAssertion],
2397    stamp: &str,
2398) -> Result<usize> {
2399    if edges.is_empty() {
2400        return Ok(0);
2401    }
2402
2403    // Before the transaction opens: a batch that contradicts itself is refused
2404    // without taking the write lock at all (D-060).
2405    reject_overlaps_within(edges)?;
2406
2407    let tx = conn
2408        .transaction_with_behavior(libsql::TransactionBehavior::Immediate)
2409        .await?;
2410
2411    // Inside the transaction, so the rows this checks against cannot change
2412    // between the check and the insert.
2413    // One preparation for the whole chunk, not one per row — see
2414    // `check_prepared`, and D-056 for the same lesson learned on `INSERT_LINK`.
2415    let guard = tx.prepare(OVERLAP_CANDIDATES).await?;
2416    for edge in edges {
2417        if let Err(e) = check_prepared(&guard, edge).await {
2418            // Released before the rollback: a live statement on the connection
2419            // is what makes SQLite refuse to end a transaction.
2420            drop(guard);
2421            let _ = tx.rollback().await;
2422            return Err(e);
2423        }
2424    }
2425    drop(guard);
2426
2427    let stmt = tx.prepare(INSERT_LINK).await?;
2428
2429    for edge in edges {
2430        stmt.reset();
2431        let res = stmt
2432            .execute(libsql::params![
2433                edge.source.as_str(),
2434                edge.target.as_str(),
2435                edge.edge_type.as_str(),
2436                edge.valid_from.as_str(),
2437                edge.valid_to.as_str(),
2438                edge.weight,
2439                edge.properties.as_str(),
2440                stamp
2441            ])
2442            .await;
2443
2444        if let Err(e) = res {
2445            let typed = classify(
2446                &tx,
2447                e,
2448                WriteOp::Edge {
2449                    source_id: &edge.source,
2450                    target_id: &edge.target,
2451                    edge_type: &edge.edge_type,
2452                },
2453            )
2454            .await;
2455            // Released before the rollback: a live statement on the connection
2456            // is exactly what makes SQLite refuse to end a transaction.
2457            drop(stmt);
2458            let _ = tx.rollback().await;
2459            return Err(typed);
2460        }
2461    }
2462
2463    drop(stmt);
2464    tx.commit().await?;
2465    Ok(edges.len())
2466}
2467
2468/// Write every concept or none, under a single stamp.
2469/// Upsert one chunk of derived annotations in a single transaction (D-041).
2470///
2471/// `stamp` is the actor's clock reading, exactly as for every other chunk — but
2472/// it lands in `computed_at`, not in a `recorded_at`, and the difference is not
2473/// cosmetic. `recorded_at` is the transaction-time axis and is subject to
2474/// Doctrine II and the monotonicity guard; `computed_at` is a note about when a
2475/// derivation last ran, on a table the ledger does not see. Rerunning an
2476/// algorithm therefore replaces the row and advances the note, rather than
2477/// versioning a concept the world did not change.
2478async fn write_annotations_atomic(
2479    conn: &libsql::Connection,
2480    annotations: &[Annotation],
2481    stamp: &str,
2482) -> Result<usize> {
2483    if annotations.is_empty() {
2484        return Ok(0);
2485    }
2486
2487    let tx = conn
2488        .transaction_with_behavior(libsql::TransactionBehavior::Immediate)
2489        .await?;
2490
2491    let stmt = tx
2492        .prepare(
2493            "INSERT INTO analytics_annotations (concept_id, label, value, computed_at) \
2494             VALUES (?1, ?2, ?3, ?4) \
2495             ON CONFLICT(concept_id, label) DO UPDATE SET \
2496                 value = excluded.value, computed_at = excluded.computed_at",
2497        )
2498        .await?;
2499
2500    for a in annotations {
2501        stmt.reset();
2502        let res = stmt
2503            .execute(libsql::params![
2504                a.concept_id.as_str(),
2505                a.label.as_str(),
2506                a.value.as_str(),
2507                stamp
2508            ])
2509            .await;
2510        if let Err(e) = res {
2511            drop(stmt);
2512            let _ = tx.rollback().await;
2513            return Err(DbError::Engine(e));
2514        }
2515    }
2516
2517    drop(stmt);
2518    tx.commit().await?;
2519    Ok(annotations.len())
2520}
2521
2522async fn write_concepts_atomic(
2523    conn: &libsql::Connection,
2524    concepts: &[ConceptUpsert],
2525    stamp: &str,
2526) -> Result<usize> {
2527    if concepts.is_empty() {
2528        return Ok(0);
2529    }
2530
2531    let tx = conn
2532        .transaction_with_behavior(libsql::TransactionBehavior::Immediate)
2533        .await?;
2534
2535    // Prepared once, like the edge chunk (D-056). This no longer routes through
2536    // [`upsert_concept`] — that function prepares per call by construction — but
2537    // it shares that function's statement text and parameter row, so the two
2538    // cannot upsert different columns.
2539    let stmt = tx.prepare(UPSERT_CONCEPT).await?;
2540
2541    for concept in concepts {
2542        stmt.reset();
2543        let res = stmt.execute(concept_params(concept, stamp)).await;
2544
2545        if let Err(e) = res {
2546            let typed = classify(
2547                &tx,
2548                e,
2549                WriteOp::Concept {
2550                    id: &concept.id,
2551                    recorded_at: stamp,
2552                },
2553            )
2554            .await;
2555            drop(stmt);
2556            let _ = tx.rollback().await;
2557            return Err(typed);
2558        }
2559    }
2560
2561    drop(stmt);
2562    tx.commit().await?;
2563    Ok(concepts.len())
2564}
2565
2566#[cfg(test)]
2567mod tests {
2568    use super::*;
2569
2570    fn edge(target: &str, micros: usize) -> EdgeAssertion {
2571        EdgeAssertion::new("src", target, "LINKS")
2572            .valid_from(format!("2026-01-01T00:00:00.{micros:06}Z"))
2573            .valid_to(format!("2026-01-01T00:00:00.{:06}Z", micros + 1))
2574    }
2575
2576    /// The estimate must depend on the batch's **shape**, not only its size.
2577    ///
2578    /// This is the correction T1.3's "rows × per-row cost" needed. Two batches
2579    /// of the same length whose measured holds differ by 7× must not be
2580    /// predicted identically, and the direction matters: a model that averages
2581    /// the two under-predicts the expensive shape, which is the only one anyone
2582    /// needs warning about.
2583    #[test]
2584    fn two_batches_of_one_size_are_not_predicted_alike() {
2585        const N: usize = 20_000;
2586        let fanout: Vec<_> = (0..N).map(|i| edge(&format!("t{i:07}"), i)).collect();
2587        let history: Vec<_> = (0..N).map(|i| edge("t0", i)).collect();
2588
2589        let (a, b) = (estimated_bulk_hold(&fanout), estimated_bulk_hold(&history));
2590        assert!(
2591            b > a * 5,
2592            "the guard's expensive path is 16x dearer per pair and this batch \
2593             takes it on every pair, but the estimates are {a:?} and {b:?}"
2594        );
2595    }
2596
2597    /// Measured on libSQL 0.9.30: 2.5 s and 18.6 s for those two batches. The
2598    /// estimator tracked both within 5%, and this pins that it still does — a
2599    /// coefficient edited without re-measuring fails here.
2600    #[test]
2601    fn the_estimate_matches_what_was_measured() {
2602        const N: usize = 20_000;
2603        let fanout: Vec<_> = (0..N).map(|i| edge(&format!("t{i:07}"), i)).collect();
2604        let history: Vec<_> = (0..N).map(|i| edge("t0", i)).collect();
2605
2606        for (batch, measured_ms, label) in
2607            [(fanout, 2_618u128, "fanout"), (history, 18_057, "history")]
2608        {
2609            let predicted = estimated_bulk_hold(&batch).as_millis();
2610            let ratio = predicted as f64 / measured_ms as f64;
2611            assert!(
2612                (0.8..1.25).contains(&ratio),
2613                "{label}: predicted {predicted} ms against a measured \
2614                 {measured_ms} ms ({ratio:.2}x). Re-run \
2615                 examples/bulk_atomic_diag.rs before changing the coefficients."
2616            );
2617        }
2618    }
2619
2620    /// An empty or single-edge batch has no pairs, and the arithmetic must not
2621    /// underflow computing it.
2622    #[test]
2623    fn a_batch_too_small_to_have_pairs_still_estimates() {
2624        assert_eq!(estimated_bulk_hold(&[]), std::time::Duration::ZERO);
2625        let one = [edge("t0", 0)];
2626        assert_eq!(
2627            estimated_bulk_hold(&one),
2628            std::time::Duration::from_nanos(73_000)
2629        );
2630    }
2631
2632    /// The warning threshold sits well above the bound this path is exempt from.
2633    ///
2634    /// Warning at `CHUNK_BUDGET` would fire on batches working exactly as
2635    /// designed — the exemption is a contract (D-014), not a failure — and a
2636    /// warning that fires on correct behaviour gets filtered out, taking the
2637    /// 18-second case with it.
2638    #[test]
2639    fn the_warning_threshold_is_not_the_chunk_budget() {
2640        assert!(BULK_ATOMIC_WARN_HOLD > CHUNK_BUDGET * 10);
2641    }
2642}