Skip to main content

macrame/
connection.rs

1use std::path::{Path, PathBuf};
2use std::sync::Arc;
3use tokio::sync::{mpsc, oneshot};
4
5use crate::error::{classify, DbError, Result, WriteOp};
6use crate::graph::edge::EdgeAssertion;
7use crate::integrity::{rebuild_current, RebuildReport};
8use crate::schema::migrations;
9use crate::temporal::archive::{archive, rehydrate, ArchiveReport, RehydrateReport};
10use crate::temporal::interval::Interval;
11use crate::temporal::snapshot::{self, SnapshotCadence};
12use crate::util::clock::{Clock, SystemClock};
13use crate::util::timestamp;
14use crate::vector::ModelName;
15
16/// Rows per chunk on the background write paths (§5.1.5, D-011, D-014, D-058).
17///
18/// The Write Actor holds the sole write connection, so a single large statement
19/// blocks every other writer for its duration. Chunking bounds that stall; the
20/// cost is that a bulk import is *not* atomic across chunks, which is why it is
21/// a separate command from [`HighPriCommand::WriteBulkAtomic`] rather than a
22/// tuning parameter on it.
23///
24/// # Why these are four constants and not one
25///
26/// Through 0.5.5 this was a single `CHUNK_ROWS = 1000` for all four bulk paths.
27/// The golden rule it was meant to serve is a bound on *duration* — a background
28/// chunk must commit fast enough that an interactive write queued behind it is
29/// not made to wait — and one row count cannot express one duration across paths
30/// whose measured per-row costs differ by 60× (D-058). At 1,000 rows the four
31/// paths took 3.5 ms, 24 ms, 89 ms and 143 ms: the same constant, four answers,
32/// three of them far outside the bound.
33///
34/// Each size below is derived from `benches/budgets.rs`'s `chunk_scaling`
35/// sweep against [`CHUNK_BUDGET`], then verified by measuring that size directly.
36/// They are *measurements of this machine*, not universal constants — D-055's
37/// reasoning about reference hardware applies here too, and re-deriving them on
38/// materially different storage is a `cargo bench` away.
39///
40/// # Sized for the tail, not the median
41///
42/// The first derivation solved `f + c·n = 3 ms` exactly and produced sizes whose
43/// *median* commit was 2.93 ms and whose upper estimate was 2.96 — inside the
44/// bound as reported and outside it for any chunk slower than typical. A latency
45/// bound is a statement about the chunk an unlucky interactive write actually
46/// queues behind, so these solve for ≈2.5 ms instead, leaving the remainder as
47/// headroom for the tail. That costs a few percent of throughput on the two
48/// linear paths and nothing on the two superlinear ones.
49///
50/// As measured by `chunk_budget`, each at its own size: edges **2.39 ms**,
51/// concepts **2.35 ms**, annotations **2.36 ms**, embeddings **2.06 ms**, no
52/// upper estimate above 2.42.
53///
54/// # Known limitation: these are empty-database figures
55///
56/// `chunk_budget` seeds concepts and starts with **no links and no vectors**,
57/// and D-059 established that per-row cost on the edge and embedding paths grows
58/// with the size of the structure being written, not with the chunk. The same
59/// 90-edge chunk takes 47.7 ms against an 8,000-edge hub. So the bound is met as
60/// measured and *not* met on a large database, most of that gap being the schema
61/// defect D-059 documents. Re-deriving these against a realistic fixture needs a
62/// decision about what "realistic" is, which is why it has not been done
63/// silently.
64pub mod chunk_rows {
65    /// Edge assertions (`bulk_import`).
66    ///
67    /// Per-row cost on this path rises with the size of `links_current`, not
68    /// with the chunk (D-059) — so cutting the chunk buys latency and costs
69    /// throughput, ~11% for 1,000 edges. An earlier version of this comment
70    /// claimed it was 3.3× *faster*; that came from multiplying eleven copies of
71    /// a chunk measured into an empty database.
72    ///
73    /// **This size does not meet the 3 ms bound on a large database.** 90 edges
74    /// into an 8,000-edge hub take 47.7 ms, because `trg_links_single_open`'s
75    /// `EXISTS` is served by `idx_lc_traversal_cover` with only `source_id`
76    /// bound and therefore scans the whole out-degree. That is a schema defect
77    /// with a proven fix, recorded in D-059 and not applied here.
78    pub const EDGES: usize = 90;
79
80    /// Concept upserts (`write_concepts`).
81    ///
82    /// Linear at ~23 µs per row, so unlike [`EDGES`] this size *is* a genuine
83    /// throughput sacrifice: 1,000-row chunks ran at 23.6 µs per row against
84    /// ~35 µs here. Paid deliberately — a 1,000-row chunk takes 24 ms, eight
85    /// times the bound.
86    pub const CONCEPTS: usize = 70;
87
88    /// Analytics annotations (`write_analytics_annotations`).
89    ///
90    /// The one path where the old constant was nearly right, and the only bulk
91    /// table with no triggers at all: ~2.5 µs per row, linear, so the bound buys
92    /// a large chunk. 1,000 rows would be 3.5 ms — over, but only just.
93    pub const ANNOTATIONS: usize = 600;
94
95    /// Embedding vectors (`upsert_embeddings`).
96    ///
97    /// The smallest by a wide margin, because DiskANN index maintenance makes an
98    /// embedding the most expensive row in the system. That cost grows with the
99    /// **corpus**, not the chunk (D-059): a fixed 30-vector chunk costs 49 µs per
100    /// vector into an empty corpus and 224 µs into an 8,000-vector one. Graph
101    /// insertion getting dearer as the graph grows is what DiskANN is, so unlike
102    /// [`EDGES`] there is nothing here to fix — but it does mean this size buys
103    /// latency at some throughput, not for free.
104    pub const EMBEDDINGS: usize = 30;
105}
106
107/// The latency bound [`chunk_rows`] is derived from (§5.1.5, D-058).
108///
109/// This is the golden rule's actual content. §9 has carried it as a row count
110/// with a duration attached — "chunk commit, 500 rows ≤ 3 ms" — which reads as
111/// two requirements and is one: the duration is the requirement, and the row
112/// count is whatever satisfies it on a given path and machine.
113///
114/// 3 ms is §9's number, kept rather than renegotiated. What it buys, end to end:
115/// an interactive assertion arriving at the worst possible moment waits for the
116/// chunk in flight (≤ 3 ms, because the SQLite write lock is not preemptible —
117/// see [`HighPriCommand`]) and then runs its own write (≤ 5 ms, §9), so ≤ 8 ms
118/// worst case. That fits inside a 60 Hz frame with room, which is the standard
119/// this bound is ultimately answerable to.
120///
121/// # Three operations are exempt, and the exemption is a contract, not an oversight
122///
123/// This was recorded in three separate rustdoc notes and nowhere near the bound
124/// itself, which is where a reader looks for its scope (§8.6). Stated here, with
125/// Wave 3's measurements:
126///
127/// | Path | Bound | Why it cannot be chunked |
128/// |---|---|---|
129/// | [`Database::write_bulk_atomic`] | none — caller-sized `Vec` | D-014: the batch is *one act* under one stamp. Splitting it is the thing the method exists not to do |
130/// | [`Database::archive`] | measured **26.8 ms** for 2,000 archivable edges; see [`Database::archive_windowed`] | D-012: copy-then-delete must be atomic, or a crash between the phases duplicates or loses rows |
131/// | `rebuild_current` | measured **24.6 / 104 / 318 ms** at 4K / 16K / 40K rows in `links` (was "~50 s per 10M edges", which nothing had measured) | D-023: the window between `DELETE` and `INSERT` is the whole of current belief; a reader landing in it sees a graph with no edges and no error |
132///
133/// The `archive` figure is end-to-end through this method, so it **includes**
134/// the re-derivation `archive()` runs inside its transaction — but it does not
135/// attribute it, and until D-077 more than half of that re-derivation was an
136/// audit comparing `links_current` against the query that had just filled it.
137/// Note also which variable that cost scales with: `rebuild_within` reprojects
138/// **all of `links`**, so the archive's repair term grows with the *surviving*
139/// table and not with the batch being archived. A budget stated per "100K closed
140/// intervals" ([§9](../docs/architecture/s6-s10-flows-to-dependencies.md)) is
141/// therefore parameterised on the wrong quantity.
142///
143/// All three are atomic **by contract**, which is why "cap the batch" and "add a
144/// third tier" were both considered and neither was taken: capping breaks the
145/// guarantee the operation exists to provide, and a third tier changes which
146/// caller waits without changing how long the lock is held. What was wrong was
147/// never the exemption — it was that the bound was stated as though it had none.
148///
149/// A caller who needs the latency bound and not the atomicity has
150/// [`Database::bulk_import`], which is the same write chunked at
151/// [`chunk_rows::EDGES`] and explicitly *not* atomic overall (D-011).
152///
153/// # One of the three is no longer unbounded (T1.1, D-080)
154///
155/// `archive` was the worst of them, because its hold is a function of *how long
156/// since the last archive* rather than of anything the caller chose.
157/// [`Database::archive_windowed`] runs the same work as N sessions, each
158/// atomic, each its own actor turn. Measured on an 8,000-key fixture with four
159/// generations of superseded history: the longest single hold falls from
160/// **3.3 s to 0.77 s** at one-hour windows, for total wall time that is flat
161/// within this cycle's noise.
162///
163/// The same measurement at 2,000 keys goes the other way — the hold falls
164/// 260 ms → 117 ms while total time rises 260 ms → 671 ms — so windowing is a
165/// trade and not a free improvement. It pays when the backlog is large, which
166/// is when the unwindowed hold is a problem in the first place. `archive` is
167/// kept, not deprecated, for exactly that reason.
168pub const CHUNK_BUDGET: std::time::Duration = std::time::Duration::from_millis(3);
169
170/// Predicted hold above which [`Database::write_bulk_atomic`] warns (T1.3).
171///
172/// 250 ms is fifteen frames at 60 Hz: not a hitch, a visible freeze. It is well
173/// above [`CHUNK_BUDGET`] on purpose — this path is exempt from that bound by
174/// contract, so warning at 3 ms would fire on batches that are working exactly
175/// as designed and train the reader to filter the message out.
176pub const BULK_ATOMIC_WARN_HOLD: std::time::Duration = std::time::Duration::from_millis(250);
177
178/// Roughly how long [`Database::write_bulk_atomic`] will hold the actor for
179/// this batch (T1.3, D-081).
180///
181/// # Three terms, because the cost is neither linear nor a function of size
182///
183/// T1.3 asks for "rows × measured per-row cost". That model is wrong twice over,
184/// and both corrections came out of measuring it.
185///
186/// First, the cost is not linear. `write_edges_atomic` opens with
187/// `reject_overlaps_within`, which compares **every pair** in the batch before a
188/// row is written. Second — and this is the one that matters — the quadratic
189/// term's constant depends on the batch's *shape*, not its size. The pairwise
190/// loop starts with an early `continue` on mismatched `(source, target,
191/// edge_type)`; pairs that share all three fall through to `Interval::new` and
192/// `overlaps`, which is **sixteen times** dearer per pair.
193///
194/// ```text
195/// hold ≈ 73 µs · rows  +  5.5 ns · mismatched pairs  +  86 ns · matching pairs
196/// ```
197///
198/// Two batches of 20,000 edges, measured on the same machine: one fanning out to
199/// distinct targets holds the actor for **2.5 s**, and one asserting 20,000
200/// corrections to a single relationship's history holds it for **18.6 s**. A
201/// size-only model is off by 7× between those two, in the direction that
202/// matters — it under-predicts the bad case. So this counts the matching pairs
203/// rather than guessing, with one `HashMap` pass over the batch. That pass is
204/// O(rows) against an operation about to spend milliseconds per row.
205///
206/// # What this is calibrated against, and where it will be wrong
207///
208/// libSQL 0.9.30, one machine, best of three, over 100–20,000 rows in both
209/// shapes; within 5% across that range except below ~500 rows, where fixed costs
210/// dominate and it over-predicts by 3× — harmless, since nothing that small can
211/// approach [`BULK_ATOMIC_WARN_HOLD`].
212///
213/// It is machine-specific and says nothing about disk. It exists to turn
214/// "uncapped" into an order of magnitude a caller can act on — the difference
215/// between 30 ms and 18 s — and should not be read more precisely than that.
216/// `examples/bulk_atomic_diag.rs` prints predicted against measured, so the
217/// model's drift is visible rather than assumed.
218pub fn estimated_bulk_hold(edges: &[EdgeAssertion]) -> std::time::Duration {
219    let rows = edges.len() as u64;
220    let all_pairs = rows.saturating_mul(rows.saturating_sub(1)) / 2;
221
222    // Pairs sharing all three key columns, which is exactly the set that reaches
223    // the guard's expensive path. Grouped rather than sorted: the batch is
224    // borrowed, and sorting would either clone it or reorder the caller's data.
225    let mut groups: std::collections::HashMap<(&str, &str, &str), u64> =
226        std::collections::HashMap::new();
227    for e in edges {
228        *groups
229            .entry((&e.source, &e.target, &e.edge_type))
230            .or_insert(0) += 1;
231    }
232    let matching: u64 = groups.values().map(|&g| g * (g - 1) / 2).sum();
233    let mismatched = all_pairs - matching;
234
235    // Nanoseconds throughout, saturating: a caller who passes a batch large
236    // enough to overflow this has a problem the arithmetic cannot express, and
237    // saturating to ~584 years still crosses every threshold above.
238    std::time::Duration::from_nanos(
239        (73_000u64.saturating_mul(rows))
240            .saturating_add(mismatched.saturating_mul(11) / 2)
241            .saturating_add(matching.saturating_mul(86)),
242    )
243}
244
245/// Most sessions [`Database::archive_windowed`] will run for one call (T1.1).
246///
247/// A limit exists because the session count is a function of *transaction-time
248/// span divided by window*, and both come from the caller — a one-second window
249/// over a decade of history is ten million actor turns, each opening a
250/// transaction and writing a horizon row. That is not a slow archive, it is a
251/// caller who meant something else.
252///
253/// 4,096 is chosen against the operation it bounds rather than against a clock:
254/// at the measured 26.8 ms for a session with work in it, a full run of this
255/// many is about two minutes of background writing, and the whole point of
256/// windowing is that those two minutes are interruptible. It is a refusal
257/// rather than a clamp — see [`DbError::ArchiveWindow`] for why.
258pub const MAX_ARCHIVE_SESSIONS: usize = 4_096;
259
260/// A concept assertion: the payload of an upsert.
261#[derive(Debug, Clone, PartialEq)]
262pub struct ConceptUpsert {
263    pub id: String,
264    pub title: String,
265    pub content: String,
266    pub embedding_model: Option<String>,
267    pub valid_from: String,
268    pub valid_to: String,
269    pub retired: bool,
270}
271
272impl ConceptUpsert {
273    pub fn new(id: impl Into<String>, title: impl Into<String>) -> Self {
274        Self {
275            id: id.into(),
276            title: title.into(),
277            content: String::new(),
278            embedding_model: None,
279            valid_from: String::new(),
280            valid_to: timestamp::OPEN_SENTINEL.to_string(),
281            retired: false,
282        }
283    }
284
285    pub fn content(mut self, content: impl Into<String>) -> Self {
286        self.content = content.into();
287        self
288    }
289
290    pub fn embedding_model(mut self, model: impl Into<String>) -> Self {
291        self.embedding_model = Some(model.into());
292        self
293    }
294
295    pub fn valid_from(mut self, ts: impl Into<String>) -> Self {
296        self.valid_from = ts.into();
297        self
298    }
299
300    pub fn valid_to(mut self, ts: impl Into<String>) -> Self {
301        self.valid_to = ts.into();
302        self
303    }
304
305    pub fn retired(mut self, retired: bool) -> Self {
306        self.retired = retired;
307        self
308    }
309
310    /// Put the timestamps in canonical form (D-029) before they cross the channel.
311    pub fn normalized(mut self) -> Result<Self> {
312        crate::util::ids::validate_id(&self.id)?;
313        self.valid_from = timestamp::normalize(&self.valid_from)?;
314        self.valid_to = timestamp::normalize(&self.valid_to)?;
315        Ok(self)
316    }
317}
318
319/// One derived analytics result for one concept (§5.4, D-041).
320///
321/// Not a `ConceptUpsert`. The distinction is the whole of D-041: a concept
322/// upsert is a statement about the world and belongs in the ledger, while an
323/// annotation is a function of an algorithm applied to a graph and belongs in
324/// `analytics_annotations`, which carries no log trigger. Writing one as the
325/// other overwrote the concept's `content` with the label and recorded every
326/// analytics rerun as a fresh version of the world.
327#[derive(Debug, Clone, PartialEq, Eq)]
328pub struct Annotation {
329    pub concept_id: String,
330    /// Namespaced by convention, e.g. `louvain.community`, `kcore.shell`.
331    pub label: String,
332    /// JSON-encoded payload. Opaque to this crate.
333    pub value: String,
334}
335
336impl Annotation {
337    pub fn new(
338        concept_id: impl Into<String>,
339        label: impl Into<String>,
340        value: impl Into<String>,
341    ) -> Self {
342        Self {
343            concept_id: concept_id.into(),
344            label: label.into(),
345            value: value.into(),
346        }
347    }
348}
349
350/// Commands sent to the Write Actor on the high-priority channel (UI-driven work).
351pub enum HighPriCommand {
352    AssertEdge {
353        edge: EdgeAssertion,
354        responder: oneshot::Sender<Result<()>>,
355    },
356    RetireEdge {
357        source: String,
358        target: String,
359        edge_type: String,
360        valid_from: String,
361        valid_to: String,
362        responder: oneshot::Sender<Result<()>>,
363    },
364    UpsertConcept {
365        concept: ConceptUpsert,
366        responder: oneshot::Sender<Result<()>>,
367    },
368    WriteBulkAtomic {
369        edges: Vec<EdgeAssertion>,
370        responder: oneshot::Sender<Result<usize>>,
371    },
372    RebuildCurrent {
373        responder: oneshot::Sender<Result<RebuildReport>>,
374    },
375    /// Create a model's embedding table and its DiskANN index (D-037, D-048).
376    ///
377    /// High priority despite being setup work: it is one small transaction, and
378    /// every embedding write for the model blocks on it, so queueing it behind a
379    /// bulk job would stall the thing it gates.
380    RegisterModel {
381        model: ModelName,
382        dim: usize,
383        responder: oneshot::Sender<Result<()>>,
384    },
385    Shutdown {
386        responder: oneshot::Sender<Result<()>>,
387    },
388}
389
390/// Commands sent to the Write Actor on the low-priority channel (background work).
391pub enum LowPriCommand {
392    /// One chunk of **concepts** — a ledger write, logged and versioned.
393    WriteConceptsChunk {
394        chunk: Vec<ConceptUpsert>,
395        responder: oneshot::Sender<Result<usize>>,
396    },
397    /// One chunk of **derived annotations** — off-ledger, no log trigger (D-041).
398    ///
399    /// The pair is named apart deliberately: this variant was `WriteAnalyticsChunk`
400    /// beside a `WriteAnnotationsChunk` that carried concepts, which is the
401    /// crossing D-075 undid.
402    WriteAnalyticsChunk {
403        chunk: Vec<Annotation>,
404        responder: oneshot::Sender<Result<usize>>,
405    },
406    /// One chunk of vectors for one model (§5.9, D-048).
407    ///
408    /// Low priority: embedding is bulk derived work and must never preempt an
409    /// interactive assertion.
410    UpsertEmbeddingChunk {
411        model: ModelName,
412        chunk: Vec<(String, Vec<f32>)>,
413        responder: oneshot::Sender<Result<usize>>,
414    },
415    BulkImportChunk {
416        chunk: Vec<EdgeAssertion>,
417        responder: oneshot::Sender<Result<usize>>,
418    },
419    Archive {
420        cutoff: String,
421        archive_path: PathBuf,
422        responder: oneshot::Sender<Result<ArchiveReport>>,
423    },
424    /// Move named concepts back out of the cold file (0.9.0, C3).
425    ///
426    /// Low priority for the same reason `Archive` is: it is bulk physical
427    /// movement with no latency bound, and it holds the write lock for its whole
428    /// transaction.
429    Rehydrate {
430        ids: Vec<String>,
431        archive_path: PathBuf,
432        responder: oneshot::Sender<Result<RehydrateReport>>,
433    },
434    /// Reconstruct the FTS index from `concepts` (§5.9, D-036, D-051).
435    ///
436    /// Low priority: it is maintenance on a derivative table, and a search index
437    /// that is a few seconds stale is a smaller cost than an interactive write
438    /// that waits behind a full reindex.
439    RebuildFts {
440        responder: oneshot::Sender<Result<()>>,
441    },
442    /// One step of a chunked shadow rebuild (§5.8, T1.2, D-082).
443    ///
444    /// Low priority, and one command per step rather than one per rebuild: the
445    /// whole value of building beside the live table is that the actor returns
446    /// here between chunks. See [`Database::rebuild_current_chunked`].
447    ShadowRebuild {
448        step: crate::integrity::ShadowStep,
449        responder: oneshot::Sender<Result<crate::integrity::ShadowOutcome>>,
450    },
451}
452
453enum LoopCtl {
454    Continue,
455    Break,
456}
457
458/// Primary database handle for Macrame bitemporal ledger.
459pub struct Database {
460    db: libsql::Database,
461    /// The file this handle opened, kept so [`Database::diagnostic_conn`] can
462    /// open it again under different flags (T5.1, D-091). `archive_path` and
463    /// `snapshots_dir` are derived from it and were previously the only trace
464    /// of it on the struct.
465    path: PathBuf,
466    read_conn: libsql::Connection,
467    highpri_tx: mpsc::Sender<HighPriCommand>,
468    lowpri_tx: mpsc::Sender<LowPriCommand>,
469    clock: Arc<dyn Clock>,
470    archive_path: PathBuf,
471    snapshots_dir: PathBuf,
472    schema_version: u32,
473    writer: Option<tokio::task::JoinHandle<Result<()>>>,
474    /// Stops the snapshot cadence. Dropping it stops the task too, which is what
475    /// keeps a `Database` that is dropped rather than closed from leaving a task
476    /// running against a connection whose database is going away.
477    cadence_stop: Option<tokio::sync::watch::Sender<bool>>,
478    cadence: Option<tokio::task::JoinHandle<()>>,
479    /// Set by [`Database::close`]. Read only by [`Drop`], which warns when it is
480    /// still false — see that impl for why the omission is worth a warning.
481    closed: bool,
482    /// Shared with the actor (T1.4, T1.2). Held here rather than behind
483    /// `#[cfg(feature = "metrics")]` so `open_inner` has one shape; with the
484    /// feature off the metrics half is a zero-sized type and only
485    /// [`Database::metrics`] is gated — which is also why the field is unread in
486    /// the default build: the actor holds the other `Arc` and does the writing.
487    #[cfg_attr(not(feature = "metrics"), allow(dead_code))]
488    shared: Arc<ActorShared>,
489}
490
491impl Database {
492    /// Open a database file at `path`, configuring pragmas, running migrations, and spawning the Write Actor.
493    ///
494    /// The snapshot cadence runs with [`SnapshotCadence::default`]. Use
495    /// [`Database::open_with_cadence`] to tune or disable it.
496    pub async fn open(path: impl AsRef<Path>) -> Result<Self> {
497        Self::open_with_cadence(path, Some(SnapshotCadence::default())).await
498    }
499
500    /// Open with an explicit snapshot cadence, or `None` to run without one
501    /// (§5.5, D-053).
502    ///
503    /// `None` restores the pre-0.5.5 behaviour, where `close()` is the only
504    /// thing that ever writes an anchor. That is the right setting for a
505    /// short-lived process that will not accumulate a delta worth bounding, and
506    /// for tests that assert on the contents of the snapshot directory.
507    pub async fn open_with_cadence(
508        path: impl AsRef<Path>,
509        cadence: Option<SnapshotCadence>,
510    ) -> Result<Self> {
511        Self::open_inner(path.as_ref(), cadence, None).await
512    }
513
514    /// Open with an injected clock (§5.1.2, **defect K**, D-062).
515    ///
516    /// The reason this exists is testing: `recorded_at` is the transaction-time
517    /// axis, and until now every test that wanted to assert on one had to either
518    /// avoid it or drive a raw connection, because `open()` hardcoded
519    /// [`SystemClock`]. `FakeClock` has been public and constructed in the test
520    /// harness since 0.5.2 with nothing to inject it into — the compiler warned
521    /// about the dead field on every build for three releases.
522    ///
523    /// **The clock is floored against the database before the actor starts.**
524    /// [`Clock::raise_floor`] is called with the newest `recorded_at` in the
525    /// ledger, so an injected clock cannot issue a stamp below what is already
526    /// stored — which would abort the next concept write on
527    /// `trg_concepts_monotonic_ra` rather than merely being odd. This is the
528    /// step whose absence kept the defect open: the obvious implementation
529    /// (take an `Arc<dyn Clock>`, use it) produces a `Database` that fails on
530    /// its first write against any non-empty file.
531    ///
532    /// On a fresh database there is no floor, so an injected `FakeClock` issues
533    /// exactly the stamps it was given.
534    pub async fn open_with_clock(
535        path: impl AsRef<Path>,
536        cadence: Option<SnapshotCadence>,
537        clock: Arc<dyn Clock>,
538    ) -> Result<Self> {
539        Self::open_inner(path.as_ref(), cadence, Some(clock)).await
540    }
541
542    async fn open_inner(
543        path: &Path,
544        cadence: Option<SnapshotCadence>,
545        injected: Option<Arc<dyn Clock>>,
546    ) -> Result<Self> {
547        let db = libsql::Builder::new_local(path).build().await?;
548        let write_conn = configure(db.connect()?).await?;
549        let read_conn = configure(db.connect()?).await?;
550
551        // PRAGMA query_only = ON on reader connection (§5.1.2)
552        read_conn.execute("PRAGMA query_only = ON", ()).await?;
553
554        let migration = migrations::run(&write_conn).await?;
555
556        let (highpri_tx, highpri_rx) = mpsc::channel(256);
557        let (lowpri_tx, lowpri_rx) = mpsc::channel(64);
558
559        // Floored after `migrations::run`, so the tables the floor is read from
560        // are guaranteed to exist.
561        let clock: Arc<dyn Clock> = match injected {
562            Some(clock) => {
563                if let Some(floor) = crate::util::clock::recorded_at_floor(&read_conn).await? {
564                    clock.raise_floor(floor);
565                }
566                clock
567            }
568            None => Arc::new(SystemClock::new(&read_conn).await?),
569        };
570        let shared = Arc::new(ActorShared::default());
571        let writer = tokio::spawn(run_writer_actor(
572            write_conn,
573            Arc::clone(&clock),
574            highpri_rx,
575            lowpri_rx,
576            Arc::clone(&shared),
577        ));
578
579        let archive_path = derive_archive_path(path);
580        let snapshots_dir = derive_snapshots_dir(path);
581
582        // **The cadence gets its own connection (Wave 4.1).** It used to share
583        // `read_conn`, on the reasoning that `libsql::Connection` is an
584        // Arc-backed handle and R15 makes every extra local connection a cost worth
585        // not paying for nothing. The cost it was not paying for turned out to be
586        // real: `reconstruct` brackets a fold with `ATTACH cold … DETACH cold`,
587        // that region is per-connection state, and it is not synchronised. Two
588        // folds on one connection can therefore interleave so that one DETACHes
589        // the handle the other is mid-fold on.
590        //
591        // Recorded in §8.5 as a hazard rather than a defect because it **did not
592        // reproduce**: 200 concurrent reconstructions against a 1 ms cadence with
593        // an archive present produced zero errors, since the cadence anchors at
594        // `MAX(recorded_at)` and so almost always takes the hot path. Narrow, and
595        // real — a write landing between `log_head` and the fold opens it.
596        //
597        // Separate connections remove the interleaving rather than ordering it,
598        // which is why this is preferred to a mutex around the region: there is
599        // no shared state left to race on, and nothing to remember to hold. The
600        // R15 objection does not apply — that fault is about *concurrent* opens,
601        // and this is one more sequential open during `open()`.
602        let (cadence_stop, cadence) = match cadence {
603            Some(cadence) => {
604                let cadence_conn = configure(db.connect()?).await?;
605                cadence_conn.execute("PRAGMA query_only = ON", ()).await?;
606                let (tx, rx) = tokio::sync::watch::channel(false);
607                let handle = tokio::spawn(snapshot::run_cadence(
608                    cadence_conn,
609                    snapshots_dir.clone(),
610                    archive_path.clone(),
611                    cadence,
612                    rx,
613                ));
614                (Some(tx), Some(handle))
615            }
616            None => (None, None),
617        };
618
619        let handle = Self {
620            db,
621            path: path.to_path_buf(),
622            read_conn,
623            highpri_tx,
624            lowpri_tx,
625            clock,
626            archive_path,
627            snapshots_dir,
628            schema_version: migrations::current_version(),
629            writer: Some(writer),
630            cadence_stop,
631            cadence,
632            closed: false,
633            shared,
634        };
635
636        // **Re-anchor after a migration (Wave 4.4).**
637        //
638        // D-043 makes a `SCHEMA_VERSION` bump invalidate every snapshot on disk,
639        // which is correct — a snapshot is a serialised `MaterializedState` and a
640        // schema change can change what that means. What was missing is the other
641        // half: nothing wrote a replacement, so the first `reconstruct` after an
642        // upgrade skipped every file as incompatible and folded from genesis. On
643        // a database with a large log that is the difference between reading one
644        // snapshot and folding the whole history, and the only trace was a
645        // `warn!` per skipped file.
646        //
647        // Written here rather than left to the cadence because the cadence fires
648        // on log *growth* (D-053): an upgraded database that is then read but not
649        // written would never re-anchor at all.
650        //
651        // Failure is logged, not returned. A missing anchor costs time and no
652        // information — snapshots are derivative under Doctrine VI — so refusing
653        // to open a database because its optimisation could not be rebuilt would
654        // trade a real capability for a performance one.
655        //
656        // Gated on the cadence being enabled, as well as on an actual upgrade:
657        // `open_with_cadence(None)` means *this handle writes no snapshots except
658        // at close()*, and a one-off write at open would contradict that for a
659        // caller who asked for the quiet mode precisely to control when files
660        // appear. They still get an anchor from `close()`.
661        if migration.upgraded() && handle.cadence.is_some() {
662            let ts = handle.clock.now();
663            let archive = handle
664                .archive_path
665                .exists()
666                .then_some(handle.archive_path.as_path());
667            match snapshot::write_final(&handle.read_conn, &handle.snapshots_dir, &ts, archive)
668                .await
669            {
670                Ok(path) => tracing::info!(
671                    "schema moved v{} -> v{}; re-anchored snapshots at {:?}",
672                    migration.from,
673                    migration.to,
674                    path
675                ),
676                Err(e) => tracing::warn!(
677                    "schema moved v{} -> v{} but the re-anchor failed: {e}. \
678                     Reconstruction stays correct and folds from genesis until the \
679                     cadence writes one.",
680                    migration.from,
681                    migration.to
682                ),
683            }
684        }
685
686        Ok(handle)
687    }
688
689    /// Read connection handle for queries, traversals, and folds.
690    pub fn read_conn(&self) -> &libsql::Connection {
691        &self.read_conn
692    }
693
694    /// The file this handle opened.
695    pub fn path(&self) -> &Path {
696        &self.path
697    }
698
699    /// A **new, independently owned, OS-level read-only** connection to this
700    /// database, for diagnostics (§4.7, T5.1, D-091).
701    ///
702    /// # Why this exists when `read_conn()` already does
703    ///
704    /// Two different things, and the difference is the point:
705    ///
706    /// * `read_conn()` returns a shared `&Connection` carrying
707    ///   `PRAGMA query_only = ON`. That pragma is **per-connection and
708    ///   reversible by its holder in one statement**, so it is a guardrail
709    ///   against accident, not a capability boundary. And because the reference
710    ///   is shared, a caller who runs a long reporting query on it is competing
711    ///   with every traversal and fold in the process.
712    /// * This returns a connection opened with `SQLITE_OPEN_READ_ONLY`, which is
713    ///   enforced by the engine below the pragma layer, and it is the caller's
714    ///   own.
715    ///
716    /// **Measured on libSQL 0.9.30 rather than assumed**
717    /// (`examples/readonly_open_probe.rs`), against a live WAL database with the
718    /// write actor running:
719    ///
720    /// | | `read_conn()` | `diagnostic_conn()` |
721    /// |---|---|---|
722    /// | `SELECT`, `EXPLAIN QUERY PLAN` | allowed | allowed |
723    /// | `INSERT` | refused | refused |
724    /// | `PRAGMA query_only = OFF` | **allowed** | allowed |
725    /// | `INSERT` after that | **allowed** | **refused** |
726    ///
727    /// The third and fourth rows are the whole difference: turning the pragma
728    /// off restores writes on `read_conn()` and does not here. That is what
729    /// "boundary rather than guardrail" means, and it is now a number rather
730    /// than a claim.
731    ///
732    /// # One way this is *more* permissive, which is worth knowing
733    ///
734    /// `CREATE TEMP TABLE` **succeeds** here and is refused by `read_conn()`.
735    /// Temp tables live in a separate temporary database that is writable
736    /// regardless of how the main one was opened, whereas `query_only` refuses
737    /// them outright — which is the mechanism [D-050] measured when it removed
738    /// `TwoPhaseTempTable` for returning `SQLITE_READONLY (8)` on the read
739    /// connection. So the stronger boundary is not uniformly stronger, and a
740    /// strategy that needs a temp table has a connection it could run on. That
741    /// is recorded, not acted on: D-050 removed the strategy for two reasons and
742    /// this addresses one of them.
743    ///
744    /// # Errors
745    ///
746    /// The file must already exist. `SQLITE_OPEN_READ_ONLY` drops
747    /// `SQLITE_OPEN_CREATE` with it, so a missing file is `SQLITE_CANTOPEN`
748    /// rather than a fresh empty database — which is the right failure, and is
749    /// surfaced as a typed error rather than as libSQL's error 14.
750    pub async fn diagnostic_conn(&self) -> Result<libsql::Connection> {
751        let fail = |reason: String| DbError::DiagnosticConn {
752            path: self.path.display().to_string(),
753            reason,
754        };
755        if !self.path.exists() {
756            return Err(fail(
757                "the file does not exist, and a read-only open cannot create it".to_string(),
758            ));
759        }
760        let db = libsql::Builder::new_local(&self.path)
761            .flags(libsql::OpenFlags::SQLITE_OPEN_READ_ONLY)
762            .build()
763            .await
764            .map_err(|e| fail(e.to_string()))?;
765        db.connect().map_err(|e| fail(e.to_string()))
766    }
767
768    /// Cross-check the snapshot chain against a fold from genesis (§5.5, T5.3,
769    /// D-092).
770    ///
771    /// `write_final` composes onto the previous snapshot, so snapshot *n* is
772    /// derived from snapshot *n−1* and nothing in the chain ever folds the whole
773    /// log. An error at any link propagates forward forever and every read
774    /// agrees with it, because every read descends from it. This is the check
775    /// that would notice.
776    ///
777    /// # When to run it
778    ///
779    /// **Not on a schedule this crate chooses.** A genesis fold is precisely the
780    /// cost snapshots exist to avoid, so running it periodically by default
781    /// would give every application the bill snapshots were bought to remove —
782    /// on a database whose log is large enough for snapshots to matter, which is
783    /// the only kind where this is worth doing. The plan calls it a scheduling
784    /// problem and it is the caller's schedule: an idle period, a nightly job,
785    /// or once per *N* anchors, chosen against a log size this crate cannot see.
786    ///
787    /// The cadence is deliberately left alone for the same reason — it runs on a
788    /// connection shared with nothing and a fold there would compete with
789    /// interactive reads at a moment nobody chose.
790    ///
791    /// # It reports; it does not repair
792    ///
793    /// A divergence means the snapshots are a wrong **cache**, not that the
794    /// ledger is corrupt: [Doctrine VI] makes them disposable, so deleting
795    /// [`Self::snapshots_dir`] restores correctness and costs only speed.
796    /// Rewriting the file here would destroy the evidence that composition has a
797    /// defect, which is the only thing this can tell you that you did not
798    /// already know.
799    ///
800    /// Pair it with the actor counters ([`Self::metrics`], D-079) so a
801    /// divergence found by a scheduled run is visible beside the write latency
802    /// of the period that produced it.
803    ///
804    /// [Doctrine VI]: ../../docs/architecture/s0-s3-foundations.md#doctrine-vi
805    pub async fn verify_snapshot_chain(&self, ts: &str) -> Result<crate::temporal::ChainCheck> {
806        let archive = self
807            .archive_path
808            .exists()
809            .then_some(self.archive_path.as_path());
810        crate::temporal::verify_snapshot_chain(&self.read_conn, ts, archive, &self.snapshots_dir)
811            .await
812    }
813
814    /// The clock every write is stamped with (§5.1.1).
815    pub fn clock(&self) -> &Arc<dyn Clock> {
816        &self.clock
817    }
818
819    /// Schema version this handle opened against.
820    pub fn schema_version(&self) -> u32 {
821        self.schema_version
822    }
823
824    /// Cold database path, derived by convention from the main file.
825    pub fn archive_path(&self) -> &Path {
826        &self.archive_path
827    }
828
829    /// Snapshot directory, derived by convention from the main file.
830    pub fn snapshots_dir(&self) -> &Path {
831        &self.snapshots_dir
832    }
833
834    /// What the write actor has done since this handle was opened (T1.4, D-079).
835    ///
836    /// Requires the `metrics` feature. The counters are per-handle and start at
837    /// zero on `open()` — they are not read from the database, because the thing
838    /// being measured is *this process's* actor and merging two processes'
839    /// histograms would produce a number about neither.
840    ///
841    /// The intended first question is [`crate::metrics::MetricsSnapshot::budget_violations`]:
842    ///
843    /// ```no_run
844    /// # async fn f(db: &macrame::Database) {
845    /// # #[cfg(feature = "metrics")] {
846    /// for k in db.metrics().budget_violations() {
847    ///     eprintln!("{} broke the 3 ms bound {} times", k.kind, k.over_budget);
848    /// }
849    /// # }
850    /// # }
851    /// ```
852    ///
853    /// Reading this does not stop the actor — see
854    /// [`crate::metrics::ActorMetrics::snapshot`] for what that costs in
855    /// consistency, and why the trade goes that way.
856    #[cfg(feature = "metrics")]
857    pub fn metrics(&self) -> crate::metrics::MetricsSnapshot {
858        self.shared.metrics.snapshot()
859    }
860
861    /// The underlying libSQL database, for callers that need their own connection.
862    ///
863    /// # Actor containment is a convention above this line, not a guarantee
864    ///
865    /// **Kept public, and the honest statement of what that costs (Wave 4.3).**
866    /// §5.1 says the write actor is the sole writer, and two mechanisms make that
867    /// true of the handle: every write method goes through a channel, and
868    /// [`Self::read_conn`] carries `PRAGMA query_only = ON`. **Nothing protects a
869    /// connection obtained from here.** A caller can open one, write to `links`
870    /// directly, and the actor will not know — the triggers still fire and the
871    /// ledger stays internally consistent, but the single-writer property that
872    /// [`crate::CHUNK_BUDGET`]'s latency argument rests on is gone, and so is the
873    /// serialisation the overlap guard (D-060) relies on.
874    ///
875    /// This is the same shape as the limit stated in §4.2 for that guard, and it
876    /// is one fact rather than two: **the storage layer permits what this API
877    /// refuses.** Making it private would not change that — the database file is
878    /// reachable by any SQLite client on the machine — it would only remove the
879    /// supported way to do the thing, which is how escape hatches become
880    /// `unsafe`-adjacent folklore.
881    ///
882    /// The free functions [`crate::register_model`] and
883    /// [`crate::upsert_embedding`] take a bare connection for the same reason and
884    /// carry the same caveat; prefer [`Self::register_model`] and
885    /// [`Self::upsert_embeddings`], which go through the actor.
886    ///
887    /// # The legitimate-use list is now one item long (T5.1, D-091)
888    ///
889    /// It used to read: `EXPLAIN QUERY PLAN` and other diagnostics, read-only
890    /// reporting queries wanting their own connection rather than sharing the
891    /// reader, and provoking a guard in a test. The first two are exactly what
892    /// [`Self::diagnostic_conn`] now does, and it does them behind an OS-level
893    /// read-only open rather than on a handle that can write. **Use that.**
894    ///
895    /// What is left is the one use that genuinely requires write access through
896    /// a connection the actor does not own: *provoking a guard* — writing the
897    /// state §4.7 says the storage layer permits and this API refuses, so a test
898    /// can assert the gap is still where the document says it is. That is the
899    /// only thing this crate's own suite uses it for.
900    ///
901    /// # Why `#[doc(hidden)]` and not a `raw-access` feature
902    ///
903    /// T5.1 offers either. The feature is the stronger declaration — it shows up
904    /// in the consumer's `Cargo.toml`, where a reviewer sees it — and it was
905    /// **not** taken, for a reason specific to what uses this:
906    ///
907    /// Cargo features are additive and cannot be *required* by a test target
908    /// except through `required-features`, which makes a plain `cargo test`
909    /// **skip** that binary silently. The binaries that call this are
910    /// `storage_boundary_tests` and `wave1_regression_tests` — the §4.7
911    /// tripwires, whose entire job is to fail when a documented gap moves. Gating
912    /// them behind a feature would mean the ordinary `cargo test` stopped running
913    /// the tests that enforce the section this item is about, to make a
914    /// declaration about a hatch. That trade is the wrong way round, and it is
915    /// the same failure the project already names: a suite that quietly does less
916    /// than it appears to.
917    ///
918    /// So the hatch stays reachable and stops being *discoverable*: it is absent
919    /// from the docs, and the documented path for every non-write use is
920    /// [`Self::diagnostic_conn`]. [D-068] is unchanged — removing it would buy
921    /// the appearance of a guarantee, since the file is reachable by any SQLite
922    /// client on the machine.
923    ///
924    /// [D-068]: ../../docs/architecture/s13-decision-register.md#d-068
925    #[doc(hidden)]
926    pub fn raw(&self) -> &libsql::Database {
927        &self.db
928    }
929
930    // -- write surface (§5.1, Appendix A) --
931    //
932    // Every method here validates and canonicalises before the value crosses the
933    // channel, so a bad edge type or a second-precision timestamp is a typed
934    // error at the call site rather than an engine `CHECK` failure surfacing
935    // from the far side of an actor with no context attached.
936    //
937    // NOTE (§5.1.8, D-028): awaiting one of these waits on a Rust channel, not
938    // in SQLite, so `busy_timeout` does not bound it. During an in-flight
939    // `rebuild_current` or `archive` the caller stalls for that transaction's
940    // duration. Wrap in `tokio::time::timeout` if you need a bound — but a
941    // timeout is not a cancellation: the command stays queued and commits when
942    // the actor reaches it.
943
944    /// Assert an edge (Doctrine III: a new row, never an update).
945    pub async fn assert_edge(&self, edge: EdgeAssertion) -> Result<()> {
946        let edge = edge.normalized()?;
947        self.high(|responder| HighPriCommand::AssertEdge { edge, responder })
948            .await
949    }
950
951    /// Close an open interval by asserting its replacement (Doctrine III).
952    pub async fn retire_edge(
953        &self,
954        source: impl Into<String>,
955        target: impl Into<String>,
956        edge_type: impl Into<String>,
957        valid_from: &str,
958        valid_to: &str,
959    ) -> Result<()> {
960        let edge_type = edge_type.into();
961        crate::graph::edge::validate_edge_type(&edge_type)?;
962        let valid_from = timestamp::normalize(valid_from)?;
963        let valid_to = timestamp::normalize(valid_to)?;
964        let (source, target) = (source.into(), target.into());
965
966        self.high(|responder| HighPriCommand::RetireEdge {
967            source,
968            target,
969            edge_type,
970            valid_from,
971            valid_to,
972            responder,
973        })
974        .await
975    }
976
977    /// Insert or update a concept.
978    pub async fn upsert_concept(&self, concept: ConceptUpsert) -> Result<()> {
979        let concept = concept.normalized()?;
980        self.high(|responder| HighPriCommand::UpsertConcept { concept, responder })
981            .await
982    }
983
984    /// Assert many edges in one transaction under one stamp (D-014).
985    ///
986    /// # This is the one write with no latency bound, and here is what it costs
987    ///
988    /// The batch is one act under one `recorded_at`, so it cannot be chunked —
989    /// splitting it is the thing this method exists not to do. That makes the
990    /// actor's hold a function of `edges.len()`, and until now the only
991    /// statement of that anywhere was the prose "uncapped" in
992    /// [`CHUNK_BUDGET`]'s table. A caller who stalls every other writer for
993    /// eight seconds should have been able to predict it from the signature.
994    ///
995    /// Measured on libSQL 0.9.30 (T1.3, D-081), holding the actor for:
996    ///
997    /// | rows | hold |
998    /// |---|---|
999    /// | 500 | ~34 ms |
1000    /// | 2,000 | ~155 ms |
1001    /// | 10,000 | ~1.0 s |
1002    /// | 20,000 | ~2.6 s |
1003    ///
1004    /// [`estimated_bulk_hold`] is that curve as a function, and this method
1005    /// emits a `tracing::warn!` when it predicts more than
1006    /// [`BULK_ATOMIC_WARN_HOLD`]. **The estimate is a shape, not a promise** —
1007    /// see [`estimated_bulk_hold`] for what it is calibrated against and where
1008    /// it will be wrong.
1009    ///
1010    /// A caller who needs the latency bound and not the atomicity wants
1011    /// [`Self::bulk_import`], which is the same write chunked and explicitly not
1012    /// atomic overall (D-011).
1013    pub async fn write_bulk_atomic(&self, edges: Vec<EdgeAssertion>) -> Result<usize> {
1014        let estimate = estimated_bulk_hold(&edges);
1015        if estimate > BULK_ATOMIC_WARN_HOLD {
1016            // Warned here rather than in the actor, and before the send: this is
1017            // the caller's own task, so the log line lands with their span
1018            // attached and names the call site that chose the batch size. By the
1019            // time the actor has it, the only context left is "a large batch".
1020            tracing::warn!(
1021                rows = edges.len(),
1022                estimated_hold_ms = estimate.as_millis() as u64,
1023                "write_bulk_atomic will hold the write actor for roughly \
1024                 {estimate:?} — it is atomic by contract (D-014) and cannot be \
1025                 chunked. Every other writer waits that long. Use bulk_import \
1026                 if the batch does not need to be all-or-nothing."
1027            );
1028        }
1029
1030        let edges = normalize_all(edges)?;
1031        self.high(|responder| HighPriCommand::WriteBulkAtomic { edges, responder })
1032            .await
1033    }
1034
1035    /// Rebuild `links_current` from `links` and verify zero drift (§5.8).
1036    pub async fn rebuild_current(&self) -> Result<RebuildReport> {
1037        self.high(|responder| HighPriCommand::RebuildCurrent { responder })
1038            .await
1039    }
1040
1041    /// Rebuild `links_current` beside itself, in chunks (§5.8, T1.2, D-082).
1042    ///
1043    /// Same result as [`Self::rebuild_current`], different latency profile.
1044    /// `rebuild_current` is one transaction holding the write lock for its whole
1045    /// duration, because D-023 will not let the `DELETE` and the `INSERT` be
1046    /// split: a reader landing between them sees a graph with no edges and no
1047    /// error. This builds the replacement in a shadow table instead — the live
1048    /// table stays live and trigger-maintained throughout — and swaps it in at
1049    /// the end.
1050    ///
1051    /// Each step is its own actor turn, so an interactive assertion can jump the
1052    /// queue between chunks. That is the whole of the improvement, and it is why
1053    /// the loop is here rather than inside the actor's arm (the same reasoning
1054    /// as [`Self::archive_windowed`] and [`Self::bulk_import`]).
1055    ///
1056    /// # What the swap still costs
1057    ///
1058    /// Not microseconds. Index names are global and SQLite has no `ALTER INDEX
1059    /// … RENAME`, so the shadow cannot be built carrying `links_current`'s index
1060    /// names while `links_current` still holds them — and building it under
1061    /// other names would leave the table permanently indexed under names absent
1062    /// from [`CREATE_INDICES`](crate::schema::ddl::CREATE_INDICES), so the next
1063    /// migration would create a second copy of each.
1064    /// `DROP TABLE` frees the names, so the swap transaction is where
1065    /// the three indexes get built. What the chunking moves off the lock is the
1066    /// **projection** — the window function over all of `links` — which is the
1067    /// O(E log E) term.
1068    ///
1069    /// # When this returns an error rather than a repair
1070    ///
1071    /// [`DbError::RebuildInterrupted`] means an archive committed while the
1072    /// shadow was being built. Its deletions are invisible to a catch-up pass
1073    /// keyed on `recorded_at` — a deleted row has no `recorded_at` left to find
1074    /// it by — so the work is discarded rather than swapped in. `links_current`
1075    /// is untouched and the call can simply be retried.
1076    ///
1077    /// Use [`Self::rebuild_current`] when the repair must be one atomic act, or
1078    /// when nothing else is contending for the actor and the extra turns are
1079    /// pure overhead.
1080    pub async fn rebuild_current_chunked(&self) -> Result<RebuildReport> {
1081        use crate::integrity::{ShadowOutcome, ShadowStep};
1082
1083        // Each `else` arm is unreachable: the actor maps each step to its own
1084        // outcome variant. Written as a refutable pattern rather than an
1085        // `unwrap` so that adding a step cannot turn a mismatch into a panic on
1086        // the write path — and `WriterDroppedResponder` is the honest name for
1087        // "the actor answered with something this cannot use".
1088        let ShadowOutcome::Started { build_start, epoch } =
1089            self.shadow_step(ShadowStep::Begin).await?
1090        else {
1091            return Err(DbError::WriterDroppedResponder);
1092        };
1093
1094        let mut after: Option<String> = None;
1095        loop {
1096            let ShadowOutcome::Filled { last } = self
1097                .shadow_step(ShadowStep::Fill {
1098                    after: after.take(),
1099                })
1100                .await?
1101            else {
1102                return Err(DbError::WriterDroppedResponder);
1103            };
1104            match last {
1105                Some(last) => after = Some(last),
1106                None => break,
1107            }
1108        }
1109
1110        let ShadowOutcome::Swapped { rows } = self
1111            .shadow_step(ShadowStep::Swap { build_start, epoch })
1112            .await?
1113        else {
1114            return Err(DbError::WriterDroppedResponder);
1115        };
1116
1117        Ok(RebuildReport {
1118            rows_rebuilt: rows,
1119            // Not audited. The chunked path's whole argument is that the
1120            // expensive work happens off the lock, and `audit_current` is two
1121            // `EXCEPT` passes over the projection — the cost D-077 removed from
1122            // the archive for the same reason. A caller who wants the check has
1123            // `audit_current` on the read connection, where it costs nobody the
1124            // write lock.
1125            drift_after: 0,
1126        })
1127    }
1128
1129    /// Run one step of a chunked rebuild, for a caller doing its own scheduling.
1130    ///
1131    /// [`Self::rebuild_current_chunked`] is this in a loop and is what almost
1132    /// everyone wants. This exists because that loop offers no seam: it drives
1133    /// `Begin`, then `Fill` to exhaustion, then `Swap`, and a caller who needs to
1134    /// do something *between* steps — pace them against a frame budget, abandon
1135    /// a rebuild that has run long enough, or provoke the archive interlock in a
1136    /// test — cannot get in.
1137    ///
1138    /// The obligation that comes with it: `epoch` from
1139    /// [`ShadowOutcome::Started`](crate::integrity::ShadowOutcome) must be handed
1140    /// back to [`ShadowStep::Swap`](crate::integrity::ShadowStep), or the
1141    /// archive interlock is defeated and a stale projection can be swapped in.
1142    /// The looping version cannot get that wrong; this one can.
1143    pub async fn shadow_step(
1144        &self,
1145        step: crate::integrity::ShadowStep,
1146    ) -> Result<crate::integrity::ShadowOutcome> {
1147        self.low(|responder| LowPriCommand::ShadowRebuild { step, responder })
1148            .await
1149    }
1150
1151    /// Import edges on the background channel, chunked (D-011).
1152    ///
1153    /// Atomic *per chunk*, not overall: a failure partway leaves earlier chunks
1154    /// committed. That is the tradeoff [`chunk_rows`] documents — use
1155    /// [`Database::write_bulk_atomic`] when the batch must be all-or-nothing.
1156    ///
1157    /// Chunked at [`chunk_rows::EDGES`], which is also faster in total than the
1158    /// larger chunks this used through 0.5.5 (D-058).
1159    pub async fn bulk_import(&self, edges: Vec<EdgeAssertion>) -> Result<usize> {
1160        let edges = normalize_all(edges)?;
1161        let chunks: Vec<_> = edges.chunks(chunk_rows::EDGES).map(<[_]>::to_vec).collect();
1162        self.low_chunked(chunks, |chunk, responder| LowPriCommand::BulkImportChunk {
1163            chunk,
1164            responder,
1165        })
1166        .await
1167    }
1168
1169    /// Upsert many **concepts** on the background channel, chunked (D-011).
1170    ///
1171    /// This is the bulk concept path, and every row it writes is a ledger write:
1172    /// it versions the concept and lands in `transaction_log`. Derived analytics
1173    /// output does not belong here — see
1174    /// [`Database::write_analytics_annotations`] and D-041.
1175    ///
1176    /// Called `write_annotations` through 0.5.6, from when the two writes were
1177    /// one call. D-041 split them and the name stayed on the wrong one for three
1178    /// releases, so the crate had a `write_annotations` that wrote concepts
1179    /// sitting beside a `write_analytics_annotations` that wrote annotations
1180    /// (D-075).
1181    pub async fn write_concepts(&self, concepts: Vec<ConceptUpsert>) -> Result<usize> {
1182        let concepts: Vec<ConceptUpsert> = concepts
1183            .into_iter()
1184            .map(ConceptUpsert::normalized)
1185            .collect::<Result<_>>()?;
1186        let chunks: Vec<_> = concepts
1187            .chunks(chunk_rows::CONCEPTS)
1188            .map(<[_]>::to_vec)
1189            .collect();
1190        self.low_chunked(chunks, |chunk, responder| {
1191            LowPriCommand::WriteConceptsChunk { chunk, responder }
1192        })
1193        .await
1194    }
1195
1196    /// State as believed at `ts` (§5.5, D-026, D-049).
1197    ///
1198    /// A read: it runs on `read_conn` and never touches the Write Actor, so a
1199    /// reconstruction and a full-speed write-back do not slow each other.
1200    ///
1201    /// Prefer this to calling [`crate::temporal::reconstruct`] directly. The
1202    /// free function takes the archive path and the snapshot directory as
1203    /// arguments, and a caller who passes `None` for the second gets a correct
1204    /// answer that folds the whole log every time — the composition is opt-in
1205    /// at that layer and easy to leave off by accident. Here both come from the
1206    /// handle, so the fast path is the default one.
1207    pub async fn reconstruct(&self, ts: &str) -> Result<crate::temporal::MaterializedState> {
1208        let ts = timestamp::normalize(ts)?;
1209        crate::temporal::reconstruct(
1210            &self.read_conn,
1211            &ts,
1212            Some(&self.archive_path),
1213            Some(&self.snapshots_dir),
1214        )
1215        .await
1216    }
1217
1218    /// Create a model's embedding table and DiskANN index (§5.9, D-048).
1219    ///
1220    /// Idempotent: registering a model that already exists at the same
1221    /// dimension succeeds, and at a different dimension fails with
1222    /// [`DbError::DimMismatch`] naming both, rather than no-opping through
1223    /// `IF NOT EXISTS` and leaving the caller believing the dimension they
1224    /// asked for is the one in force.
1225    ///
1226    /// This issues DDL, which everywhere else in the crate is the migration
1227    /// runner's exclusive business (D-032). The exception is bounded and
1228    /// deliberate: a model's table is created once, by an explicit call, and
1229    /// the alternative — a caller-supplied write connection — is the very thing
1230    /// the Write Actor exists to make impossible.
1231    ///
1232    /// # Latency
1233    ///
1234    /// One small transaction, but it queues like any other write: see §5.1.8.
1235    pub async fn register_model(&self, model: &ModelName, dim: usize) -> Result<()> {
1236        let model = model.clone();
1237        self.high(|responder| HighPriCommand::RegisterModel {
1238            model,
1239            dim,
1240            responder,
1241        })
1242        .await
1243    }
1244
1245    /// Store or replace vectors for `model`, chunked (§5.9, D-011, D-048).
1246    ///
1247    /// The write path for embeddings. Before 0.5.4 there was none:
1248    /// [`crate::vector::upsert_embedding`] takes a raw connection, `read_conn`
1249    /// is `query_only`, and the write connection lives inside the actor — so an
1250    /// application could search vectors it had no way to store.
1251    ///
1252    /// Low priority and chunked at [`chunk_rows::EMBEDDINGS`], because embedding
1253    /// is bulk derived work: a 50,000-vector backfill must yield to an
1254    /// interactive assertion at every chunk boundary. That constant is the
1255    /// smallest of the four by a wide margin — DiskANN index maintenance makes an
1256    /// embedding the most expensive row in the system (D-058). Atomic per chunk, not overall, which
1257    /// is the same trade [`Database::bulk_import`] makes and is safer here than
1258    /// there — an embedding is derived (Doctrine VII), so a partially written
1259    /// batch is recoverable by re-embedding.
1260    ///
1261    /// Fails with [`DbError::ModelNotRegistered`] if `model` has no table, and
1262    /// [`DbError::DimMismatch`] if a vector's length is not the declared
1263    /// dimension. The dimension is read from the schema once per chunk (D-037):
1264    /// the crate keeps no registry of its own to fall out of date.
1265    pub async fn upsert_embeddings(
1266        &self,
1267        model: &ModelName,
1268        rows: Vec<(String, Vec<f32>)>,
1269    ) -> Result<usize> {
1270        let chunks: Vec<_> = rows
1271            .chunks(chunk_rows::EMBEDDINGS)
1272            .map(<[_]>::to_vec)
1273            .collect();
1274        self.low_chunked(chunks, |chunk, responder| {
1275            LowPriCommand::UpsertEmbeddingChunk {
1276                model: model.clone(),
1277                chunk,
1278                responder,
1279            }
1280        })
1281        .await
1282    }
1283
1284    /// Reconstruct the concept-text search index from the ledger (§5.9, D-036).
1285    ///
1286    /// The FTS index is derivative: D-036 promises every derivative table can be
1287    /// rebuilt from the ledger tables, and this is that promise made callable
1288    /// for `concepts_fts`. Needed after a restore that skipped the shadow
1289    /// tables, or if the index is ever suspected of drifting from the text —
1290    /// and, as a matter of policy, cheaper to run than to reason about.
1291    ///
1292    /// The work is `INSERT INTO concepts_fts(concepts_fts) VALUES('rebuild')`,
1293    /// which is FTS5's own operation over the content table, so this is not a
1294    /// second implementation of the sync triggers that could disagree with them.
1295    pub async fn rebuild_fts(&self) -> Result<()> {
1296        self.low(|responder| LowPriCommand::RebuildFts { responder })
1297            .await
1298    }
1299
1300    // **There is deliberately no `verify_fts()` (§5.9, D-071).**
1301    //
1302    // `rebuild_fts` is the repair with no way to ask whether it is needed, and
1303    // Wave 5 set out to add the missing half. FTS5 offers `'integrity-check'`,
1304    // which looked like exactly the engine-provided answer this crate prefers.
1305    // It is not: on libSQL 0.9.30 it verifies the index's *internal* consistency
1306    // and not its agreement with the content table. Measured — after
1307    // `'delete-all'` the index matches nothing where it matched ten rows, and
1308    // both `'integrity-check'` and `'integrity-check', 0` still report success.
1309    //
1310    // A `verify_fts()` on that footing would answer "healthy" for an empty
1311    // index, which is worse than having no method at all: it is the shape of
1312    // defect AC, a function that looks like it checks something and does not.
1313    // `an_emptied_fts_index_still_passes_integrity_check` pins the limitation so
1314    // that if a later libSQL fixes it, the test fails and says so.
1315
1316    /// Write derived analytics results on the background channel, chunked
1317    /// (§5.4, D-041).
1318    ///
1319    /// Rows go to `analytics_annotations`, which has no log trigger, so nothing
1320    /// written here reaches `transaction_log` and nothing here versions a
1321    /// concept. Rerunning an algorithm replaces the previous pass rather than
1322    /// recording that the world changed.
1323    ///
1324    /// Low priority and chunked at [`chunk_rows::ANNOTATIONS`] — the largest of
1325    /// the four, because this is the only bulk table carrying no triggers at all
1326    /// and its rows are correspondingly cheap (D-058) — so a 50,000-label Louvain
1327    /// save yields to interactive writes at every chunk boundary and carries the
1328    /// per-chunk fidelity boundary of §5.1.6 — a partially written pass is
1329    /// recoverable by rerunning, which is the property that makes derived state
1330    /// safe to write this way and assertions not.
1331    pub async fn write_analytics_annotations(&self, annotations: Vec<Annotation>) -> Result<usize> {
1332        let chunks: Vec<_> = annotations
1333            .chunks(chunk_rows::ANNOTATIONS)
1334            .map(<[_]>::to_vec)
1335            .collect();
1336        self.low_chunked(chunks, |chunk, responder| {
1337            LowPriCommand::WriteAnalyticsChunk { chunk, responder }
1338        })
1339        .await
1340    }
1341
1342    /// Move closed intervals and superseded log rows older than `cutoff` to the
1343    /// cold database (§5.7, D-012).
1344    pub async fn archive(&self, cutoff: &str) -> Result<ArchiveReport> {
1345        let cutoff = timestamp::normalize(cutoff)?;
1346        let archive_path = self.archive_path.clone();
1347        self.low(|responder| LowPriCommand::Archive {
1348            cutoff,
1349            archive_path,
1350            responder,
1351        })
1352        .await
1353    }
1354
1355    /// Move the named concepts back from the cold database into the hot tables
1356    /// (§2.3, C3).
1357    ///
1358    /// Rehydration is a **physical move back, not a write**: it mints no
1359    /// transaction-time facts and is invisible to both clocks. An id that is not
1360    /// in the cold file is skipped rather than being an error — the caller
1361    /// generally has a list from a cold-side query, and a partially-stale list is
1362    /// the normal case rather than a mistake. The report says how many actually
1363    /// moved.
1364    ///
1365    /// See [`RehydrateReport::rowids_reassigned`] for the one way a rehydrated
1366    /// row can differ from the row that was archived.
1367    pub async fn rehydrate(&self, ids: &[&str]) -> Result<RehydrateReport> {
1368        let ids: Vec<String> = ids.iter().map(|s| (*s).to_string()).collect();
1369        let archive_path = self.archive_path.clone();
1370        self.low(|responder| LowPriCommand::Rehydrate {
1371            ids,
1372            archive_path,
1373            responder,
1374        })
1375        .await
1376    }
1377
1378    /// Archive up to `cutoff` as a sequence of sessions, each covering at most
1379    /// `window` of **transaction** time (T1.1, D-080).
1380    ///
1381    /// `archive(cutoff)` is one transaction whose size is set by how long it has
1382    /// been since the last one, which makes it the least bounded of the three
1383    /// operations exempt from [`CHUNK_BUDGET`] — its hold is a function of
1384    /// operational history rather than of anything a caller chose. This runs the
1385    /// same work as *N* complete sessions, each with its own marker, horizon row
1386    /// and rebuild, and returns one [`ArchiveReport`] per session in order.
1387    ///
1388    /// # D-012 is satisfied per session, and that is what it requires
1389    ///
1390    /// The atomicity D-012 demands is that copy-then-delete never be split — a
1391    /// crash between the phases duplicates or loses rows. *N* small sessions
1392    /// satisfy that exactly as one large one does. The obligation windowing adds
1393    /// is that a partial run leave a coherent intermediate state, which it does:
1394    /// each session commits a valid horizon, so a failure at window *k* leaves a
1395    /// database archived up to boundary *k−1* and nothing in between. **The
1396    /// sequence is not atomic and does not claim to be** — on error, the reports
1397    /// for the sessions that did commit are lost with it, but their effect is
1398    /// not, and re-running with the same `cutoff` completes the job.
1399    ///
1400    /// # Each session is its own actor turn, and that is the entire point
1401    ///
1402    /// This loop lives here, on the handle, rather than inside the actor's
1403    /// `Archive` arm. Putting it there would have produced *N* small
1404    /// transactions inside **one** hold, which shrinks the transaction and
1405    /// changes the latency not at all: the actor is single-threaded, so nothing
1406    /// else writes until its turn returns regardless of how many `COMMIT`s the
1407    /// turn contains. Sending *N* commands returns the actor to its `select!`
1408    /// between sessions, which is where an interactive assertion gets to jump
1409    /// the queue — and it is high-priority, so it does.
1410    ///
1411    /// The same reasoning is why [`Self::bulk_import`] chunks here and not
1412    /// there, and it is the trap T1.2 names for `CREATE TABLE … AS SELECT`.
1413    ///
1414    /// # Choosing a window
1415    ///
1416    /// The bound is on *transaction* time, so the session count is set by how
1417    /// far back the hot file goes, not by how much it holds. A window is
1418    /// rejected rather than clamped if it would need more than
1419    /// [`MAX_ARCHIVE_SESSIONS`] sessions — see [`DbError::ArchiveWindow`].
1420    ///
1421    /// Windows containing nothing archivable are cheap but not free: each still
1422    /// opens a transaction and writes a horizon row. What they no longer do is
1423    /// re-project `links_current`, which `archive_session` now skips when its
1424    /// `DELETE` removed no rows — without that, windowing costs *more* in total
1425    /// than not windowing, because the repair term scales with the surviving
1426    /// table and not with the batch (D-077).
1427    pub async fn archive_windowed(
1428        &self,
1429        cutoff: &str,
1430        window: std::time::Duration,
1431    ) -> Result<Vec<ArchiveReport>> {
1432        let cutoff = timestamp::normalize(cutoff)?;
1433        let boundaries = self.archive_boundaries(&cutoff, window).await?;
1434
1435        let mut reports = Vec::with_capacity(boundaries.len());
1436        for boundary in boundaries {
1437            let archive_path = self.archive_path.clone();
1438            reports.push(
1439                self.low(|responder| LowPriCommand::Archive {
1440                    cutoff: boundary,
1441                    archive_path,
1442                    responder,
1443                })
1444                .await?,
1445            );
1446        }
1447        Ok(reports)
1448    }
1449
1450    /// The cutoffs [`Self::archive_windowed`] will run, ascending, ending at
1451    /// `cutoff` exactly.
1452    ///
1453    /// Read on `read_conn`, not on the actor: this is two `MIN`s and the actor
1454    /// has no reason to hold its lock for them.
1455    ///
1456    /// The lower end comes from the data rather than from the clock. Stepping
1457    /// from some fixed epoch would make the session count a function of the
1458    /// calendar — a database opened yesterday would still be asked to archive
1459    /// 1970 — whereas the oldest `recorded_at` actually present is the earliest
1460    /// boundary that can contain anything.
1461    async fn archive_boundaries(
1462        &self,
1463        cutoff: &str,
1464        window: std::time::Duration,
1465    ) -> Result<Vec<String>> {
1466        // A single session at `cutoff` is exactly `archive(cutoff)`, and it is
1467        // the right answer for an empty hot file: it still writes the horizon
1468        // row, so windowed and unwindowed runs leave the same observable state.
1469        let Some(oldest) = self.oldest_hot_stamp(cutoff).await? else {
1470            return Ok(vec![cutoff.to_string()]);
1471        };
1472
1473        let start = timestamp::parse(&oldest)?;
1474        let end = timestamp::parse(cutoff)?;
1475        let Ok(span) = end.duration_since(start) else {
1476            // Everything in the hot file is at or after the cutoff, so there is
1477            // nothing in range to divide.
1478            return Ok(vec![cutoff.to_string()]);
1479        };
1480
1481        if window.is_zero() {
1482            return Err(DbError::ArchiveWindow {
1483                window,
1484                reason: "a zero-length window never advances past the first boundary".into(),
1485            });
1486        }
1487
1488        // `div_ceil` on nanos: a span of 90 minutes in 60-minute windows is two
1489        // sessions, not one. `as_nanos` is u128, so neither the division nor the
1490        // span can overflow for any timestamp this crate can store.
1491        let sessions = span.as_nanos().div_ceil(window.as_nanos());
1492        if sessions > MAX_ARCHIVE_SESSIONS as u128 {
1493            return Err(DbError::ArchiveWindow {
1494                window,
1495                reason: format!(
1496                    "a span of {span:?} would need {sessions} sessions (limit \
1497                     {MAX_ARCHIVE_SESSIONS}); widen the window"
1498                ),
1499            });
1500        }
1501
1502        let mut boundaries = Vec::with_capacity(sessions as usize);
1503        for k in 1..sessions {
1504            boundaries.push(timestamp::format(start + window * k as u32));
1505        }
1506        // The last boundary is `cutoff` itself and not `start + n*window`, which
1507        // would overshoot and archive rows the caller excluded.
1508        boundaries.push(cutoff.to_string());
1509        Ok(boundaries)
1510    }
1511
1512    /// Oldest `recorded_at` below `cutoff` in either hot table, or `None`.
1513    async fn oldest_hot_stamp(&self, cutoff: &str) -> Result<Option<String>> {
1514        let mut oldest: Option<String> = None;
1515        for table in ["links", "transaction_log"] {
1516            let found: Option<String> = self
1517                .read_conn
1518                .query(
1519                    &format!("SELECT MIN(recorded_at) FROM {table} WHERE recorded_at < ?1"),
1520                    libsql::params![cutoff],
1521                )
1522                .await?
1523                .next()
1524                .await?
1525                .and_then(|row| row.get(0).ok());
1526            if let Some(found) = found {
1527                if oldest.as_ref().is_none_or(|o| found < *o) {
1528                    oldest = Some(found);
1529                }
1530            }
1531        }
1532        Ok(oldest)
1533    }
1534
1535    /// Send a high-priority command and wait for its answer.
1536    ///
1537    /// The two error mappings here are the whole reason this helper exists.
1538    /// `send` failing means the actor is gone — `WriterUnavailable`. The
1539    /// responder being dropped without an answer means the actor took the
1540    /// command and never replied — `WriterDroppedResponder`, which is a bug in
1541    /// the actor rather than a condition the caller can retry. Both variants
1542    /// existed in `error.rs` from 0.4.5 and neither was ever constructed, so a
1543    /// dead actor and a hung one were both just a caller waiting forever.
1544    async fn high<T>(
1545        &self,
1546        make: impl FnOnce(oneshot::Sender<Result<T>>) -> HighPriCommand,
1547    ) -> Result<T> {
1548        let (tx, rx) = oneshot::channel();
1549        self.highpri_tx
1550            .send(make(tx))
1551            .await
1552            .map_err(|_| DbError::WriterUnavailable)?;
1553        rx.await.map_err(|_| DbError::WriterDroppedResponder)?
1554    }
1555
1556    /// Send each chunk in turn and sum the counts — the shape all four bulk
1557    /// paths share (T3.4, D-086).
1558    ///
1559    /// # This is sequential on purpose, and the purpose is a measurement
1560    ///
1561    /// T3.4 proposed pipelining: send *k* chunks ahead so the actor never finds
1562    /// an empty queue. The reasoning is that awaiting each chunk before building
1563    /// the next leaves the actor idle for a channel round trip every time, which
1564    /// on a 1M-edge import is ~11,000 idle gaps.
1565    ///
1566    /// Both halves of that are true and the conclusion does not follow. The gaps
1567    /// are real; they are also **four orders of magnitude smaller than the work
1568    /// they interrupt**. A tokio mpsc hop is sub-microsecond and a chunk takes
1569    /// 13–21 ms. Implemented and swept at depths 1, 2, 4, 8 and 16 over 20K and
1570    /// 100K edges: every cell landed within 1% of sequential, in both directions
1571    /// — see `examples/pipeline_diag.rs`, which is kept precisely so this is not
1572    /// re-proposed from the same reasoning.
1573    ///
1574    /// So the pipelining was removed and the deduplication kept. It was not free
1575    /// to hold: with chunks in flight, a failure at chunk `i` no longer leaves a
1576    /// **prefix** committed, because `i+1 ..= i+k-1` were already sent and commit
1577    /// anyway. D-011 promises "earlier chunks committed", and paying for that
1578    /// with a weaker recovery story in exchange for nothing measurable is the
1579    /// wrong trade.
1580    ///
1581    /// Sending stops at the first error, so what commits is exactly the prefix
1582    /// before the failure.
1583    async fn low_chunked<C>(
1584        &self,
1585        chunks: Vec<C>,
1586        make: impl Fn(C, oneshot::Sender<Result<usize>>) -> LowPriCommand,
1587    ) -> Result<usize> {
1588        let mut written = 0usize;
1589        for chunk in chunks {
1590            let (tx, rx) = oneshot::channel();
1591            self.lowpri_tx
1592                .send(make(chunk, tx))
1593                .await
1594                .map_err(|_| DbError::WriterUnavailable)?;
1595            written += rx.await.map_err(|_| DbError::WriterDroppedResponder)??;
1596        }
1597        Ok(written)
1598    }
1599
1600    async fn low<T>(
1601        &self,
1602        make: impl FnOnce(oneshot::Sender<Result<T>>) -> LowPriCommand,
1603    ) -> Result<T> {
1604        let (tx, rx) = oneshot::channel();
1605        self.lowpri_tx
1606            .send(make(tx))
1607            .await
1608            .map_err(|_| DbError::WriterUnavailable)?;
1609        rx.await.map_err(|_| DbError::WriterDroppedResponder)?
1610    }
1611
1612    /// Clean shutdown: stop the Write Actor, then write the final snapshot (§5.1.7).
1613    ///
1614    /// Order matters. The snapshot is taken *after* the actor has stopped and
1615    /// been joined, so no write can land between the fold and the file — the
1616    /// anchor it records is the last thing that happened, not the last thing
1617    /// that happened to be visible.
1618    ///
1619    /// A failed snapshot is reported rather than swallowed. It is not a
1620    /// durability loss — the ledger is in the WAL and the log replays without
1621    /// it — but it means the next open starts from an older anchor, and a caller
1622    /// that never hears about it cannot know why startup got slower.
1623    ///
1624    /// **The cadence stops first (§5.5, D-053).** Both it and `write_final` end
1625    /// by running retention over the snapshot directory, and retention deletes
1626    /// files. Letting them overlap would mean one pass enumerating the directory
1627    /// while the other removes from it — not a correctness problem for the
1628    /// ledger, which is why the ordering is stated rather than locked, but a
1629    /// source of spurious warnings and of a final anchor that could be deleted
1630    /// by a cleanup that started before it existed. Stopping the cadence, then
1631    /// the actor, then taking the snapshot leaves exactly one writer at each
1632    /// step.
1633    pub async fn close(mut self) -> Result<()> {
1634        if let Some(stop) = self.cadence_stop.take() {
1635            let _ = stop.send(true);
1636        }
1637        if let Some(handle) = self.cadence.take() {
1638            let _ = handle.await;
1639        }
1640
1641        let (tx, rx) = oneshot::channel();
1642        let _ = self
1643            .highpri_tx
1644            .send(HighPriCommand::Shutdown { responder: tx })
1645            .await;
1646        let _ = rx.await;
1647
1648        // **The writer's `Result` is propagated, not discarded (Wave 4.2).**
1649        // It used to be `let _ = handle.await`, so an actor that had panicked or
1650        // returned an error closed "successfully" and the caller's last chance to
1651        // learn that the write path had died was spent silently. A `JoinError`
1652        // here means the actor panicked; the inner `Result` is whatever it
1653        // returned.
1654        //
1655        // Ordered before the final snapshot on purpose: a snapshot written after
1656        // a failed writer records a state the caller has no reason to trust, and
1657        // returning the writer's error while also having written that file is
1658        // worse than not writing it.
1659        if let Some(handle) = self.writer.take() {
1660            match handle.await {
1661                Ok(res) => res?,
1662                Err(e) => {
1663                    return Err(DbError::WriterStopped(format!(
1664                        "the write actor did not exit cleanly: {e}"
1665                    )))
1666                }
1667            }
1668        }
1669
1670        let ts = self.clock.now();
1671        let archive = self
1672            .archive_path
1673            .exists()
1674            .then_some(self.archive_path.as_path());
1675        snapshot::write_final(&self.read_conn, &self.snapshots_dir, &ts, archive).await?;
1676
1677        // Marks the handle closed so `Drop` knows not to complain.
1678        self.closed = true;
1679        Ok(())
1680    }
1681}
1682
1683/// Notes a missed `close()` at `warn!`, and deliberately does **not** assert.
1684///
1685/// **§7.3 offered option B — document `close()` as mandatory and `debug_assert`
1686/// in `Drop` — and Wave 4.2 implemented it, measured the consequence, and
1687/// reduced it to a warning.** The assert fired on roughly thirty tests on its
1688/// first run. That is the signal it was built to produce, and the right reading
1689/// of it was not "thirty tests are wrong".
1690///
1691/// What dropping actually costs is one final snapshot. Nothing else: every
1692/// public write method awaits its responder, so by the time a caller *can* drop
1693/// the handle, every write it issued has already committed; and the cadence stops
1694/// on its own, because `cadence_stop` is a `watch::Sender` whose drop signals the
1695/// task. A snapshot is derivative state under Doctrine VI — disposable,
1696/// reconstructible, and never the only copy of anything. Losing one makes the
1697/// next `reconstruct` fold from an older anchor, which is **slower, not wrong**.
1698///
1699/// A `debug_assert` aborts a test run. Spending that on a performance loss, in a
1700/// project whose own notes say a suite that fails for reasons unrelated to the
1701/// code under test trains people to ignore red, is the wrong trade — and paying
1702/// it in thirty places would have made `close()` look mandatory by ceremony
1703/// rather than by consequence. `close()` remains the right thing to call, and
1704/// the two reasons to call it are now stated where they can be acted on: the
1705/// snapshot, and the writer's `Result`, which only `close()` can return.
1706///
1707/// Option A ("abort the actor and log") stays rejected, for the reason it was
1708/// rejected twice before: `Drop` cannot await, so it cannot drain, and cleanup
1709/// that cannot clean up is worse than none — it looks like cleanup.
1710impl Drop for Database {
1711    fn drop(&mut self) {
1712        if !self.closed {
1713            tracing::warn!(
1714                "Database dropped without close(): the final snapshot was not written, \
1715                 so the next reconstruct folds from an older anchor, and the write \
1716                 actor's exit status was not checked. Prefer close().await."
1717            );
1718        }
1719    }
1720}
1721
1722fn normalize_all(edges: Vec<EdgeAssertion>) -> Result<Vec<EdgeAssertion>> {
1723    edges.into_iter().map(EdgeAssertion::normalized).collect()
1724}
1725
1726/// Identical pragma configuration on every connection.
1727async fn configure(conn: libsql::Connection) -> Result<libsql::Connection> {
1728    // NOTE: `journal_mode` and `busy_timeout` return their resulting value as a
1729    // row, and libsql's `execute()` rejects any statement that yields rows
1730    // ("Execute returned rows"). They must be issued through `query()`.
1731    let _ = conn.query("PRAGMA journal_mode = WAL", ()).await?;
1732    let _ = conn.query("PRAGMA busy_timeout = 5000", ()).await?;
1733    conn.execute("PRAGMA synchronous = NORMAL", ()).await?;
1734    conn.execute("PRAGMA foreign_keys = ON", ()).await?;
1735    conn.execute("PRAGMA recursive_triggers = OFF", ()).await?;
1736    Ok(conn)
1737}
1738
1739/// Helper to derive the snapshot directory by convention: foo.db -> foo_snapshots/
1740fn derive_snapshots_dir(path: &Path) -> PathBuf {
1741    let mut dir = path.to_path_buf();
1742    let stem = path
1743        .file_stem()
1744        .and_then(|s| s.to_str())
1745        .unwrap_or("macrame");
1746    dir.set_file_name(format!("{stem}_snapshots"));
1747    dir
1748}
1749
1750/// Helper to derive archive database path by convention: foo.db -> foo_archive.db
1751fn derive_archive_path(path: &Path) -> PathBuf {
1752    let mut archive = path.to_path_buf();
1753    if let Some(stem) = path.file_stem().and_then(|s| s.to_str()) {
1754        let ext = path.extension().and_then(|e| e.to_str()).unwrap_or("db");
1755        archive.set_file_name(format!("{stem}_archive.{ext}"));
1756    } else {
1757        archive.set_extension("archive.db");
1758    }
1759    archive
1760}
1761
1762/// Dedicated Write Actor event loop prioritizing high-priority UI requests over low-priority background work.
1763///
1764/// # The turn is the unit, not the statement (T1.4)
1765///
1766/// One iteration of this loop is one *hold*: the actor is single-threaded and
1767/// the SQLite write lock is not preemptible, so from the moment a command starts
1768/// executing until it returns, nothing else writes. That is the quantity
1769/// [`CHUNK_BUDGET`] bounds, and so it is the quantity
1770/// [`crate::metrics::ActorMetrics`] measures — deliberately around the whole
1771/// `execute` call rather than inside it. Timing the SQL alone would have
1772/// reported a bound that held while callers waited.
1773///
1774/// Queue depth is sampled *before* the `select!`, so it is the backlog the turn
1775/// found on arrival rather than the one it left behind.
1776async fn run_writer_actor(
1777    conn: libsql::Connection,
1778    clock: Arc<dyn Clock>,
1779    mut highpri_rx: mpsc::Receiver<HighPriCommand>,
1780    mut lowpri_rx: mpsc::Receiver<LowPriCommand>,
1781    shared: Arc<ActorShared>,
1782) -> Result<()> {
1783    loop {
1784        shared
1785            .metrics
1786            .record_turn(highpri_rx.len(), lowpri_rx.len());
1787
1788        let ctl = tokio::select! {
1789            biased;
1790            Some(cmd) = highpri_rx.recv() => {
1791                let turn = Turn::start(cmd.kind(), &shared);
1792                cmd.execute(&conn, &*clock, &turn).await
1793            }
1794            Some(cmd) = lowpri_rx.recv() => {
1795                let turn = Turn::start(cmd.kind(), &shared);
1796                cmd.execute(&conn, &*clock, &turn).await
1797            }
1798            else => LoopCtl::Break,
1799        };
1800        if matches!(ctl, LoopCtl::Break) {
1801            break;
1802        }
1803    }
1804    Ok(())
1805}
1806
1807/// One command's hold: the timer, its label, and the counters it reports to.
1808///
1809/// # The hold is recorded *before* the caller is answered, and it has to be
1810///
1811/// The obvious placement — time the whole `execute` call from the loop — is
1812/// wrong in a way that only shows up under test. Every arm of `execute` ends by
1813/// sending on a `oneshot`, which wakes the waiting caller; the actor then
1814/// returns to the loop and records. Those are two tasks, so a caller that awaits
1815/// its own write and immediately reads [`Database::metrics`] can be scheduled
1816/// first and see a turn count that does not include the write it just did.
1817///
1818/// Not a correctness bug in the ledger, and it would never have been noticed in
1819/// production — a dashboard sampling every few seconds cannot see the window.
1820/// It makes every test and diagnostic of the counters flaky, which is worse: the
1821/// instrumentation would have been *believed* while being wrong exactly when
1822/// someone tried to check it. `examples/bulk_atomic_diag.rs` was the thing that
1823/// caught it, reporting a 20,000-row batch as a 0 ms hold.
1824///
1825/// So `answer` records and then sends, in that order, and the ordering is the
1826/// method's whole reason to exist. What it costs is that the `oneshot::send`
1827/// itself falls outside the measurement, which is a few nanoseconds against a
1828/// turn measured in microseconds at best.
1829struct Turn<'a> {
1830    kind: crate::metrics::CommandKind,
1831    timer: crate::metrics::HoldTimer,
1832    shared: &'a ActorShared,
1833}
1834
1835/// State the actor owns and a `Turn` needs to reach.
1836///
1837/// `archive_epoch` is here rather than in [`crate::metrics::ActorMetrics`]
1838/// because it is **not** a metric: T1.2's shadow rebuild reads it to decide
1839/// whether its work is still valid, so it has to be present in every build, not
1840/// only under the `metrics` feature. Counting archives happens to be what both
1841/// want; only one of them is allowed to be compiled out.
1842#[derive(Default)]
1843struct ActorShared {
1844    metrics: crate::metrics::ActorMetrics,
1845    archive_epoch: std::sync::atomic::AtomicU64,
1846}
1847
1848impl<'a> Turn<'a> {
1849    fn start(kind: crate::metrics::CommandKind, shared: &'a ActorShared) -> Self {
1850        Self {
1851            kind,
1852            timer: crate::metrics::HoldTimer::start(),
1853            shared,
1854        }
1855    }
1856
1857    fn epoch(&self) -> u64 {
1858        self.shared
1859            .archive_epoch
1860            .load(std::sync::atomic::Ordering::Relaxed)
1861    }
1862
1863    /// Record that an archive session committed.
1864    ///
1865    /// Bumped on **success only**: a failed archive rolls back, so it deletes
1866    /// nothing and invalidates no shadow build.
1867    fn archive_committed(&self) {
1868        self.shared
1869            .archive_epoch
1870            .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
1871    }
1872
1873    /// Close the hold and hand the result back. Never the other way round.
1874    ///
1875    /// The `let _ =` on the send is deliberate and predates this: a caller that
1876    /// dropped its receiver — `tokio::time::timeout` around a write, which
1877    /// [`Database`]'s write surface explicitly documents — is not an actor
1878    /// error, and the command committed regardless.
1879    fn answer<T>(&self, responder: oneshot::Sender<Result<T>>, res: Result<T>) {
1880        self.shared
1881            .metrics
1882            .record_hold(self.kind, self.timer.elapsed());
1883        let _ = responder.send(res);
1884    }
1885}
1886
1887const INSERT_LINK: &str = "INSERT INTO links \
1888     (source_id, target_id, edge_type, valid_from, valid_to, weight, properties, recorded_at) \
1889     VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)";
1890
1891/// Shared by the single-concept write and the chunked one, so the two paths
1892/// cannot drift into upserting different column sets — and so the chunk has a
1893/// statement text it can prepare once (D-056).
1894const UPSERT_CONCEPT: &str = "INSERT INTO concepts \
1895     (id, title, content, embedding_model, valid_from, valid_to, recorded_at, retired) \
1896     VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8) \
1897     ON CONFLICT(id) DO UPDATE SET \
1898         title = excluded.title, \
1899         content = excluded.content, \
1900         embedding_model = excluded.embedding_model, \
1901         valid_from = excluded.valid_from, \
1902         valid_to = excluded.valid_to, \
1903         recorded_at = excluded.recorded_at, \
1904         retired = excluded.retired";
1905
1906/// The parameter row for [`UPSERT_CONCEPT`], in one place for the same reason.
1907fn concept_params<'a>(concept: &'a ConceptUpsert, stamp: &'a str) -> [libsql::Value; 8] {
1908    [
1909        concept.id.as_str().into(),
1910        concept.title.as_str().into(),
1911        concept.content.as_str().into(),
1912        concept
1913            .embedding_model
1914            .as_deref()
1915            .map_or(libsql::Value::Null, Into::into),
1916        concept.valid_from.as_str().into(),
1917        concept.valid_to.as_str().into(),
1918        stamp.into(),
1919        (concept.retired as i64).into(),
1920    ]
1921}
1922
1923impl HighPriCommand {
1924    /// The metrics label for this variant (T1.4).
1925    ///
1926    /// Exhaustive for the same reason `execute` is: a new variant that silently
1927    /// borrowed another's label would attribute its holds to the wrong command,
1928    /// and the one question the counters exist to answer is *which* command
1929    /// broke the budget.
1930    fn kind(&self) -> crate::metrics::CommandKind {
1931        use crate::metrics::CommandKind as K;
1932        match self {
1933            HighPriCommand::AssertEdge { .. } => K::AssertEdge,
1934            HighPriCommand::RetireEdge { .. } => K::RetireEdge,
1935            HighPriCommand::UpsertConcept { .. } => K::UpsertConcept,
1936            HighPriCommand::WriteBulkAtomic { .. } => K::WriteBulkAtomic,
1937            HighPriCommand::RebuildCurrent { .. } => K::RebuildCurrent,
1938            HighPriCommand::RegisterModel { .. } => K::RegisterModel,
1939            HighPriCommand::Shutdown { .. } => K::Shutdown,
1940        }
1941    }
1942
1943    /// Run one command and answer its caller.
1944    ///
1945    /// Deliberately exhaustive — there is no `_` arm. The 0.4.5–0.5.4 actor
1946    /// matched `Shutdown` and `AssertEdge` and sent everything else to
1947    /// `_ => LoopCtl::Continue`, which **dropped the responder**: the caller's
1948    /// `rx.await` resolved to a `RecvError` that no code mapped, so four of six
1949    /// commands were indistinguishable from a hung database. An exhaustive match
1950    /// makes that failure a compile error instead of a runtime silence, which is
1951    /// why adding a variant should break this function.
1952    async fn execute(
1953        self,
1954        conn: &libsql::Connection,
1955        clock: &dyn Clock,
1956        turn: &Turn<'_>,
1957    ) -> LoopCtl {
1958        match self {
1959            HighPriCommand::Shutdown { responder } => {
1960                turn.answer(responder, Ok(()));
1961                return LoopCtl::Break;
1962            }
1963            HighPriCommand::AssertEdge { edge, responder } => {
1964                let stamp = clock.now();
1965                if let Err(e) = reject_overlapping_interval(conn, &edge).await {
1966                    turn.answer(responder, Err(e));
1967                    return LoopCtl::Continue;
1968                }
1969                let res = match conn
1970                    .execute(
1971                        INSERT_LINK,
1972                        libsql::params![
1973                            edge.source.as_str(),
1974                            edge.target.as_str(),
1975                            edge.edge_type.as_str(),
1976                            edge.valid_from.as_str(),
1977                            edge.valid_to.as_str(),
1978                            edge.weight,
1979                            edge.properties.as_str(),
1980                            stamp.as_str()
1981                        ],
1982                    )
1983                    .await
1984                {
1985                    Ok(_) => Ok(()),
1986                    Err(e) => Err(classify(
1987                        conn,
1988                        e,
1989                        WriteOp::Edge {
1990                            source_id: &edge.source,
1991                            target_id: &edge.target,
1992                            edge_type: &edge.edge_type,
1993                        },
1994                    )
1995                    .await),
1996                };
1997                turn.answer(responder, res);
1998            }
1999            HighPriCommand::RetireEdge {
2000                source,
2001                target,
2002                edge_type,
2003                valid_from,
2004                valid_to,
2005                responder,
2006            } => {
2007                let stamp = clock.now();
2008                let res = retire_edge(
2009                    conn,
2010                    &source,
2011                    &target,
2012                    &edge_type,
2013                    &valid_from,
2014                    &valid_to,
2015                    &stamp,
2016                )
2017                .await;
2018                turn.answer(responder, res);
2019            }
2020            HighPriCommand::UpsertConcept { concept, responder } => {
2021                let stamp = clock.now();
2022                let res = upsert_concept(conn, &concept, &stamp).await;
2023                turn.answer(responder, res);
2024            }
2025            HighPriCommand::WriteBulkAtomic { edges, responder } => {
2026                // One stamp for the whole batch (D-014): the rows were asserted
2027                // by one act, and giving them different transaction times would
2028                // invent an ordering the caller never expressed.
2029                let stamp = clock.now();
2030                let res = write_edges_atomic(conn, &edges, &stamp).await;
2031                turn.answer(responder, res);
2032            }
2033            HighPriCommand::RebuildCurrent { responder } => {
2034                turn.answer(responder, rebuild_current(conn).await);
2035            }
2036            HighPriCommand::RegisterModel {
2037                model,
2038                dim,
2039                responder,
2040            } => {
2041                turn.answer(
2042                    responder,
2043                    crate::vector::register_model(conn, &model, dim).await,
2044                );
2045            }
2046        }
2047        LoopCtl::Continue
2048    }
2049}
2050
2051impl LowPriCommand {
2052    /// The metrics label for this variant (T1.4). See [`HighPriCommand::kind`].
2053    fn kind(&self) -> crate::metrics::CommandKind {
2054        use crate::metrics::CommandKind as K;
2055        match self {
2056            LowPriCommand::WriteConceptsChunk { .. } => K::WriteConceptsChunk,
2057            LowPriCommand::WriteAnalyticsChunk { .. } => K::WriteAnalyticsChunk,
2058            LowPriCommand::UpsertEmbeddingChunk { .. } => K::UpsertEmbeddingChunk,
2059            LowPriCommand::BulkImportChunk { .. } => K::BulkImportChunk,
2060            LowPriCommand::Archive { .. } => K::Archive,
2061            // No counter of its own: rehydration is the archive path run
2062            // backwards and shares its budget, and a `CommandKind` variant is a
2063            // public enum addition (D-036 periphery, but still a break).
2064            LowPriCommand::Rehydrate { .. } => K::Archive,
2065            LowPriCommand::RebuildFts { .. } => K::RebuildFts,
2066            LowPriCommand::ShadowRebuild { .. } => K::ShadowRebuild,
2067        }
2068    }
2069
2070    /// Run one background command and answer its caller.
2071    ///
2072    /// Also exhaustive. The pre-0.5.4 version was a single `LoopCtl::Continue`
2073    /// for *every* variant — every background write silently discarded, its
2074    /// caller waiting forever.
2075    async fn execute(
2076        self,
2077        conn: &libsql::Connection,
2078        clock: &dyn Clock,
2079        turn: &Turn<'_>,
2080    ) -> LoopCtl {
2081        match self {
2082            LowPriCommand::BulkImportChunk { chunk, responder } => {
2083                // A stamp per chunk, not per batch: the chunks commit
2084                // separately, so a shared stamp would claim a simultaneity the
2085                // storage does not have.
2086                let stamp = clock.now();
2087                turn.answer(responder, write_edges_atomic(conn, &chunk, &stamp).await);
2088            }
2089            LowPriCommand::WriteConceptsChunk { chunk, responder } => {
2090                let stamp = clock.now();
2091                turn.answer(responder, write_concepts_atomic(conn, &chunk, &stamp).await);
2092            }
2093            LowPriCommand::WriteAnalyticsChunk { chunk, responder } => {
2094                let stamp = clock.now();
2095                turn.answer(
2096                    responder,
2097                    write_annotations_atomic(conn, &chunk, &stamp).await,
2098                );
2099            }
2100            LowPriCommand::UpsertEmbeddingChunk {
2101                model,
2102                chunk,
2103                responder,
2104            } => {
2105                // No clock reading: an embedding carries no timestamp on either
2106                // axis. It is a derived artifact of a model applied to content
2107                // (Doctrine VII), and the ledger already records when the
2108                // content changed.
2109                turn.answer(
2110                    responder,
2111                    crate::vector::search::upsert_embedding_chunk(conn, &model, &chunk).await,
2112                );
2113            }
2114            LowPriCommand::Archive {
2115                cutoff,
2116                archive_path,
2117                responder,
2118            } => {
2119                // The archive *time*, not the cutoff. `archive_horizon` records
2120                // both and they are different facts — see `archive()` (Wave 4.5).
2121                let archived_at = clock.now();
2122                let res = archive(conn, &cutoff, &archived_at, &archive_path).await;
2123                // Before the answer, so a shadow rebuild that reads the epoch on
2124                // its next turn cannot miss an archive that has already deleted
2125                // rows out from under it (T1.2).
2126                if res.is_ok() {
2127                    turn.archive_committed();
2128                }
2129                turn.answer(responder, res);
2130            }
2131            LowPriCommand::Rehydrate {
2132                ids,
2133                archive_path,
2134                responder,
2135            } => {
2136                let refs: Vec<&str> = ids.iter().map(String::as_str).collect();
2137                let res = rehydrate(conn, &refs, &archive_path).await;
2138                // Same reason as `Archive`: rehydration moves rows into `links`'
2139                // parent table, so a shadow rebuild in flight must see the epoch
2140                // move before the caller is answered (T1.2).
2141                if res.is_ok() {
2142                    turn.archive_committed();
2143                }
2144                turn.answer(responder, res);
2145            }
2146            LowPriCommand::ShadowRebuild { step, responder } => {
2147                use crate::integrity::{shadow, ShadowOutcome, ShadowStep};
2148                let res = match step {
2149                    ShadowStep::Begin => {
2150                        shadow::begin(conn)
2151                            .await
2152                            .map(|build_start| ShadowOutcome::Started {
2153                                build_start,
2154                                epoch: turn.epoch(),
2155                            })
2156                    }
2157                    ShadowStep::Fill { after } => shadow::fill_chunk(conn, after.as_deref())
2158                        .await
2159                        .map(|last| ShadowOutcome::Filled { last }),
2160                    ShadowStep::Swap { build_start, epoch } => {
2161                        shadow::swap(conn, &build_start, epoch, turn.epoch())
2162                            .await
2163                            .map(|rows| ShadowOutcome::Swapped { rows })
2164                    }
2165                };
2166                turn.answer(responder, res);
2167            }
2168            LowPriCommand::RebuildFts { responder } => {
2169                let res = conn
2170                    .execute(crate::schema::ddl::REBUILD_CONCEPTS_FTS, ())
2171                    .await
2172                    .map(|_| ())
2173                    .map_err(Into::into);
2174                turn.answer(responder, res);
2175            }
2176        }
2177        LoopCtl::Continue
2178    }
2179}
2180
2181/// Close an open interval by asserting its successor (Doctrine III).
2182///
2183/// Never an `UPDATE`. The replacement row copies weight and properties from
2184/// current belief and differs only in `valid_to` and `recorded_at`, so the
2185/// original assertion survives intact and `reconstruct` at an earlier instant
2186/// still sees the interval open — which is the entire point of a bitemporal
2187/// ledger.
2188async fn retire_edge(
2189    conn: &libsql::Connection,
2190    source: &str,
2191    target: &str,
2192    edge_type: &str,
2193    valid_from: &str,
2194    valid_to: &str,
2195    stamp: &str,
2196) -> Result<()> {
2197    let affected = conn
2198        .execute(
2199            "INSERT INTO links \
2200                 (source_id, target_id, edge_type, valid_from, valid_to, weight, properties, recorded_at) \
2201             SELECT source_id, target_id, edge_type, valid_from, ?5, weight, properties, ?6 \
2202             FROM links_current \
2203             WHERE source_id = ?1 AND target_id = ?2 AND edge_type = ?3 AND valid_from = ?4",
2204            libsql::params![source, target, edge_type, valid_from, valid_to, stamp],
2205        )
2206        .await
2207        .map_err(DbError::Engine)?;
2208
2209    if affected == 0 {
2210        return Err(DbError::NotFound(format!(
2211            "{source} -> {target} ({edge_type}) at {valid_from}"
2212        )));
2213    }
2214    Ok(())
2215}
2216
2217async fn upsert_concept(
2218    conn: &libsql::Connection,
2219    concept: &ConceptUpsert,
2220    stamp: &str,
2221) -> Result<()> {
2222    let res = conn
2223        .execute(UPSERT_CONCEPT, concept_params(concept, stamp))
2224        .await;
2225
2226    match res {
2227        Ok(_) => Ok(()),
2228        Err(e) => Err(classify(
2229            conn,
2230            e,
2231            WriteOp::Concept {
2232                id: &concept.id,
2233                recorded_at: stamp,
2234            },
2235        )
2236        .await),
2237    }
2238}
2239
2240/// Every recorded interval for one relationship key, for [`Interval::overlaps`]
2241/// to judge.
2242///
2243/// **Three equalities and nothing else, deliberately — and the "and nothing
2244/// else" was measured, not assumed.** The first version added
2245/// `AND valid_from < :new_valid_to`, a provably safe narrowing (overlap requires
2246/// `max(start) < min(end)`, so an interval starting at or after the new one's end
2247/// cannot overlap it). It cost **9.8 ms on a 90-edge chunk into a 2,000-edge
2248/// hub**, because it walked the planner straight into D-059's trap:
2249///
2250/// ```text
2251/// with the range:     SEARCH links_current USING COVERING INDEX
2252///                     idx_lc_traversal_cover (source_id=? AND valid_from<?)
2253/// without it:         SEARCH links_current USING COVERING INDEX
2254///                     idx_lc_open_interval (source_id=? AND target_id=? AND edge_type=?)
2255/// ```
2256///
2257/// `idx_lc_traversal_cover` leads on `(source_id, valid_from, …)` and contains
2258/// every column this query mentions, so with a `valid_from` range available it
2259/// wins as a covering index while binding **one** equality column — and the
2260/// guard scans the source's entire out-degree. That is the same shape as the
2261/// defect D-059 diagnosed in `trg_links_single_open`, reintroduced by an
2262/// optimisation, one wave after it was fixed.
2263///
2264/// Dropping the range makes the query a pure three-column point lookup that
2265/// `idx_lc_open_interval` serves exactly, and the rows it returns are the
2266/// intervals recorded for one `(source, target, edge_type)` — a version count,
2267/// not an out-degree. **A narrowing predicate is not free if it changes the
2268/// plan**, which is the general lesson and the reason this constant carries its
2269/// own `EXPLAIN` output.
2270const OVERLAP_CANDIDATES: &str = "SELECT valid_from, valid_to FROM links_current \
2271     WHERE source_id = ?1 AND target_id = ?2 AND edge_type = ?3 \
2272       AND valid_from <> ?4";
2273
2274/// Whether this pair is the storage layer's case rather than this guard's.
2275///
2276/// Two **open** intervals overlap — they share every instant from the later
2277/// start onwards — so a naive overlap check reports them, and reporting them
2278/// here would leave `DbError::SingleOpenViolation` constructible by nothing.
2279/// That variant is the more specific error, it is enforced by
2280/// `trg_links_single_open` rather than by this function, and its field names
2281/// were ratified in §1.2. Shadowing it with a general one would be defect Q's
2282/// shape reintroduced by a fix: a typed error that no code path can produce.
2283///
2284/// So the two guards partition the space rather than overlapping it. Both open
2285/// belongs to the trigger. Everything else — open against closed, closed against
2286/// closed — is unguarded at the storage layer and belongs here. That the split
2287/// is exactly the trigger's `WHEN` clause is not a coincidence; it is the
2288/// definition of what was missing.
2289fn defer_to_single_open(proposed: &Interval, existing: &Interval) -> bool {
2290    proposed.is_open() && existing.is_open()
2291}
2292
2293/// Refuse an assertion whose valid-time interval overlaps one already recorded
2294/// for the same `(source, target, edge_type)` — **defect AA, D-060**.
2295///
2296/// `trg_links_single_open` fires only `WHEN NEW.valid_to = '9999-…'`, so it
2297/// guards the open sentinel and nothing else. Two *closed* intervals that
2298/// overlap were accepted without complaint, and `query_as_of_edges` at an
2299/// instant inside both returned one relationship as two edges.
2300///
2301/// **This runs in the write actor, which is what makes it sound.** The obvious
2302/// place is `EdgeAssertion::normalized`, and it cannot go there — `normalized`
2303/// is a pure function with no connection, and doing the read at the API boundary
2304/// instead would leave a check-then-write race between the read and the actor's
2305/// insert. Inside the actor there is one writer by construction (D-014), and for
2306/// the batch paths this runs inside the same transaction as the insert, so the
2307/// window does not exist rather than being small.
2308///
2309/// **What it does not cover, and §4.2 now says so:** raw SQL against the same
2310/// file. The storage layer permits what this API refuses, which is the honest
2311/// cost of not putting the check in a trigger. The alternative was a second
2312/// index probe inside `trg_links_single_open` on every insert — on the path
2313/// D-059 has just finished making fast — for a guarantee that only holds against
2314/// callers who were going through the actor anyway.
2315///
2316/// `valid_from <> ?4` excludes the row being re-asserted. Re-assertion at the
2317/// same `valid_from` is Doctrine III's ordinary case — a new belief about the
2318/// same interval — and is settled by the primary key and the single-open
2319/// trigger, not here.
2320/// The single-assertion path prepares one statement for one check, which is what
2321/// `AssertEdge` needs; the batch path prepares once and calls
2322/// [`check_prepared`] per row.
2323async fn reject_overlapping_interval(
2324    conn: &libsql::Connection,
2325    edge: &EdgeAssertion,
2326) -> Result<()> {
2327    let stmt = conn.prepare(OVERLAP_CANDIDATES).await?;
2328    check_prepared(&stmt, edge).await
2329}
2330
2331/// The guard's body, against a statement the caller has already prepared.
2332///
2333/// **Split out because preparing per row was worth 10.4 ms on a 90-edge chunk**
2334/// (§8.8) — the same defect D-056 and D-057 diagnosed and fixed for
2335/// `INSERT_LINK`, reintroduced by the Wave 2 guard that was written beside it.
2336/// Measured with and without the guard, on a 2,000-edge hub: 8.65 ms → 19.25 ms,
2337/// and *identical* with and without `idx_lc_open_interval`, which is what
2338/// identified preparation rather than a scan as the cost. A guard that reads an
2339/// index correctly and prepares its statement 90 times is indistinguishable, at
2340/// the call site, from one that scans.
2341///
2342/// `reset()` between rows is not optional: libsql binds and steps without
2343/// resetting, so a reused statement must be returned to its initial state.
2344async fn check_prepared(stmt: &libsql::Statement, edge: &EdgeAssertion) -> Result<()> {
2345    let proposed = Interval::new(edge.valid_from.clone(), edge.valid_to.clone());
2346
2347    stmt.reset();
2348    let mut rows = stmt
2349        .query(libsql::params![
2350            edge.source.as_str(),
2351            edge.target.as_str(),
2352            edge.edge_type.as_str(),
2353            edge.valid_from.as_str()
2354        ])
2355        .await?;
2356
2357    while let Some(row) = rows.next().await? {
2358        let existing = Interval::new(row.get::<String>(0)?, row.get::<String>(1)?);
2359        if defer_to_single_open(&proposed, &existing) {
2360            continue;
2361        }
2362        if proposed.overlaps(&existing) {
2363            return Err(DbError::OverlappingInterval {
2364                overlap: Box::new(crate::error::Overlap {
2365                    source_id: edge.source.clone(),
2366                    target_id: edge.target.clone(),
2367                    edge_type: edge.edge_type.clone(),
2368                    valid_from: edge.valid_from.clone(),
2369                    valid_to: edge.valid_to.clone(),
2370                    existing_from: existing.valid_from,
2371                    existing_to: existing.valid_to,
2372                }),
2373            });
2374        }
2375    }
2376
2377    Ok(())
2378}
2379
2380/// The same guard applied *within* a batch, before any of it is written.
2381///
2382/// The database check cannot see rows that are not in the database yet, so a
2383/// batch carrying two overlapping intervals for one relationship would pass
2384/// every per-row check and commit the overlap in one transaction. Quadratic in
2385/// the batch, which is affordable because the chunk is bounded at
2386/// [`chunk_rows::EDGES`] = 90 and because the comparison is a pair of string
2387/// compares — and because grouping first means the inner loop only ever runs
2388/// over edges sharing a key, which is normally one.
2389fn reject_overlaps_within(edges: &[EdgeAssertion]) -> Result<()> {
2390    for (i, a) in edges.iter().enumerate() {
2391        let ia = Interval::new(a.valid_from.clone(), a.valid_to.clone());
2392        for b in &edges[i + 1..] {
2393            if a.source != b.source || a.target != b.target || a.edge_type != b.edge_type {
2394                continue;
2395            }
2396            // Identical valid_from is re-assertion within one batch: the last
2397            // writer wins by seq_id, as it does across batches. Not an overlap.
2398            if a.valid_from == b.valid_from {
2399                continue;
2400            }
2401            let ib = Interval::new(b.valid_from.clone(), b.valid_to.clone());
2402            // Both open is the trigger's case; it fires during the insert and
2403            // rolls the batch back with the more specific error.
2404            if defer_to_single_open(&ia, &ib) {
2405                continue;
2406            }
2407            if ia.overlaps(&ib) {
2408                return Err(DbError::OverlappingInterval {
2409                    overlap: Box::new(crate::error::Overlap {
2410                        source_id: a.source.clone(),
2411                        target_id: a.target.clone(),
2412                        edge_type: a.edge_type.clone(),
2413                        valid_from: a.valid_from.clone(),
2414                        valid_to: a.valid_to.clone(),
2415                        existing_from: ib.valid_from,
2416                        existing_to: ib.valid_to,
2417                    }),
2418                });
2419            }
2420        }
2421    }
2422    Ok(())
2423}
2424
2425/// Write every edge or none, under a single stamp.
2426///
2427/// **The statement is prepared once for the whole chunk (§9, D-056).** It used to
2428/// be `tx.execute(INSERT_LINK, …)` per row, which re-prepares on every call — and
2429/// `links` carries two triggers, so each preparation compiles their bodies along
2430/// with the insert.
2431///
2432/// Measured at 500 rows: **≈62 ms → ≈37 ms, a 41% saving.** Preparation was a
2433/// large cost and *not* the dominant one, which the first guess had it as. The
2434/// residual is the triggers themselves: the same 500 rows with
2435/// `trg_links_log_insert` and `trg_links_current_sync` dropped commit in **2.96
2436/// ms**, so trigger amplification is ~92% of what remains. There is no further
2437/// win available here without changing what the ledger records, and Doctrine IV
2438/// is what says it must be recorded. See D-056 for what that implies about §9's
2439/// ≤ 3 ms budget — briefly, 2.96 ms *is* the un-amplified figure, so the budget
2440/// appears to have been set without the amplification its own preamble says is
2441/// included.
2442///
2443/// `reset()` between rows is not optional: libsql's `execute` binds and steps
2444/// without resetting, so a reused statement must be returned to its initial state
2445/// or the second row steps a completed statement.
2446async fn write_edges_atomic(
2447    conn: &libsql::Connection,
2448    edges: &[EdgeAssertion],
2449    stamp: &str,
2450) -> Result<usize> {
2451    if edges.is_empty() {
2452        return Ok(0);
2453    }
2454
2455    // Before the transaction opens: a batch that contradicts itself is refused
2456    // without taking the write lock at all (D-060).
2457    reject_overlaps_within(edges)?;
2458
2459    let tx = conn
2460        .transaction_with_behavior(libsql::TransactionBehavior::Immediate)
2461        .await?;
2462
2463    // Inside the transaction, so the rows this checks against cannot change
2464    // between the check and the insert.
2465    // One preparation for the whole chunk, not one per row — see
2466    // `check_prepared`, and D-056 for the same lesson learned on `INSERT_LINK`.
2467    let guard = tx.prepare(OVERLAP_CANDIDATES).await?;
2468    for edge in edges {
2469        if let Err(e) = check_prepared(&guard, edge).await {
2470            // Released before the rollback: a live statement on the connection
2471            // is what makes SQLite refuse to end a transaction.
2472            drop(guard);
2473            let _ = tx.rollback().await;
2474            return Err(e);
2475        }
2476    }
2477    drop(guard);
2478
2479    let stmt = tx.prepare(INSERT_LINK).await?;
2480
2481    for edge in edges {
2482        stmt.reset();
2483        let res = stmt
2484            .execute(libsql::params![
2485                edge.source.as_str(),
2486                edge.target.as_str(),
2487                edge.edge_type.as_str(),
2488                edge.valid_from.as_str(),
2489                edge.valid_to.as_str(),
2490                edge.weight,
2491                edge.properties.as_str(),
2492                stamp
2493            ])
2494            .await;
2495
2496        if let Err(e) = res {
2497            let typed = classify(
2498                &tx,
2499                e,
2500                WriteOp::Edge {
2501                    source_id: &edge.source,
2502                    target_id: &edge.target,
2503                    edge_type: &edge.edge_type,
2504                },
2505            )
2506            .await;
2507            // Released before the rollback: a live statement on the connection
2508            // is exactly what makes SQLite refuse to end a transaction.
2509            drop(stmt);
2510            let _ = tx.rollback().await;
2511            return Err(typed);
2512        }
2513    }
2514
2515    drop(stmt);
2516    tx.commit().await?;
2517    Ok(edges.len())
2518}
2519
2520/// Write every concept or none, under a single stamp.
2521/// Upsert one chunk of derived annotations in a single transaction (D-041).
2522///
2523/// `stamp` is the actor's clock reading, exactly as for every other chunk — but
2524/// it lands in `computed_at`, not in a `recorded_at`, and the difference is not
2525/// cosmetic. `recorded_at` is the transaction-time axis and is subject to
2526/// Doctrine II and the monotonicity guard; `computed_at` is a note about when a
2527/// derivation last ran, on a table the ledger does not see. Rerunning an
2528/// algorithm therefore replaces the row and advances the note, rather than
2529/// versioning a concept the world did not change.
2530async fn write_annotations_atomic(
2531    conn: &libsql::Connection,
2532    annotations: &[Annotation],
2533    stamp: &str,
2534) -> Result<usize> {
2535    if annotations.is_empty() {
2536        return Ok(0);
2537    }
2538
2539    let tx = conn
2540        .transaction_with_behavior(libsql::TransactionBehavior::Immediate)
2541        .await?;
2542
2543    let stmt = tx
2544        .prepare(
2545            "INSERT INTO analytics_annotations (concept_id, label, value, computed_at) \
2546             VALUES (?1, ?2, ?3, ?4) \
2547             ON CONFLICT(concept_id, label) DO UPDATE SET \
2548                 value = excluded.value, computed_at = excluded.computed_at",
2549        )
2550        .await?;
2551
2552    for a in annotations {
2553        stmt.reset();
2554        let res = stmt
2555            .execute(libsql::params![
2556                a.concept_id.as_str(),
2557                a.label.as_str(),
2558                a.value.as_str(),
2559                stamp
2560            ])
2561            .await;
2562        if let Err(e) = res {
2563            drop(stmt);
2564            let _ = tx.rollback().await;
2565            return Err(DbError::Engine(e));
2566        }
2567    }
2568
2569    drop(stmt);
2570    tx.commit().await?;
2571    Ok(annotations.len())
2572}
2573
2574async fn write_concepts_atomic(
2575    conn: &libsql::Connection,
2576    concepts: &[ConceptUpsert],
2577    stamp: &str,
2578) -> Result<usize> {
2579    if concepts.is_empty() {
2580        return Ok(0);
2581    }
2582
2583    let tx = conn
2584        .transaction_with_behavior(libsql::TransactionBehavior::Immediate)
2585        .await?;
2586
2587    // Prepared once, like the edge chunk (D-056). This no longer routes through
2588    // [`upsert_concept`] — that function prepares per call by construction — but
2589    // it shares that function's statement text and parameter row, so the two
2590    // cannot upsert different columns.
2591    let stmt = tx.prepare(UPSERT_CONCEPT).await?;
2592
2593    for concept in concepts {
2594        stmt.reset();
2595        let res = stmt.execute(concept_params(concept, stamp)).await;
2596
2597        if let Err(e) = res {
2598            let typed = classify(
2599                &tx,
2600                e,
2601                WriteOp::Concept {
2602                    id: &concept.id,
2603                    recorded_at: stamp,
2604                },
2605            )
2606            .await;
2607            drop(stmt);
2608            let _ = tx.rollback().await;
2609            return Err(typed);
2610        }
2611    }
2612
2613    drop(stmt);
2614    tx.commit().await?;
2615    Ok(concepts.len())
2616}
2617
2618#[cfg(test)]
2619mod tests {
2620    use super::*;
2621
2622    fn edge(target: &str, micros: usize) -> EdgeAssertion {
2623        EdgeAssertion::new("src", target, "LINKS")
2624            .valid_from(format!("2026-01-01T00:00:00.{micros:06}Z"))
2625            .valid_to(format!("2026-01-01T00:00:00.{:06}Z", micros + 1))
2626    }
2627
2628    /// The estimate must depend on the batch's **shape**, not only its size.
2629    ///
2630    /// This is the correction T1.3's "rows × per-row cost" needed. Two batches
2631    /// of the same length whose measured holds differ by 7× must not be
2632    /// predicted identically, and the direction matters: a model that averages
2633    /// the two under-predicts the expensive shape, which is the only one anyone
2634    /// needs warning about.
2635    #[test]
2636    fn two_batches_of_one_size_are_not_predicted_alike() {
2637        const N: usize = 20_000;
2638        let fanout: Vec<_> = (0..N).map(|i| edge(&format!("t{i:07}"), i)).collect();
2639        let history: Vec<_> = (0..N).map(|i| edge("t0", i)).collect();
2640
2641        let (a, b) = (estimated_bulk_hold(&fanout), estimated_bulk_hold(&history));
2642        assert!(
2643            b > a * 5,
2644            "the guard's expensive path is 16x dearer per pair and this batch \
2645             takes it on every pair, but the estimates are {a:?} and {b:?}"
2646        );
2647    }
2648
2649    /// Measured on libSQL 0.9.30: 2.5 s and 18.6 s for those two batches. The
2650    /// estimator tracked both within 5%, and this pins that it still does — a
2651    /// coefficient edited without re-measuring fails here.
2652    #[test]
2653    fn the_estimate_matches_what_was_measured() {
2654        const N: usize = 20_000;
2655        let fanout: Vec<_> = (0..N).map(|i| edge(&format!("t{i:07}"), i)).collect();
2656        let history: Vec<_> = (0..N).map(|i| edge("t0", i)).collect();
2657
2658        for (batch, measured_ms, label) in
2659            [(fanout, 2_618u128, "fanout"), (history, 18_057, "history")]
2660        {
2661            let predicted = estimated_bulk_hold(&batch).as_millis();
2662            let ratio = predicted as f64 / measured_ms as f64;
2663            assert!(
2664                (0.8..1.25).contains(&ratio),
2665                "{label}: predicted {predicted} ms against a measured \
2666                 {measured_ms} ms ({ratio:.2}x). Re-run \
2667                 examples/bulk_atomic_diag.rs before changing the coefficients."
2668            );
2669        }
2670    }
2671
2672    /// An empty or single-edge batch has no pairs, and the arithmetic must not
2673    /// underflow computing it.
2674    #[test]
2675    fn a_batch_too_small_to_have_pairs_still_estimates() {
2676        assert_eq!(estimated_bulk_hold(&[]), std::time::Duration::ZERO);
2677        let one = [edge("t0", 0)];
2678        assert_eq!(
2679            estimated_bulk_hold(&one),
2680            std::time::Duration::from_nanos(73_000)
2681        );
2682    }
2683
2684    /// The warning threshold sits well above the bound this path is exempt from.
2685    ///
2686    /// Warning at `CHUNK_BUDGET` would fire on batches working exactly as
2687    /// designed — the exemption is a contract (D-014), not a failure — and a
2688    /// warning that fires on correct behaviour gets filtered out, taking the
2689    /// 18-second case with it.
2690    #[test]
2691    fn the_warning_threshold_is_not_the_chunk_budget() {
2692        assert!(BULK_ATOMIC_WARN_HOLD > CHUNK_BUDGET * 10);
2693    }
2694}