macrame/connection.rs
1use std::path::{Path, PathBuf};
2use std::sync::Arc;
3use tokio::sync::{mpsc, oneshot};
4
5use crate::error::{classify, BulkInterrupted, BulkResult, DbError, Result, WriteOp};
6use crate::graph::edge::EdgeAssertion;
7use crate::graph::lineage::{Ancestor, LineageShape, Lineages};
8use crate::integrity::{rebuild_current, RebuildReport};
9use crate::plan::ReadPlan;
10use crate::schema::migrations;
11use crate::temporal::archive::{archive, rehydrate, ArchiveReport, RehydrateReport};
12use crate::temporal::interval::Interval;
13use crate::temporal::snapshot::{self, SnapshotCadence};
14use crate::util::clock::FutureStampPolicy;
15use crate::util::clock::{Clock, SystemClock};
16use crate::util::timestamp;
17use crate::vector::ModelName;
18
19/// Rows per chunk on the background write paths (§5.1.5, D-011, D-014, D-058).
20///
21/// The Write Actor holds the sole write connection, so a single large statement
22/// blocks every other writer for its duration. Chunking bounds that stall; the
23/// cost is that a bulk import is *not* atomic across chunks, which is why
24/// all-or-nothing is [`Database::write_bulk_atomic`] — a separate entry point,
25/// with its own command on the actor's protocol — rather than a tuning
26/// parameter here.
27///
28/// # Why these are four constants and not one
29///
30/// Through 0.5.5 this was a single `CHUNK_ROWS = 1000` for all four bulk paths.
31/// The golden rule it was meant to serve is a bound on *duration* — a background
32/// chunk must commit fast enough that an interactive write queued behind it is
33/// not made to wait — and one row count cannot express one duration across paths
34/// whose measured per-row costs differ by 60× (D-058). At 1,000 rows the four
35/// paths took 3.5 ms, 24 ms, 89 ms and 143 ms: the same constant, four answers,
36/// three of them far outside the bound.
37///
38/// Each size below is derived from `benches/budgets.rs`'s `chunk_scaling`
39/// sweep against [`CHUNK_BUDGET`], then verified by measuring that size directly.
40/// They are *measurements of this machine*, not universal constants — D-055's
41/// reasoning about reference hardware applies here too, and re-deriving them on
42/// materially different storage is a `cargo bench` away.
43///
44/// # Sized for the tail, not the median
45///
46/// The first derivation solved `f + c·n = 3 ms` exactly and produced sizes whose
47/// *median* commit was 2.93 ms and whose upper estimate was 2.96 — inside the
48/// bound as reported and outside it for any chunk slower than typical. A latency
49/// bound is a statement about the chunk an unlucky interactive write actually
50/// queues behind, so these solve for ≈2.5 ms instead, leaving the remainder as
51/// headroom for the tail. That costs a few percent of throughput on the two
52/// linear paths and nothing on the two superlinear ones.
53///
54/// As measured by `chunk_budget`, each at its own size: edges **2.39 ms**,
55/// concepts **2.35 ms**, annotations **2.36 ms**, embeddings **2.06 ms**, no
56/// upper estimate above 2.42.
57///
58/// # Known limitation: these are empty-database figures
59///
60/// `chunk_budget` seeds concepts and starts with **no links and no vectors**,
61/// and D-059 established that per-row cost on the edge and embedding paths grows
62/// with the size of the structure being written, not with the chunk. The same
63/// 90-edge chunk takes **9.06 ms** into an 8,000-edge table. So the bound is met
64/// as measured here and *not* met on a populated database.
65///
66/// That gap was published as 47.7 ms until 0.10.0 and attributed to the schema
67/// defect D-059 documents. The defect was fixed by the `v5 → v6` rung and the
68/// figure was never updated. 9.08 ms is a 0.10.0 measurement, not D-059's 8.0 ms
69/// carried forward: `chunk_budget` gained a seeded arm, because until it did,
70/// nothing in the bench suite wrote a chunk into a populated table and this
71/// number was unfalsifiable. It agrees with D-059 once the session is accounted
72/// for — the empty arm read 2.69 and 2.65 ms beside it against the 2.39 ms
73/// published above, so the *ratio* is 3.4× here and 3.35× there.
74///
75/// **The residual is attributed as of 0.11.0 (D-142).** It is not the missing
76/// index, which shipped in 0.5.6; it is the `links_current` write. Dropping the
77/// three `links` insert triggers one at a time puts effectively all of the
78/// growth in `trg_links_current_sync` — the single-open guard contributes none,
79/// the log trigger and the base insert ~0.35 ms of a 4.15 ms rise — and within
80/// that trigger, 89% of the growth is maintenance of `idx_lc_traversal_cover`
81/// and `idx_lc_open_interval` rather than the upsert itself, which costs 0.49 ms
82/// run directly against the same table. Page-cache size, foreign keys and the
83/// fixture's key distribution were each tested and are each not the cause.
84///
85/// Knowing the cause does not by itself change the constant: the expensive index
86/// is D-042's covering index for the traversal, so narrowing it moves cost onto
87/// the read path it exists to protect. Re-deriving these constants against the
88/// D-088 fixture matrix is the named successor.
89///
90/// # These are ceilings as of 0.12.0, not sizes
91///
92/// D-143 re-derived all four against the D-088 matrix and the edge path came
93/// back **20** against a shipped 90 — and 20 would have been wrong at 80,000
94/// edges for the same reason 90 is wrong at 8,000, because per-row cost there
95/// grows with `links_current`. The finding was that no row count can bound a
96/// duration on such a path.
97///
98/// So the chunk loop stopped trying to pick one ahead of time. Each chunk is
99/// timed by the actor and its measured hold chooses the next size; these
100/// constants are the **largest** size that will ever be asked for, and every
101/// derivation below still applies to them as such. A path may run well under its
102/// constant on a populated database and at exactly it on an empty one, and both
103/// are the bound being met rather than a size being missed.
104pub mod chunk_rows {
105 /// Edge assertions (`bulk_import`).
106 ///
107 /// Per-row cost on this path rises with the size of `links_current`, not
108 /// with the chunk (D-059) — so cutting the chunk buys latency and costs
109 /// throughput, ~11% for 1,000 edges. An earlier version of this comment
110 /// claimed it was 3.3× *faster*; that came from multiplying eleven copies of
111 /// a chunk measured into an empty database.
112 ///
113 /// **This size does not meet the 3 ms bound on a populated database.** 90
114 /// edges into an 8,000-edge table take **9.06 ms** — measured, two sessions
115 /// at 9.08 and 9.05, against an empty-table arm of 2.69 and 2.65 beside
116 /// them (D-136).
117 ///
118 /// The reason given here until 0.10.0 — that `trg_links_single_open`'s
119 /// `EXISTS` scans the whole out-degree, "a schema defect with a proven fix,
120 /// recorded in D-059 and not applied here" — described 0.5.5. The fix *was*
121 /// applied, as the `v5 → v6` rung, and took this from 47.7 ms to ~8 ms.
122 /// What survives is the miss: the bound is still exceeded ~3×. Its cause is
123 /// no longer unknown — D-142 attributes it to `trg_links_current_sync`, and
124 /// within that to secondary-index maintenance on `links_current` — and the
125 /// guard this comment used to blame contributes **no** growth at all.
126 ///
127 /// **The constant is unchanged, and that is now a measured decision**
128 /// (D-143). Re-derived against all four D-088 shapes at 8,000 edges, they
129 /// agree that the largest size meeting the bound is **20**. It stays at 90
130 /// because 20 is the same miss at a larger population — per-row cost grows
131 /// with `links_current`, so a constant fitted at 8,000 edges is wrong at
132 /// 80,000 — while the throughput cost of turning eleven chunks into fifty
133 /// is certain and immediate (D-058). The fix is not a row count: it is for
134 /// the chunk loop to stop on elapsed time, **delivered in 0.12.0**. This
135 /// number is now the ceiling that loop starts from and never exceeds; on a
136 /// populated table it converges below it within a chunk or two.
137 ///
138 /// D-134 retired the growth claim on the neighbouring *single-assertion*
139 /// path and did not measure this one; D-136 is why this line now carries a
140 /// measurement rather than a figure quoted from 0.5.6.
141 pub const EDGES: usize = 90;
142
143 /// Concept upserts (`write_concepts`).
144 ///
145 /// Linear at ~23 µs per row, so unlike [`EDGES`] this size *is* a genuine
146 /// throughput sacrifice: 1,000-row chunks ran at 23.6 µs per row against
147 /// ~35 µs here. Paid deliberately — a 1,000-row chunk takes 24 ms, eight
148 /// times the bound.
149 pub const CONCEPTS: usize = 70;
150
151 /// Analytics annotations (`write_analytics_annotations`).
152 ///
153 /// The one path where the old constant was nearly right, and the only bulk
154 /// table with no triggers at all: ~2.5 µs per row, linear, so the bound buys
155 /// a large chunk. 1,000 rows would be 3.5 ms — over, but only just.
156 pub const ANNOTATIONS: usize = 600;
157
158 /// Embedding vectors (`upsert_embeddings`).
159 ///
160 /// The smallest by a wide margin, because DiskANN index maintenance makes an
161 /// embedding the most expensive row in the system. That cost grows with the
162 /// **corpus**, not the chunk (D-059): a fixed 30-vector chunk costs 49 µs per
163 /// vector into an empty corpus and 224 µs into an 8,000-vector one. Graph
164 /// insertion getting dearer as the graph grows is what DiskANN is, so unlike
165 /// [`EDGES`] there is nothing here to fix — but it does mean this size buys
166 /// latency at some throughput, not for free.
167 pub const EMBEDDINGS: usize = 30;
168}
169
170/// What one chunk transaction cost, reported by the actor to the caller-side
171/// chunk loop (0.12.0, W1).
172///
173/// `held` is measured **inside** the actor, around its own transaction, and
174/// therefore excludes the time the command spent queued. That exclusion is the
175/// point: queue time is what strict preemption *does*, and a controller fed
176/// `send + await` would shrink chunks as punishment for the actor correctly
177/// serving an interactive write first.
178///
179/// Crate-internal, along with the command enums that carry it. It was `pub`
180/// through 0.13.32 only because they were (D-206).
181#[derive(Debug, Clone, Copy, PartialEq, Eq)]
182pub(crate) struct ChunkOutcome {
183 /// Rows the transaction actually wrote.
184 pub rows: usize,
185 /// How long the actor held the write lock for them.
186 pub held: std::time::Duration,
187}
188
189/// A flag a caller can raise to stop a chunked bulk write (0.13.8, W7.6, D-181).
190///
191/// Cheap to clone and safe to set from any thread, which is the whole point: the
192/// task running the import is the one thing that cannot cancel it. Hand a clone
193/// to whatever *can* — a signal handler, a UI thread, a timeout task — and it
194/// takes effect at the next chunk boundary.
195///
196/// **A boundary, not an abort.** Nothing rolls back and no in-flight
197/// transaction is interrupted: the loop notices between chunks and stops
198/// sending. The chunks that committed stay committed, and
199/// [`BulkInterrupted::written`](crate::BulkInterrupted::written) says how many
200/// rows those were. That is the same per-chunk boundary
201/// [`Database::bulk_import`] already documents, so cancellation adds a reason to
202/// stop and no new failure mode.
203///
204/// Setting it after the last chunk has committed does nothing — a finished
205/// write reports success, because it succeeded.
206#[derive(Clone, Debug, Default)]
207pub struct CancelToken(Arc<std::sync::atomic::AtomicBool>);
208
209impl CancelToken {
210 /// A token that has not been cancelled.
211 pub fn new() -> Self {
212 Self::default()
213 }
214
215 /// Ask the bulk write holding a clone of this token to stop at its next
216 /// chunk boundary. Idempotent; a token never un-cancels.
217 pub fn cancel(&self) {
218 // `Relaxed` on both sides is sufficient and deliberate: nothing is
219 // published *through* this flag. The rows are ordered by the database
220 // and the chunk results by the response channel, so the only thing the
221 // reader needs is to observe the store eventually, which every ordering
222 // guarantees.
223 self.0.store(true, std::sync::atomic::Ordering::Relaxed);
224 }
225
226 /// Whether [`Self::cancel`] has been called on this token or any clone.
227 pub fn is_cancelled(&self) -> bool {
228 self.0.load(std::sync::atomic::Ordering::Relaxed)
229 }
230}
231
232/// One chunk's worth of progress, handed to the callback on
233/// [`BulkControl::on_progress`] (0.13.8, W7.6).
234///
235/// Reported *after* the chunk has committed, so `written` is a count of rows
236/// that are in the database and will stay there even if the next chunk fails.
237#[derive(Debug, Clone, Copy, PartialEq, Eq)]
238#[non_exhaustive]
239pub struct BulkProgress {
240 /// Rows committed so far, across every chunk including this one.
241 pub written: usize,
242 /// Rows in the batch the caller passed. `written` reaching this means the
243 /// last chunk has committed.
244 pub total: usize,
245 /// Rows this chunk wrote. Not a constant: the loop resizes chunks against
246 /// [`CHUNK_BUDGET`] as it measures them (D-058).
247 pub rows: usize,
248 /// How long the actor held the write lock for this chunk — the same figure
249 /// the controller steers on. Measured inside the actor, around its own
250 /// transaction, so it excludes the time the command spent queued.
251 pub held: std::time::Duration,
252}
253
254/// Cancellation and progress for the four chunked bulk paths (0.13.8, W7.6,
255/// D-181).
256///
257/// Default is "neither", which is what [`Database::bulk_import`] and its three
258/// siblings pass. The `_with` variants take one of these:
259///
260/// ```no_run
261/// # use macrame::{BulkControl, CancelToken, Database};
262/// # async fn f(db: &Database, edges: Vec<macrame::prelude::EdgeAssertion>) {
263/// let token = CancelToken::new();
264/// let stopper = token.clone();
265/// tokio::spawn(async move {
266/// tokio::time::sleep(std::time::Duration::from_secs(30)).await;
267/// stopper.cancel();
268/// });
269///
270/// let control = BulkControl::new()
271/// .cancel_with(token)
272/// .on_progress(|p| println!("{}/{} rows", p.written, p.total));
273///
274/// match db.bulk_import_with(edges, control).await {
275/// Ok(n) => println!("imported {n}"),
276/// Err(e) => println!("stopped after {}: {}", e.written, e.cause),
277/// }
278/// # }
279/// ```
280///
281/// **The callback runs on the importing task, between chunks.** It is therefore
282/// on the critical path: whatever it does is time the next chunk is not being
283/// sent in. Printing or updating a counter is what it is for; a blocking write
284/// is not, and neither is anything that calls back into the same `Database`,
285/// which would deadlock the loop against a channel it is itself draining.
286#[derive(Default, Clone)]
287pub struct BulkControl {
288 cancel: Option<CancelToken>,
289 on_progress: Option<Arc<dyn Fn(BulkProgress) + Send + Sync>>,
290}
291
292impl std::fmt::Debug for BulkControl {
293 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
294 f.debug_struct("BulkControl")
295 .field("cancel", &self.cancel)
296 .field("on_progress", &self.on_progress.is_some())
297 .finish()
298 }
299}
300
301impl BulkControl {
302 /// Neither cancellation nor progress — what the plain bulk methods pass.
303 pub fn new() -> Self {
304 Self::default()
305 }
306
307 /// Stop at the next chunk boundary when `token` is cancelled.
308 pub fn cancel_with(mut self, token: CancelToken) -> Self {
309 self.cancel = Some(token);
310 self
311 }
312
313 /// Call `f` after every chunk commits. See the note on [`BulkControl`]
314 /// about what this closure is allowed to do.
315 pub fn on_progress(mut self, f: impl Fn(BulkProgress) + Send + Sync + 'static) -> Self {
316 self.on_progress = Some(Arc::new(f));
317 self
318 }
319
320 fn is_cancelled(&self) -> bool {
321 self.cancel.as_ref().is_some_and(CancelToken::is_cancelled)
322 }
323
324 fn report(&self, progress: BulkProgress) {
325 if let Some(f) = &self.on_progress {
326 f(progress);
327 }
328 }
329}
330
331/// Smallest chunk the adaptive loop will fall to (0.12.0, W2).
332///
333/// # A floor is a deliberate, measured violation of [`CHUNK_BUDGET`]
334///
335/// Feedback alone converges to whatever size meets the budget, and on a
336/// populated `links` table that size keeps falling — per-row cost there grows
337/// with the table (D-059, D-142), so there is no size at which the *fixed* cost
338/// of a transaction stops dominating. Left unbounded the loop reaches chunks of
339/// one or two rows, where nearly all the work is `BEGIN`/`COMMIT` and the import
340/// no longer finishes.
341///
342/// 35 is measured, and **re-measured against the loop that uses it** — the
343/// difference matters, because the figure this constant shipped with was an
344/// extrapolation. `examples/chunk_matrix.rs -- converge` runs a 900-edge
345/// `bulk_import` into each of the four D-088 shapes at 8,000 edges and reports
346/// the actor's own per-transaction readings. A 35-row chunk costs **3.11–3.43 ms**
347/// across the four shapes, two sessions, excluding the run-up. The floor misses
348/// the 3 ms bound by 0.1–0.4 ms, not by the ~1.1 ms predicted from the sweep.
349///
350/// The miss is **steady state** — not a one-chunk transient on the way down —
351/// and the defense is the argument [`CHUNK_BUDGET`] is answerable to rather than
352/// the number itself: an interactive assertion arriving at the worst moment
353/// waits ~3.2 ms for the chunk in flight and then runs its own ≤ 5 ms write, so
354/// ~8.2 ms against a 16.7 ms frame.
355///
356/// What the same measurement says about the *size*: on this path at this
357/// population the loop goes `[90, 35, 35, …]` on all four shapes and never picks
358/// anything between. The proportional shrink from a 90-row chunk proposes ~31
359/// rows, which clamps here — so on the edge path the floor is not a safety net
360/// under the controller, it **is** the operating point, and this number is
361/// carrying more weight than a backstop normally would. Re-measure it, not the
362/// controller, when the edge path's per-row cost changes.
363const CHUNK_FLOOR: usize = 35;
364
365/// Size of the next chunk, from what the last one cost (0.12.0, W2).
366///
367/// Pure on purpose — no clock, no database, no actor — so the control law can be
368/// tested for the properties that matter without a fixture. Three regimes:
369///
370/// | last hold | response | why |
371/// |---|---|---|
372/// | over `budget` | shrink to `current · budget / held`, × 0.9 | back off *fast* from a bound already being exceeded; the 0.9 undershoots so the correction does not have to be repeated |
373/// | under `budget / 2` | grow by a quarter of `current`, at least one row | approach the bound *slowly*; the dead band above it stops a size that is merely comfortable from oscillating |
374/// | otherwise | hold | in band, and moving costs more than it buys |
375///
376/// The asymmetry is the whole design. Proportional shrinking converges from
377/// above in one or two steps, which matters because every step over budget is a
378/// latency miss a caller can feel; additive growth cannot overshoot by more than
379/// 25%, which matters because the ceiling is a throughput preference and not a
380/// bound.
381///
382/// `ceiling` is the path's [`chunk_rows`] constant, which is why those constants
383/// keep their values and their derivations: they are no longer the size, they
384/// are the largest size this will ever ask for. `floor` is [`CHUNK_FLOOR`] —
385/// see there for the budget it knowingly misses.
386///
387/// Never returns 0, at any input, including `held == 0` or `current == 0`.
388fn next_chunk_size(
389 current: usize,
390 held: std::time::Duration,
391 budget: std::time::Duration,
392 floor: usize,
393 ceiling: usize,
394) -> usize {
395 let held = held.as_nanos().max(1);
396 let budget_ns = budget.as_nanos().max(1);
397 let current = current.max(1);
398
399 let next = if held > budget_ns {
400 // Integer math, and the `max(1)` matters: a chunk 200× over budget
401 // would otherwise propose 0 and the loop would stop making progress.
402 let scaled = (current as u128) * budget_ns * 9 / (held * 10);
403 (scaled as usize).max(1)
404 } else if held * 2 < budget_ns {
405 // Saturating because `current` is a `usize` and this is the one branch
406 // that adds to it. Nothing sane reaches the boundary; the clamp below
407 // makes the answer correct anyway rather than a debug panic.
408 current.saturating_add((current / 4).max(1))
409 } else {
410 current
411 };
412
413 // Applied last and unconditionally, so a caller that passes a reversed pair
414 // gets the floor rather than a panic — and `max(1)` last of all, because a
415 // chunk of zero rows is the single answer no loop can make progress from.
416 next.clamp(floor.min(ceiling), ceiling).max(1)
417}
418
419/// The latency bound [`chunk_rows`] is derived from (§5.1.5, D-058).
420///
421/// This is the golden rule's actual content. §9 has carried it as a row count
422/// with a duration attached — "chunk commit, 500 rows ≤ 3 ms" — which reads as
423/// two requirements and is one: the duration is the requirement, and the row
424/// count is whatever satisfies it on a given path and machine.
425///
426/// 3 ms is §9's number, kept rather than renegotiated. What it buys, end to end:
427/// an interactive assertion arriving at the worst possible moment waits for the
428/// chunk in flight (≤ 3 ms — the SQLite write lock is not preemptible, so
429/// priority buys the *next* turn and not this one) and then runs its own write
430/// (≤ 5 ms, §9), so ≤ 8 ms
431/// worst case. That fits inside a 60 Hz frame with room, which is the standard
432/// this bound is ultimately answerable to.
433///
434/// # Some operations are exempt, and the exemption is a contract, not an oversight
435///
436/// This was recorded in three separate rustdoc notes and nowhere near the bound
437/// itself, which is where a reader looks for its scope (§8.6). Stated here, with
438/// Wave 3's measurements:
439///
440/// | Path | Bound | Why it cannot be chunked |
441/// |---|---|---|
442/// | [`Database::write_bulk_atomic`] | none — caller-sized `Vec` | D-014: the batch is *one act* under one stamp. Splitting it is the thing the method exists not to do |
443/// | [`Database::archive`] | measured **26.8 ms** for 2,000 archivable edges; see [`Database::archive_windowed`] | D-012: copy-then-delete must be atomic, or a crash between the phases duplicates or loses rows |
444/// | `rebuild_current` | measured **24.6 / 104 / 318 ms** at 4K / 16K / 40K rows in `links` (was "~50 s per 10M edges", which nothing had measured) | D-023: the window between `DELETE` and `INSERT` is the whole of current belief; a reader landing in it sees a graph with no edges and no error |
445/// | [`Database::rehydrate`] | unmeasured; a function of how many rows the caller named | D-012 backwards: the same copy-then-delete atomicity, in the other direction. **A row here since 0.12.9 only because it was previously invisible** — rehydration reported as `archive` and inherited its exemption without anyone deciding on it (W4.3, D-152) |
446/// | [`Database::archive_branch`] | unmeasured; a function of how much one lineage wrote | D-012 again, and D-230's chain: the links, the log entries and the `branches` row leave together or the ledger disagrees with itself about what is currently believed. There is no smaller unit — half a forgotten lineage is a lineage whose reads are answered by its parent |
447/// | the swap turn of [`Database::rebuild_current_chunked`], counted as `shadow_swap` | measured **46.8 ms** at the largest fixture (D-082), and it grows with the table | Index names are global and SQLite has no `ALTER INDEX … RENAME`, so the shadow cannot carry `idx_lc_traversal_cover` while the live table still holds it — all three indexes are built here, under the lock. This is the residual T1.2 could not remove, and there is no smaller unit: half a swapped projection is not a projection. **Exempt since 0.14.16** (W12.16, D-233). The *fill* half keeps its own kind and is deliberately absent from this table, which is what makes a violation there a regression rather than a constant |
448/// | [`Database::checkpoint`] | a function of the WAL's size, which is a function of how long since the last checkpoint — not of anything the caller passes | It is not a transaction at all. `PRAGMA wal_checkpoint` copies frames back into the main file and there is no unit smaller than the frame it is already working in; the caller asked for exactly this, and the alternative to a long checkpoint is a WAL that keeps growing (0.12.13, W5.2, D-156) |
449///
450/// The `archive` figure is end-to-end through this method, so it **includes**
451/// the re-derivation `archive()` runs inside its transaction — but it does not
452/// attribute it, and until D-077 more than half of that re-derivation was an
453/// audit comparing `links_current` against the query that had just filled it.
454/// Note also which variable that cost scales with: `rebuild_within` reprojects
455/// **all of `links`**, so the archive's repair term grows with the *surviving*
456/// table and not with the batch being archived. A budget stated per "100K closed
457/// intervals" ([§9](../docs/architecture/s6-s10-flows-to-dependencies.md)) is
458/// therefore parameterised on the wrong quantity.
459///
460/// The first four are atomic **by contract**, which is why "cap the batch" and
461/// "add a third tier" were both considered and neither was taken: capping breaks the
462/// guarantee the operation exists to provide, and a third tier changes which
463/// caller waits without changing how long the lock is held. What was wrong was
464/// never the exemption — it was that the bound was stated as though it had none.
465///
466/// A caller who needs the latency bound and not the atomicity has
467/// [`Database::bulk_import`], which is the same write chunked at
468/// [`chunk_rows::EDGES`] and explicitly *not* atomic overall (D-011).
469///
470/// # One of them is no longer unbounded (T1.1, D-080)
471///
472/// `archive` was the worst of them, because its hold is a function of *how long
473/// since the last archive* rather than of anything the caller chose.
474/// [`Database::archive_windowed`] runs the same work as N sessions, each
475/// atomic, each its own actor turn. Measured on an 8,000-key fixture with four
476/// generations of superseded history: the longest single hold falls from
477/// **3.3 s to 0.77 s** at one-hour windows, for total wall time that is flat
478/// within this cycle's noise.
479///
480/// The same measurement at 2,000 keys goes the other way — the hold falls
481/// 260 ms → 117 ms while total time rises 260 ms → 671 ms — so windowing is a
482/// trade and not a free improvement. It pays when the backlog is large, which
483/// is when the unwindowed hold is a problem in the first place. `archive` is
484/// kept, not deprecated, for exactly that reason.
485pub const CHUNK_BUDGET: std::time::Duration = std::time::Duration::from_millis(3);
486
487/// Predicted hold above which [`Database::write_bulk_atomic`] warns (T1.3).
488///
489/// 250 ms is fifteen frames at 60 Hz: not a hitch, a visible freeze. It is well
490/// above [`CHUNK_BUDGET`] on purpose — this path is exempt from that bound by
491/// contract, so warning at 3 ms would fire on batches that are working exactly
492/// as designed and train the reader to filter the message out.
493pub const BULK_ATOMIC_WARN_HOLD: std::time::Duration = std::time::Duration::from_millis(250);
494
495/// Roughly how long [`Database::write_bulk_atomic`] will hold the actor for
496/// this batch (T1.3, D-081; re-fitted 0.13.6, W7.5, D-179).
497///
498/// # Two terms, and the batch's shape is no longer one of them
499///
500/// T1.3 asks for "rows × measured per-row cost". Through 0.13.5 that was wrong
501/// in a way worth a paragraph: `write_edges_atomic` opened with a
502/// `reject_overlaps_within` that compared **every pair**, and the quadratic
503/// term's constant depended on the batch's *shape* rather than its size, so two
504/// 20,000-edge batches held the actor for **2.6 s** and **18.1 s** — a size-only
505/// model was off by 7× between them, in the under-predicting direction.
506///
507/// W7.5 sorts and sweeps instead, and the 18.1 s batch now holds for **2.2 s**.
508/// The shape term is gone from the code and therefore from here: measured on
509/// the same machine, the two shapes are within 15% of each other at every size
510/// from 100 to 20,000 rows, which is inside the noise this model claims.
511///
512/// What is left is not flat either, and the second term is why. Per-row cost
513/// rises from ~36 µs at 100 rows to ~111 µs at 20,000, because each insert
514/// maintains indexes and two triggers against a table the batch is itself
515/// growing:
516///
517/// ```text
518/// hold ≈ rows · (7.4 µs + 7.24 µs · ⌊log₂ rows⌋)
519/// ```
520///
521/// # What this is calibrated against, and where it will be wrong
522///
523/// libSQL 0.9.30, one machine, best of three, 100–20,000 rows in both shapes;
524/// within 15% from 500 rows up. Below that it under-predicts by up to 3×, which
525/// is harmless in the same way the old 3× over-prediction was — nothing that
526/// small approaches [`BULK_ATOMIC_WARN_HOLD`].
527///
528/// **The log term reads the batch because the batch is all it has.** It stands
529/// for the depth of a structure the batch is loading, and this signature never
530/// sees the table. That is exact for the bulk import this warns about, and
531/// optimistic for a small batch appended to an already-large table — the same
532/// blind spot the flat per-row model had, now visible instead of averaged away.
533///
534/// It is machine-specific and says nothing about disk. It exists to turn
535/// "uncapped" into an order of magnitude a caller can act on, and should not be
536/// read more precisely than that. `examples/bulk_atomic_diag.rs` prints
537/// predicted against measured, so the model's drift is visible rather than
538/// assumed.
539pub fn estimated_bulk_hold(edges: &[EdgeAssertion]) -> std::time::Duration {
540 let rows = edges.len() as u64;
541 if rows == 0 {
542 return std::time::Duration::ZERO;
543 }
544
545 // Nanoseconds throughout, saturating: a caller who passes a batch large
546 // enough to overflow this has a problem the arithmetic cannot express, and
547 // saturating to ~584 years still crosses every threshold above.
548 let per_row = 7_400u64.saturating_add((rows.ilog2() as u64).saturating_mul(7_240));
549 std::time::Duration::from_nanos(rows.saturating_mul(per_row))
550}
551
552/// Most sessions [`Database::archive_windowed`] will run for one call (T1.1).
553///
554/// A limit exists because the session count is a function of *transaction-time
555/// span divided by window*, and both come from the caller — a one-second window
556/// over a decade of history is ten million actor turns, each opening a
557/// transaction and writing a horizon row. That is not a slow archive, it is a
558/// caller who meant something else.
559///
560/// 4,096 is chosen against the operation it bounds rather than against a clock:
561/// at the measured 26.8 ms for a session with work in it, a full run of this
562/// many is about two minutes of background writing, and the whole point of
563/// windowing is that those two minutes are interruptible. It is a refusal
564/// rather than a clamp — see [`DbError::ArchiveWindow`] for why.
565pub const MAX_ARCHIVE_SESSIONS: usize = 4_096;
566
567/// A concept assertion: the payload of an upsert.
568///
569/// `#[non_exhaustive]` since 0.14.8 for
570/// [`EdgeAssertion`]'s reason: `branch` is the
571/// first field added since it was written, and one break is better than a
572/// recurring one.
573#[derive(Debug, Clone, PartialEq)]
574#[non_exhaustive]
575pub struct ConceptUpsert {
576 pub id: String,
577 pub title: String,
578 pub content: String,
579 pub embedding_model: Option<String>,
580 pub valid_from: String,
581 pub valid_to: String,
582 pub retired: bool,
583 /// The lineage this concept is minted on, or `None` for the trunk (§15.2,
584 /// D-225).
585 ///
586 /// **The rule here is narrower than the edge's, and it is the schema's
587 /// rather than this crate's.** `concepts` is a current-state projection
588 /// keyed by identity — `id` is `NOT NULL UNIQUE` — so two lineages holding
589 /// different beliefs about one concept is two rows with one `id`, which the
590 /// unique index refuses on its own. `trg_concepts_cross_lineage` turns that
591 /// refusal into [`DbError::CrossLineage`] so it says which rule was broken.
592 ///
593 /// So a branch **inherits** its parent's concepts and cannot restate them;
594 /// what this field is for is a concept the branch *mints*, which is the
595 /// case the trunk has no row for. A branch that needs to disagree with its
596 /// parent about a concept's content is asking for the overlay design, which
597 /// is deferred with its reopen trigger named (D-214).
598 pub branch: Option<crate::branch::BranchId>,
599}
600
601impl ConceptUpsert {
602 pub fn new(id: impl Into<String>, title: impl Into<String>) -> Self {
603 Self {
604 id: id.into(),
605 title: title.into(),
606 content: String::new(),
607 embedding_model: None,
608 valid_from: String::new(),
609 valid_to: timestamp::OPEN_SENTINEL.to_string(),
610 retired: false,
611 branch: None,
612 }
613 }
614
615 pub fn content(mut self, content: impl Into<String>) -> Self {
616 self.content = content.into();
617 self
618 }
619
620 pub fn embedding_model(mut self, model: impl Into<String>) -> Self {
621 self.embedding_model = Some(model.into());
622 self
623 }
624
625 pub fn valid_from(mut self, ts: impl Into<String>) -> Self {
626 self.valid_from = ts.into();
627 self
628 }
629
630 pub fn valid_to(mut self, ts: impl Into<String>) -> Self {
631 self.valid_to = ts.into();
632 self
633 }
634
635 /// Mint this concept on `branch` rather than on the trunk (0.14.8).
636 ///
637 /// See [`branch`](Self::branch) for why a branch may mint a concept and may
638 /// not restate one it inherited.
639 pub fn on_branch(mut self, branch: crate::branch::BranchId) -> Self {
640 self.branch = Some(branch);
641 self
642 }
643
644 /// The lineage this upsert names, spelled out. See
645 /// [`EdgeAssertion::branch_name`](crate::graph::EdgeAssertion).
646 pub(crate) fn branch_name(&self) -> &str {
647 self.branch
648 .as_ref()
649 .map_or(crate::schema::ddl::MAIN_BRANCH, |b| b.as_str())
650 }
651
652 pub fn retired(mut self, retired: bool) -> Self {
653 self.retired = retired;
654 self
655 }
656
657 /// Put the timestamps in canonical form (D-029) before they cross the channel.
658 pub fn normalized(mut self) -> Result<Self> {
659 crate::util::ids::validate_id(&self.id)?;
660 self.valid_from = timestamp::normalize(&self.valid_from)?;
661 self.valid_to = timestamp::normalize(&self.valid_to)?;
662 Ok(self)
663 }
664}
665
666/// One derived analytics result for one concept (§5.4, D-041).
667///
668/// Not a `ConceptUpsert`. The distinction is the whole of D-041: a concept
669/// upsert is a statement about the world and belongs in the ledger, while an
670/// annotation is a function of an algorithm applied to a graph and belongs in
671/// `analytics_annotations`, which carries no log trigger. Writing one as the
672/// other overwrote the concept's `content` with the label and recorded every
673/// analytics rerun as a fresh version of the world.
674#[derive(Debug, Clone, PartialEq, Eq)]
675#[non_exhaustive]
676pub struct Annotation {
677 pub concept_id: String,
678 /// Namespaced by convention, e.g. `louvain.community`, `kcore.shell`.
679 pub label: String,
680 /// JSON-encoded payload. Opaque to this crate.
681 pub value: String,
682}
683
684impl Annotation {
685 pub fn new(
686 concept_id: impl Into<String>,
687 label: impl Into<String>,
688 value: impl Into<String>,
689 ) -> Self {
690 Self {
691 concept_id: concept_id.into(),
692 label: label.into(),
693 value: value.into(),
694 }
695 }
696}
697
698/// Commands sent to the Write Actor on the high-priority channel (UI-driven work).
699pub(crate) enum HighPriCommand {
700 AssertEdge {
701 edge: EdgeAssertion,
702 responder: oneshot::Sender<Result<()>>,
703 },
704 RetireEdge {
705 source: String,
706 target: String,
707 edge_type: String,
708 valid_from: String,
709 valid_to: String,
710 /// The lineage doing the retiring, or `None` for the trunk (0.14.8).
711 branch: Option<crate::branch::BranchId>,
712 responder: oneshot::Sender<Result<()>>,
713 },
714 UpsertConcept {
715 concept: ConceptUpsert,
716 responder: oneshot::Sender<Result<()>>,
717 },
718 WriteBulkAtomic {
719 edges: Vec<EdgeAssertion>,
720 responder: oneshot::Sender<Result<usize>>,
721 },
722 RebuildCurrent {
723 responder: oneshot::Sender<Result<RebuildReport>>,
724 },
725 /// Create a model's embedding table and its DiskANN index (D-037, D-048).
726 ///
727 /// High priority despite being setup work: it is one small transaction, and
728 /// every embedding write for the model blocks on it, so queueing it behind a
729 /// bulk job would stall the thing it gates.
730 RegisterModel {
731 model: ModelName,
732 dim: usize,
733 responder: oneshot::Sender<Result<()>>,
734 },
735 /// Move WAL frames back into the main database file (§4.5, F-30, D-156).
736 ///
737 /// High priority, and for once the reason is not latency: a caller asking
738 /// for a checkpoint is asking for it *now*, usually at the end of a bulk
739 /// load or before taking a copy of the file, and queueing it behind the
740 /// background work it was meant to follow inverts the intent. It is also
741 /// the only command here that is not a transaction.
742 Checkpoint {
743 responder: oneshot::Sender<Result<CheckpointReport>>,
744 },
745 /// Register a lineage (0.14.7, §15.4).
746 ///
747 /// High priority, and not because it is urgent: it is one insert into a
748 /// table with no secondary indices, so it is the cheapest turn the actor
749 /// takes. What makes it high priority is that everything the caller does
750 /// next is a write *on* this branch, and queueing a fork behind a bulk
751 /// import would stall the work it exists to enable — `RegisterModel`'s
752 /// argument, for the same reason.
753 ///
754 /// It goes through the actor rather than the read connection for the
755 /// ordinary reason every write does, plus one specific to it: the duplicate
756 /// and parent checks are only sound if nothing can register a colliding
757 /// name between the check and the insert, and the actor is what makes the
758 /// pair one turn.
759 Fork {
760 name: crate::branch::BranchId,
761 parent: crate::branch::BranchId,
762 responder: oneshot::Sender<Result<crate::branch::Branch>>,
763 },
764 Shutdown {
765 responder: oneshot::Sender<Result<()>>,
766 },
767}
768
769/// What `PRAGMA wal_checkpoint` returned (0.12.13, W5.2, D-156).
770///
771/// The three columns SQLite gives back, named, rather than `()` — a checkpoint
772/// that did nothing and a checkpoint that reclaimed a 400 MB WAL are the same
773/// `Ok(())`, and the difference is the entire reason a caller asked.
774#[derive(Debug, Clone, Copy, PartialEq, Eq)]
775#[non_exhaustive]
776pub struct CheckpointReport {
777 /// `true` when SQLite could not complete the requested mode because a
778 /// reader or writer was in the way.
779 ///
780 /// **This is not an error, and it is not ignorable.** `TRUNCATE` waits for
781 /// readers only as long as `busy_timeout` allows; past that it gives up and
782 /// says so, having possibly still copied frames. A caller checkpointing
783 /// before copying the file away must read this, because a busy checkpoint
784 /// means the main file is not self-contained yet.
785 pub busy: bool,
786 /// Frames left in the WAL at the end. `0` when the checkpoint completed,
787 /// since the mode run is `TRUNCATE`.
788 pub log_frames: u64,
789 /// Frames moved back into the database file.
790 ///
791 /// Read from a `FULL` pass rather than from the `TRUNCATE` — see
792 /// `run_checkpoint` for why a truncating checkpoint cannot report this
793 /// number itself.
794 pub checkpointed_frames: u64,
795}
796
797impl CheckpointReport {
798 /// The WAL was fully reclaimed: nothing blocked, and nothing is left.
799 pub fn is_complete(&self) -> bool {
800 !self.busy && self.log_frames == 0
801 }
802}
803
804/// Commands sent to the Write Actor on the low-priority channel (background work).
805pub(crate) enum LowPriCommand {
806 /// One chunk of **concepts** — a ledger write, logged and versioned.
807 WriteConceptsChunk {
808 chunk: Vec<ConceptUpsert>,
809 responder: oneshot::Sender<Result<ChunkOutcome>>,
810 },
811 /// One chunk of **derived annotations** — off-ledger, no log trigger (D-041).
812 ///
813 /// The pair is named apart deliberately: this variant was `WriteAnalyticsChunk`
814 /// beside a `WriteAnnotationsChunk` that carried concepts, which is the
815 /// crossing D-075 undid.
816 WriteAnalyticsChunk {
817 chunk: Vec<Annotation>,
818 responder: oneshot::Sender<Result<ChunkOutcome>>,
819 },
820 /// One chunk of vectors for one model (§5.9, D-048).
821 ///
822 /// Low priority: embedding is bulk derived work and must never preempt an
823 /// interactive assertion.
824 UpsertEmbeddingChunk {
825 model: ModelName,
826 chunk: Vec<(String, Vec<f32>)>,
827 responder: oneshot::Sender<Result<ChunkOutcome>>,
828 },
829 BulkImportChunk {
830 chunk: Vec<EdgeAssertion>,
831 responder: oneshot::Sender<Result<ChunkOutcome>>,
832 },
833 Archive {
834 cutoff: String,
835 archive_path: PathBuf,
836 responder: oneshot::Sender<Result<ArchiveReport>>,
837 },
838 /// Forget one lineage, moving its whole ledger to the cold file (0.14.13,
839 /// §15.4, D-230).
840 ///
841 /// Low priority for `Archive`'s reason and one of its own: it is bulk
842 /// physical movement holding the write lock for its whole transaction, and
843 /// it is the least urgent write in the crate — the rows it moves belong to
844 /// a lineage nobody is reading.
845 ArchiveBranch {
846 branch: String,
847 archive_path: PathBuf,
848 responder: oneshot::Sender<Result<ArchiveReport>>,
849 },
850 /// Move named concepts back out of the cold file (0.9.0, C3).
851 ///
852 /// Low priority for the same reason `Archive` is: it is bulk physical
853 /// movement with no latency bound, and it holds the write lock for its whole
854 /// transaction.
855 Rehydrate {
856 ids: Vec<String>,
857 archive_path: PathBuf,
858 responder: oneshot::Sender<Result<RehydrateReport>>,
859 },
860 /// Reconstruct the FTS index from `concepts` (§5.9, D-036, D-051).
861 ///
862 /// Low priority: it is maintenance on a derivative table, and a search index
863 /// that is a few seconds stale is a smaller cost than an interactive write
864 /// that waits behind a full reindex.
865 RebuildFts {
866 responder: oneshot::Sender<Result<()>>,
867 },
868 /// Refresh or top up the query planner's statistics (0.12.4, D-149).
869 ///
870 /// Low priority, and not a close call: statistics being a few seconds stale
871 /// costs a plan that was already the plan a moment ago, where preempting an
872 /// interactive assertion costs a caller their latency bound. It is a write —
873 /// it writes `sqlite_stat1` — so it takes the write lock like anything else,
874 /// and `PRAGMA analysis_limit` in `configure` is what keeps the hold a
875 /// function of the index count instead of the table size.
876 Analyze {
877 /// `true` runs `PRAGMA optimize`, which re-analyses only what SQLite
878 /// believes has gone stale; `false` runs `ANALYZE` unconditionally.
879 incremental: bool,
880 responder: oneshot::Sender<Result<()>>,
881 },
882 /// One step of a chunked shadow rebuild (§5.8, T1.2, D-082).
883 ///
884 /// Low priority, and one command per step rather than one per rebuild: the
885 /// whole value of building beside the live table is that the actor returns
886 /// here between chunks. See [`Database::rebuild_current_chunked`].
887 ShadowRebuild {
888 step: crate::integrity::ShadowStep,
889 responder: oneshot::Sender<Result<crate::integrity::ShadowOutcome>>,
890 },
891}
892
893/// What the write actor knows between turns (0.15.6, W14.3, [D-248]).
894///
895/// [`run_writer_actor`] owned a connection and nothing else. Every command was
896/// handed `&conn`, and every fact a command established died with it — so a
897/// single-edge assertion asked `branches` how many lineages exist, compiled the
898/// overlap guard, and compiled `INSERT_LINK`, on every call, having done all
899/// three on the previous one. Measured on the trunk that is 76 µs of a 160 µs
900/// write; once the database has forked it is 155 µs of a 343 µs write, because
901/// the statement being compiled each time is the guard's resolved form.
902///
903/// **Everything here is cached for one reason and invalidated by name for the
904/// same one: the actor is the only writer** (D-014). `branches` is written by
905/// `Fork` and by `ArchiveBranch` and by nothing else in the crate; the
906/// statements are bound to a connection this task owns for the process
907/// lifetime. A cache whose only writer is holding it cannot go stale behind its
908/// own back, which is why this is a plain `&mut` and not an epoch or a lock.
909///
910/// Built lazily rather than at open. Eager construction pays on a database that
911/// never asserts an edge, and it puts fallible work in the spawn path, where
912/// there is no caller to hand the error to.
913///
914/// # What is deliberately not here
915///
916/// The **hot-log intactness verdict** (review C-5). It is read on `read_conn`
917/// by every recorded-time read, not by the actor, so caching it here would put
918/// it on the wrong side of the process. It needs a shared cell and an
919/// invalidation argument about a *reader* seeing a stale answer, which is a
920/// different argument from this one and gets its own release.
921///
922/// The **batch paths' statements**. `write_edges_atomic` prepares inside its
923/// own transaction and drops before it commits, because a live statement is
924/// what makes SQLite refuse to end one. It already prepares once per chunk
925/// rather than once per row (D-056, §8.8), which is where that path's cost was.
926///
927/// [D-248]: ../../docs/architecture/s13-decision-register.md#d-248
928struct ActorState {
929 /// `branches`, as this actor last left it.
930 lineages: Option<Lineages>,
931 /// `INSERT_LINK`, compiled against the actor's connection.
932 insert_link: Option<libsql::Statement>,
933 /// The overlap guard, with the shape it was compiled for.
934 guard: Option<OverlapGuard>,
935}
936
937// `Lineages` moved to `graph::lineage` in 0.15.17 ([D-259]). It held
938// `(branch_id, is_root)` here, because the shape was all the actor needed;
939// resolving ancestry in Rust needs `parent_id` and `forked_at` as well, on
940// both sides of the crate, and two structs answering one question from one
941// table is what D-030 is about.
942//
943// [D-259]: ../docs/architecture/s13-decision-register.md#d-259
944
945impl ActorState {
946 fn new() -> Self {
947 Self {
948 lineages: None,
949 insert_link: None,
950 guard: None,
951 }
952 }
953
954 /// Forget `branches`. Called by the two commands that write it.
955 fn forget_lineages(&mut self) {
956 self.lineages = None;
957 }
958
959 /// Drop the compiled statements.
960 ///
961 /// Called by the commands that `ATTACH`, `DETACH`, or otherwise move the
962 /// schema under the connection. SQLite recompiles a statement across a
963 /// schema change on its own and this does not rely on that: a statement
964 /// dropped here costs one prepare on the next write, against a class of bug
965 /// whose symptom would be a stale plan on the archive path in production.
966 fn forget_statements(&mut self) {
967 self.insert_link = None;
968 self.guard = None;
969 }
970
971 /// Both of the above, for a command that does both.
972 fn forget_everything(&mut self) {
973 self.forget_lineages();
974 self.forget_statements();
975 }
976
977 async fn lineages(&mut self, conn: &libsql::Connection) -> Result<&Lineages> {
978 if self.lineages.is_none() {
979 self.lineages = Some(Lineages::load(conn).await?);
980 }
981 Ok(self
982 .lineages
983 .as_ref()
984 .expect("loaded on the line above or already present"))
985 }
986
987 /// [`Lineages::shape_of`], against the cache.
988 ///
989 /// This is what replaced `check_lineages`, and the round trip it replaced
990 /// is the one the review counted (C-6): one `SELECT` over `branches` per
991 /// write, for an answer that changes when a lineage is forked or forgotten.
992 async fn shape_of(
993 &mut self,
994 conn: &libsql::Connection,
995 names: &[&str],
996 ) -> Result<LineageShape> {
997 self.lineages(conn).await?.shape_of(names)
998 }
999
1000 /// The overlap guard, compiled at most once per shape.
1001 ///
1002 /// Keyed on the shape rather than on the statement text because that is the
1003 /// thing [`check_prepared`] reads: it binds four parameters for `Trunk` and
1004 /// five otherwise, so a guard held under one shape and used under another
1005 /// would bind the wrong row even where the SQL happened to match.
1006 async fn guard(
1007 &mut self,
1008 conn: &libsql::Connection,
1009 shape: LineageShape,
1010 branch: &str,
1011 ) -> Result<&OverlapGuard> {
1012 // Keyed on the lineage as well as the shape since 0.15.17: the bound
1013 // ancestry is one reader's answer, so a guard held for `main` cannot
1014 // serve a branch that happens to share its shape. A caller writing to
1015 // one lineage — which is every caller this crate has — still prepares
1016 // once and keeps it across turns.
1017 if !self
1018 .guard
1019 .as_ref()
1020 .is_some_and(|g| g.answers_for(shape, branch))
1021 {
1022 // Resolved against the actor's own cache, then dropped, so the
1023 // borrow ends before the assignment. `lineages` refreshes it when
1024 // `Fork` or `ArchiveBranch` forgot it.
1025 let lineages = self.lineages(conn).await?.clone();
1026 self.guard = Some(OverlapGuard::prepare(conn, shape, &lineages, branch).await?);
1027 }
1028 Ok(self
1029 .guard
1030 .as_ref()
1031 .expect("prepared on the line above or already present"))
1032 }
1033
1034 /// `INSERT_LINK`, compiled once.
1035 ///
1036 /// The single-edge path ran `conn.execute(INSERT_LINK, …)`, which compiles
1037 /// the statement on every call — 61 µs of it, because `links` carries the
1038 /// projection triggers and they are compiled with the insert. This is D-056
1039 /// and D-057's lesson, which was learned on the batch path and never
1040 /// carried across to the path a caller actually waits on.
1041 async fn insert_link(&mut self, conn: &libsql::Connection) -> Result<&libsql::Statement> {
1042 if self.insert_link.is_none() {
1043 self.insert_link = Some(conn.prepare(INSERT_LINK).await?);
1044 }
1045 Ok(self
1046 .insert_link
1047 .as_ref()
1048 .expect("prepared on the line above or already present"))
1049 }
1050}
1051
1052enum LoopCtl {
1053 Continue,
1054 Break,
1055}
1056
1057/// Primary database handle for Macrame bitemporal ledger.
1058///
1059/// # Why this is not `Clone`, and what a multi-consumer caller uses instead
1060///
1061/// **Share it as `Arc<Database>`.** Every method here but one takes `&self`, so
1062/// an `Arc` is a complete handle and not a workaround: reads run concurrently
1063/// off `read_conn`, writes queue behind the actor's channel exactly as they do
1064/// through a `&Database`, and nothing becomes serialised that was not
1065/// serialised already. The exception is [`Database::close`], which takes `self`,
1066/// so the last owner closes with
1067/// `Arc::into_inner(db).expect("last handle").close().await`.
1068///
1069/// That exception is the whole reason `Clone` is absent. Cloning would have to
1070/// duplicate **the right to shut down**, and each field carrying that right
1071/// breaks differently when duplicated:
1072///
1073/// - `writer` is a [`tokio::task::JoinHandle`], which is not `Clone` at all —
1074/// so a hand-written impl would have to give the copy a `None`, and
1075/// `close()` on that copy returns `Ok(())` without ever checking the actor's
1076/// exit status. That status is one of the two reasons [`Drop`] tells callers
1077/// to prefer `close()`.
1078/// - `cadence_stop` is a [`tokio::sync::watch::Sender`], which **is** `Clone`,
1079/// and that is the worse case. Its contract is that *dropping* it stops the
1080/// snapshot task; a watch channel closes when the last sender goes, so one
1081/// surviving copy keeps that task running against a database that is going
1082/// away. Nothing returns an error, which is why this is the argument rather
1083/// than the `JoinHandle`.
1084/// - `closed` is per-handle, so two copies disagree about whether the ledger
1085/// was closed: `Drop` warns about a database that *was* closed, or stays
1086/// silent about one that was not.
1087///
1088/// And the ordering `close()` documents — cadence stopped, actor joined, *then*
1089/// the final snapshot, so that no write can land between the fold and the file
1090/// — is only enforceable while one handle can perform it. A second `close()`
1091/// writes a "final" snapshot with the actor still alive.
1092///
1093/// So the missing impl is the type saying shutdown has exactly one owner. The
1094/// Python binding reached the same shape from the other side and for the same
1095/// reason: `PyDatabase` holds a `RwLock<Option<Database>>` rather than a copy
1096/// per caller (0.13.30, W11.1, D-203).
1097pub struct Database {
1098 db: libsql::Database,
1099 /// The file this handle opened, kept so [`Database::diagnostic_conn`] can
1100 /// open it again under different flags (T5.1, D-091). `archive_path` and
1101 /// `snapshots_dir` are derived from it and were previously the only trace
1102 /// of it on the struct.
1103 path: PathBuf,
1104 read_conn: libsql::Connection,
1105 highpri_tx: mpsc::Sender<HighPriCommand>,
1106 lowpri_tx: mpsc::Sender<LowPriCommand>,
1107 clock: Arc<dyn Clock>,
1108 archive_path: PathBuf,
1109 snapshots_dir: PathBuf,
1110 schema_version: u32,
1111 /// Kept so [`Database::diagnostic_conn`] can configure the connections it
1112 /// mints the same way `open()` configured the internal readers (0.12.16,
1113 /// W5.5, D-159). Before that split, each one ran with SQLite's defaults.
1114 reader_cache_size: Option<i32>,
1115 /// The `SQLITE_OPEN_READ_ONLY` connection behind
1116 /// [`Database::diagnostic_conn`], opened on first use and dropped with this
1117 /// handle (0.15.14, W15.4, review C-9, [D-256]).
1118 ///
1119 /// **The connection, not the `libsql::Database` handle**, and the
1120 /// difference is the measurement rather than a preference. The first shape
1121 /// written here cached the handle and minted a connection per call, on the
1122 /// argument that `diagnostic_conn` promises a connection the *caller* owns.
1123 /// `examples/diagnostic_conn_probe.rs` says `Builder::…build()` costs
1124 /// **0.10 µs and opens nothing** — it succeeds against a path that does not
1125 /// exist — while `connect()` costs **51.5 µs** and is where
1126 /// `SQLITE_CANTOPEN` arrives for a missing file. The handle cache removes a
1127 /// call that does no work.
1128 ///
1129 /// **`Mutex<Option<_>>` rather than the `OnceCell` 0.15.14 shipped**
1130 /// (0.15.15, W15.5, [D-257]). A `OnceCell` can be filled and never
1131 /// emptied, and this connection is handed to arbitrary SQL, so it acquires
1132 /// state that the *next* caller must not inherit — a leaked `BEGIN` above
1133 /// all, which pins a WAL read snapshot and makes both this surface's reads
1134 /// stale and [`Database::checkpoint`] a no-op. The slot has to be
1135 /// clearable for the dirty ones to be replaced, and clearable is what a
1136 /// `OnceCell` is not. `tokio`'s mutex rather than `std`'s because it is
1137 /// held across the open.
1138 ///
1139 /// A failed first attempt leaves the slot empty, so a database whose file
1140 /// appears later is not poisoned by the call that came too early — the
1141 /// property the `OnceCell` had, kept.
1142 ///
1143 /// [D-256]: ../../docs/architecture/s13-decision-register.md#d-256
1144 /// [D-257]: ../../docs/architecture/s13-decision-register.md#d-257
1145 diagnostic_conn: tokio::sync::Mutex<Option<libsql::Connection>>,
1146 writer: Option<tokio::task::JoinHandle<()>>,
1147 /// Stops the snapshot cadence. Dropping it stops the task too, which is what
1148 /// keeps a `Database` that is dropped rather than closed from leaving a task
1149 /// running against a connection whose database is going away.
1150 cadence_stop: Option<tokio::sync::watch::Sender<bool>>,
1151 cadence: Option<tokio::task::JoinHandle<()>>,
1152 /// Set by [`Database::close`]. Read only by [`Drop`], which warns when it is
1153 /// still false — see that impl for why the omission is worth a warning.
1154 closed: bool,
1155 /// Shared with the actor (T1.4, T1.2). Held here rather than behind
1156 /// `#[cfg(feature = "metrics")]` so `open_inner` has one shape; with the
1157 /// feature off the metrics half is a zero-sized type and only
1158 /// [`Database::metrics`] is gated — which is also why the field is unread in
1159 /// the default build: the actor holds the other `Arc` and does the writing.
1160 #[cfg_attr(not(feature = "metrics"), allow(dead_code))]
1161 shared: Arc<ActorShared>,
1162}
1163
1164/// What the snapshot cadence should do, for [`Tuning::cadence`].
1165///
1166/// # Why this is not `Option<SnapshotCadence>`
1167///
1168/// [`Database::open_with_cadence`] takes `Option<SnapshotCadence>`, where `None`
1169/// means *no cadence at all*. Carrying that field into [`Tuning`] unchanged
1170/// would have made it the one field in the struct whose `None` is a request to
1171/// change the behaviour rather than a request to leave it alone — and since
1172/// `Tuning` derives `Default`, `open_tuned(path, Tuning::default())` would then
1173/// have silently disabled snapshots, while `open(path)` runs them. Two calls
1174/// that read as synonyms, one of which stops writing anchors.
1175///
1176/// So the tri-state is written out. `Default` is the default cadence, matching
1177/// [`Database::open`]; `Disabled` is `open_with_cadence(path, None)`, and has to
1178/// be asked for by name.
1179#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
1180#[non_exhaustive]
1181pub enum CadencePolicy {
1182 /// [`SnapshotCadence::default`], as [`Database::open`] uses.
1183 #[default]
1184 Default,
1185 /// No cadence task. `close()` is then the only thing that writes an anchor
1186 /// (§5.5, D-053).
1187 Disabled,
1188 /// An explicit cadence.
1189 Every(SnapshotCadence),
1190}
1191
1192impl CadencePolicy {
1193 /// Collapse to the `Option` the open path has always taken.
1194 fn resolve(self) -> Option<SnapshotCadence> {
1195 match self {
1196 Self::Default => Some(SnapshotCadence::default()),
1197 Self::Disabled => None,
1198 Self::Every(cadence) => Some(cadence),
1199 }
1200 }
1201}
1202
1203/// When SQLite should checkpoint the WAL on its own, for
1204/// [`Tuning::wal_autocheckpoint`] (0.12.14, W5.3, D-157).
1205///
1206/// # Why this is not `Option<u32>`
1207///
1208/// The same reason [`CadencePolicy`] is not `Option<SnapshotCadence>`, and the
1209/// plan for this wave specified `Option<u32>` here too. In a struct that derives
1210/// `Default`, a field whose `None` means *turn the mechanism off* is a field
1211/// that turns the mechanism off for everyone who did not mention it. Absence
1212/// means "leave it alone" everywhere in [`Tuning`], and disabling the automatic
1213/// checkpointer — which is not safe without an explicit
1214/// [`Database::checkpoint`] to replace it — has to be asked for by name.
1215///
1216/// **The default does not change.** 1,000 pages is SQLite's default and stays
1217/// SQLite's default; F-30 is a control-loop perturbation, not a correctness bug,
1218/// and changing a default is a behaviour change for every existing caller.
1219///
1220/// # What disabling it actually buys, measured (0.12.14, W5.3, D-157)
1221///
1222/// F-30 says the automatic checkpointer is an unbudgeted hold *inside* 0.12.0's
1223/// adaptive chunk controller: a checkpoint firing during a chunk transaction is
1224/// charged to that chunk, and since D-146 made the measured hold the input to
1225/// `next_chunk_size`, the controller shrinks in response to work the chunk did
1226/// not do. Three rounds, 6,000 concepts of 1 KB each through `write_concepts`,
1227/// release build:
1228///
1229/// | | longest chunk hold | mean | chunks | over budget | wall |
1230/// |---|---|---|---|---|---|
1231/// | autocheckpoint on (default) | **9.3–10.3 ms** | 2.40–2.44 ms | 125–130 | 24–28 | 304–321 ms |
1232/// | autocheckpoint off | **4.50 ms** | 2.08–2.20 ms | 142–153 | 18–27 | 298–339 ms |
1233///
1234/// **The tail is the finding, and it is real and reproducible.** The longest
1235/// hold roughly halves, and the >10 ms histogram bucket is populated only with
1236/// the checkpointer on — that bucket is the checkpoint, landing inside somebody
1237/// else's transaction and being charged to it. Every round agrees.
1238///
1239/// **What it does not buy is a calmer controller.** `over_budget` overlaps
1240/// between the arms, and total wall time is the same within noise. The
1241/// controller works near the budget boundary either way, because
1242/// [D-090](../docs/architecture/s13-decision-register.md)'s ~0.8 ms
1243/// per-transaction floor and the convergence cost do not go anywhere. So the
1244/// honest statement is that disabling autocheckpoint removes an outlier, not an
1245/// oscillation.
1246///
1247/// **And the cost is deferred, not removed.** The explicit
1248/// [`Database::checkpoint`] at the end of the same fixture moved **8,400–9,100
1249/// frames in 41–45 ms** with the checkpointer off, against **~860 frames in
1250/// 5.5–6.2 ms** with it on. That is the whole trade in one line: the same work,
1251/// moved out of the latency-bounded path and into one hold the caller chose the
1252/// moment for. It is a good trade for a bulk importer and a bad one for an
1253/// interactive process, which is why this is a knob and not a new default.
1254#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
1255#[non_exhaustive]
1256pub enum WalCheckpointPolicy {
1257 /// SQLite's own default: checkpoint once the WAL passes 1,000 pages.
1258 #[default]
1259 Default,
1260 /// No automatic checkpointing.
1261 ///
1262 /// **Only correct if you call [`Database::checkpoint`] yourself.** Without
1263 /// one, the WAL grows for the life of the process and the database file is
1264 /// never brought up to date.
1265 Disabled,
1266 /// Checkpoint once the WAL passes this many pages.
1267 ///
1268 /// `0` is not special-cased to [`Self::Disabled`] even though SQLite treats
1269 /// it that way, because a caller who computed a threshold and got zero has
1270 /// a bug, and inheriting SQLite's overload would turn it into a silently
1271 /// unbounded WAL.
1272 EveryPages(u32),
1273}
1274
1275impl WalCheckpointPolicy {
1276 /// The pragma to run, or `None` to leave the connection at SQLite's
1277 /// default.
1278 fn pragma(self) -> Option<String> {
1279 match self {
1280 Self::Default => None,
1281 Self::Disabled => Some("PRAGMA wal_autocheckpoint = 0".to_string()),
1282 Self::EveryPages(pages) => Some(format!("PRAGMA wal_autocheckpoint = {pages}")),
1283 }
1284 }
1285}
1286
1287/// Everything [`Database::open_tuned`] can be told, in one growable struct
1288/// (0.12.12, W5.1, D-155).
1289///
1290/// # Why a struct rather than a fourth constructor
1291///
1292/// There were three — [`Database::open`], [`Database::open_with_cadence`],
1293/// [`Database::open_with_clock`] — and each new knob added one more, with the
1294/// combinatorics of the ones before it. 0.13.0 alone wanted three knobs
1295/// (`wal_autocheckpoint`, and a page cache each for the writer and the
1296/// readers), which is the point at which the naming stops being possible.
1297///
1298/// **Setters plus `#[non_exhaustive]` are the whole design** (0.15.13, W15.3,
1299/// [C-11], [D-255]). They make a new knob an additive change: callers write
1300/// `Tuning::default().cadence(..)`, and the fields that arrive after them are
1301/// the ones they did not ask about. That is not a hypothetical — W5.1 shipped
1302/// this struct with two fields, and W5.3/W5.4 added three more without
1303/// touching a caller.
1304///
1305/// # Why the attribute needed the setters, and why 0.5.1's answer was the
1306/// other one
1307///
1308/// D-155 specified `#[non_exhaustive]` for this struct and then could not
1309/// ship it, on a fact that is still true: a `#[non_exhaustive]` **struct**
1310/// cannot be built with literal syntax outside its own crate *at all*, and
1311/// the functional-update form is literal syntax, so
1312/// `Tuning { cadence, ..Default::default() }` is `E0639` for every external
1313/// caller — the exact expression the attribute was there to protect. (The rule
1314/// differs from `#[non_exhaustive]` on an enum, which only forces a wildcard
1315/// arm; [`CadencePolicy`] has kept it for that reason since W4.2.) D-155 named
1316/// the two ways to have both — *a builder with setters, or plain `Default`* —
1317/// and chose `Default`, because the field-literal form is the legible one.
1318///
1319/// **What changed is not the argument but the deadline.** `Default` alone
1320/// leaves the growth additive only for callers who wrote
1321/// `..Default::default()`; a caller who wrote the exhaustive literal breaks on
1322/// the next field. Before 1.0 that is a compile error with an obvious fix.
1323/// After it, it is a major version — and this struct is the one in the crate
1324/// whose whole documented purpose is to keep acquiring fields. So the release
1325/// that is still allowed to break callers pays D-155's other price and writes
1326/// the setters, which is the half of its own analysis it declined at the time.
1327///
1328/// **The fields stay `pub` and stay readable**, and on a value you own they
1329/// stay assignable: `let mut t = Tuning::default(); t.cadence = ..;` compiles
1330/// outside this crate exactly as it did. What the attribute forbids is the
1331/// *literal*, which is the one form that enumerates every field and therefore
1332/// the one form a new field can break.
1333///
1334/// [C-11]: ../../docs/Macrame%20Update%20Plan%20v0.16.0.md
1335/// [D-255]: ../../docs/architecture/s13-decision-register.md#d-255
1336///
1337/// # The three constructors stay
1338///
1339/// They delegate here and are not deprecated. `open(path)` is the right call for
1340/// most callers and should not acquire a warning for being the common case; the
1341/// consolidation is about where the *next* knob goes, not about moving anyone.
1342///
1343/// ```no_run
1344/// # use macrame::prelude::*;
1345/// # async fn f() -> macrame::Result<()> {
1346/// let db = Database::open_tuned(
1347/// "graph.db",
1348/// Tuning::default().cadence(CadencePolicy::Disabled),
1349/// )
1350/// .await?;
1351/// # Ok(()) }
1352/// ```
1353#[derive(Clone, Default)]
1354#[non_exhaustive]
1355pub struct Tuning {
1356 /// What the snapshot cadence should do. Defaults to
1357 /// [`SnapshotCadence::default`], as [`Database::open`] does.
1358 pub cadence: CadencePolicy,
1359 /// A clock to stamp `recorded_at` with, for tests (§5.1.2, D-062). `None`
1360 /// is [`SystemClock`]. Floored against the database exactly as
1361 /// [`Database::open_with_clock`] describes — read that before injecting
1362 /// one against a non-empty file.
1363 pub clock: Option<Arc<dyn Clock>>,
1364 /// When SQLite checkpoints the WAL on its own (0.12.14, W5.3, F-30).
1365 ///
1366 /// Applied to the **write connection**, which is the only connection in
1367 /// this crate that commits, and therefore the only one whose autocheckpoint
1368 /// setting can ever fire. Pair [`WalCheckpointPolicy::Disabled`] with an
1369 /// explicit [`Database::checkpoint`] or the WAL grows without bound.
1370 pub wal_autocheckpoint: WalCheckpointPolicy,
1371 /// Page cache for the **write** connection, as SQLite's `cache_size`
1372 /// (0.12.15, W5.4).
1373 ///
1374 /// `None` leaves SQLite's default of −2000, which is −2000 *kibibytes*, or
1375 /// 2 MB. **Negative values are KiB and positive values are pages** — that
1376 /// is SQLite's convention and it is preserved rather than smoothed over,
1377 /// because a caller who knows the pragma should not have to discover that
1378 /// this crate redefined it. `Some(-64_000)` is 64 MB; `Some(64_000)` is
1379 /// 64,000 pages, which at the 4 KiB page size this crate gets is 256 MB.
1380 ///
1381 /// The writer wants a large cache: it is one connection, it holds the write
1382 /// lock while it works, and every page it has to re-read from disk is time
1383 /// no other writer can use.
1384 ///
1385 /// # Unlike the two above, `None` here is not a policy enum
1386 ///
1387 /// Because SQLite's default is a *value* rather than a mechanism. Absence
1388 /// still means "leave it alone" — it just happens that leaving this alone
1389 /// is expressible as not running a pragma, where leaving the automatic
1390 /// checkpointer alone required saying which of two things "alone" meant.
1391 pub writer_cache_size: Option<i32>,
1392 /// Page cache for every **read-only** connection: the shared
1393 /// [`Database::read_conn`], the snapshot cadence's own connection, and
1394 /// (since W5.5) each [`Database::diagnostic_conn`] (0.12.15, W5.4).
1395 ///
1396 /// Same units as [`Self::writer_cache_size`], and the same `None`.
1397 ///
1398 /// Split from the writer's because the profiles are opposite and one number
1399 /// cannot serve both. There is exactly one writer and it is long-lived, so
1400 /// its cache is a fixed cost paid once. Read-only connections are plural —
1401 /// the shared reader and the cadence's — so a large value here is
1402 /// multiplied by however many exist, which is the wrong size for the one
1403 /// connection that holds the write lock.
1404 ///
1405 /// **The multiplier used to be unbounded** and is not since 0.15.14
1406 /// (W15.4, [D-256]): `diagnostic_conn` minted a connection per call, so a
1407 /// caller in a loop multiplied this number by their own call count. There
1408 /// is one such connection per `Database` now, so the count is three.
1409 ///
1410 /// [D-256]: ../../docs/architecture/s13-decision-register.md#d-256
1411 pub reader_cache_size: Option<i32>,
1412 /// What to do about a stored `recorded_at` in the future (0.13.5, W7.4,
1413 /// §3.4).
1414 ///
1415 /// The clock floors itself at `MAX(recorded_at)` so stamps stay strictly
1416 /// increasing across restarts, which means one row from the future becomes
1417 /// this process's floor and every stamp it issues inherits it — into rows
1418 /// the next open reads back. Defaults to refusing beyond
1419 /// [`crate::DEFAULT_FUTURE_STAMP_TOLERANCE`], a day.
1420 ///
1421 /// Like [`Self::wal_autocheckpoint`] and unlike the two cache sizes, this
1422 /// is a policy enum rather than an `Option`, for
1423 /// [D-155](../../docs/architecture/s13-decision-register.md)'s reason: it
1424 /// guards an invariant, and a `None` that switches it off would switch it
1425 /// off for every caller who never heard of it.
1426 pub future_stamps: FutureStampPolicy,
1427}
1428
1429impl Tuning {
1430 // The setters below are what make `#[non_exhaustive]` payable (0.15.13,
1431 // W15.3, D-255). One per field, named after it, taking `self` — so
1432 // `Tuning::default().cadence(x).writer_cache_size(y)` is an expression, and
1433 // a field added later is a method added later rather than a break. They are
1434 // deliberately not clever: no `Into`, no grouping of two knobs under one
1435 // name, nothing that would have to be redesigned the first time a field
1436 // does not fit the pattern.
1437
1438 /// What the snapshot cadence should do — the [`cadence`](Self::cadence)
1439 /// field.
1440 pub fn cadence(mut self, cadence: CadencePolicy) -> Self {
1441 self.cadence = cadence;
1442 self
1443 }
1444
1445 /// Inject a clock — the [`clock`](Self::clock) field.
1446 ///
1447 /// Takes the clock rather than an `Option`, because `None` is what
1448 /// [`Tuning::default`] already holds and a setter whose argument can undo
1449 /// itself invites `clock(None)` as a way of saying nothing.
1450 /// [`Database::open_with_clock`] documents the flooring this is subject to;
1451 /// read it before injecting one against a non-empty file.
1452 pub fn clock(mut self, clock: Arc<dyn Clock>) -> Self {
1453 self.clock = Some(clock);
1454 self
1455 }
1456
1457 /// When SQLite checkpoints the WAL on its own — the
1458 /// [`wal_autocheckpoint`](Self::wal_autocheckpoint) field.
1459 pub fn wal_autocheckpoint(mut self, policy: WalCheckpointPolicy) -> Self {
1460 self.wal_autocheckpoint = policy;
1461 self
1462 }
1463
1464 /// Page cache for the write connection, in SQLite's units — the
1465 /// [`writer_cache_size`](Self::writer_cache_size) field, which documents
1466 /// why negative means KiB and positive means pages.
1467 pub fn writer_cache_size(mut self, size: i32) -> Self {
1468 self.writer_cache_size = Some(size);
1469 self
1470 }
1471
1472 /// Page cache for every read-only connection — the
1473 /// [`reader_cache_size`](Self::reader_cache_size) field, which documents
1474 /// why this is not the same number as the writer's.
1475 pub fn reader_cache_size(mut self, size: i32) -> Self {
1476 self.reader_cache_size = Some(size);
1477 self
1478 }
1479
1480 /// What to do about a stored `recorded_at` in the future — the
1481 /// [`future_stamps`](Self::future_stamps) field.
1482 pub fn future_stamps(mut self, policy: FutureStampPolicy) -> Self {
1483 self.future_stamps = policy;
1484 self
1485 }
1486
1487 /// The `Option<SnapshotCadence>` the three older constructors take, mapped
1488 /// onto the tri-state. `None` there means *disabled*, which is why
1489 /// [`CadencePolicy`] exists — see its docs.
1490 fn from_legacy(cadence: Option<SnapshotCadence>, clock: Option<Arc<dyn Clock>>) -> Self {
1491 Self {
1492 cadence: match cadence {
1493 Some(cadence) => CadencePolicy::Every(cadence),
1494 None => CadencePolicy::Disabled,
1495 },
1496 clock,
1497 wal_autocheckpoint: WalCheckpointPolicy::default(),
1498 writer_cache_size: None,
1499 reader_cache_size: None,
1500 future_stamps: FutureStampPolicy::default(),
1501 }
1502 }
1503}
1504
1505// `Clock` is not `Debug` — it is a behavioural trait with two methods and
1506// requiring `Debug` of every implementor to print a handle here would be the
1507// tail wagging the dog. So the field is reported as present-or-absent, which is
1508// the only part of it a reader of a `Tuning` dump can act on.
1509impl std::fmt::Debug for Tuning {
1510 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1511 f.debug_struct("Tuning")
1512 .field("cadence", &self.cadence)
1513 .field("clock", &self.clock.as_ref().map(|_| "<injected>"))
1514 .field("wal_autocheckpoint", &self.wal_autocheckpoint)
1515 .field("writer_cache_size", &self.writer_cache_size)
1516 .field("reader_cache_size", &self.reader_cache_size)
1517 .finish()
1518 }
1519}
1520
1521impl Database {
1522 /// Open a database file at `path`, configuring pragmas, running migrations, and spawning the Write Actor.
1523 ///
1524 /// The snapshot cadence runs with [`SnapshotCadence::default`]. Use
1525 /// [`Database::open_with_cadence`] to tune or disable it.
1526 pub async fn open(path: impl AsRef<Path>) -> Result<Self> {
1527 Self::open_with_cadence(path, Some(SnapshotCadence::default())).await
1528 }
1529
1530 /// Open with an explicit snapshot cadence, or `None` to run without one
1531 /// (§5.5, D-053).
1532 ///
1533 /// `None` restores the pre-0.5.5 behaviour, where `close()` is the only
1534 /// thing that ever writes an anchor. That is the right setting for a
1535 /// short-lived process that will not accumulate a delta worth bounding, and
1536 /// for tests that assert on the contents of the snapshot directory.
1537 pub async fn open_with_cadence(
1538 path: impl AsRef<Path>,
1539 cadence: Option<SnapshotCadence>,
1540 ) -> Result<Self> {
1541 Self::open_inner(path.as_ref(), Tuning::from_legacy(cadence, None)).await
1542 }
1543
1544 /// Open with an injected clock (§5.1.2, **defect K**, D-062).
1545 ///
1546 /// The reason this exists is testing: `recorded_at` is the transaction-time
1547 /// axis, and until now every test that wanted to assert on one had to either
1548 /// avoid it or drive a raw connection, because `open()` hardcoded
1549 /// [`SystemClock`]. `FakeClock` has been public and constructed in the test
1550 /// harness since 0.5.2 with nothing to inject it into — the compiler warned
1551 /// about the dead field on every build for three releases.
1552 ///
1553 /// **The clock is floored against the database before the actor starts.**
1554 /// [`Clock::raise_floor`] is called with the newest `recorded_at` in the
1555 /// ledger, so an injected clock cannot issue a stamp below what is already
1556 /// stored — which would abort the next concept write on
1557 /// `trg_concepts_monotonic_ra` rather than merely being odd. This is the
1558 /// step whose absence kept the defect open: the obvious implementation
1559 /// (take an `Arc<dyn Clock>`, use it) produces a `Database` that fails on
1560 /// its first write against any non-empty file.
1561 ///
1562 /// On a fresh database there is no floor, so an injected `FakeClock` issues
1563 /// exactly the stamps it was given.
1564 pub async fn open_with_clock(
1565 path: impl AsRef<Path>,
1566 cadence: Option<SnapshotCadence>,
1567 clock: Arc<dyn Clock>,
1568 ) -> Result<Self> {
1569 Self::open_inner(path.as_ref(), Tuning::from_legacy(cadence, Some(clock))).await
1570 }
1571
1572 /// Open with an explicit [`Tuning`] (0.12.12, W5.1, D-155).
1573 ///
1574 /// The consolidated form of the three constructors above, and the one that
1575 /// grows: every knob 0.13.0 adds arrives as a field here rather than as a
1576 /// fourth `open_*`. See [`Tuning`] for why the struct is
1577 /// `#[non_exhaustive]` and why that makes the growth additive.
1578 ///
1579 /// That sentence was written at 0.12.12 and was false until 0.15.13: the
1580 /// struct was *not* `#[non_exhaustive]`, and [`Tuning`]'s own docs carried
1581 /// a section arguing at length that it should not be. Two documents in one
1582 /// file contradicting each other for eleven releases, which is the shape
1583 /// C-16 is about; W15.3 resolved it by making this one true.
1584 pub async fn open_tuned(path: impl AsRef<Path>, tuning: Tuning) -> Result<Self> {
1585 Self::open_inner(path.as_ref(), tuning).await
1586 }
1587
1588 async fn open_inner(path: &Path, tuning: Tuning) -> Result<Self> {
1589 let Tuning {
1590 cadence,
1591 clock: injected,
1592 wal_autocheckpoint,
1593 writer_cache_size,
1594 reader_cache_size,
1595 future_stamps,
1596 } = tuning;
1597 let cadence = cadence.resolve();
1598 let db = libsql::Builder::new_local(path).build().await?;
1599 let write_conn = configure(db.connect()?, writer_cache_size).await?;
1600 // The writer is the only connection that commits, so it is the only one
1601 // whose `wal_autocheckpoint` can ever fire. Setting it on the readers
1602 // would be a pragma with no path to running (0.12.14, W5.3, D-157).
1603 if let Some(pragma) = wal_autocheckpoint.pragma() {
1604 let _ = write_conn.query(&pragma, ()).await?;
1605 }
1606 let read_conn = configure(db.connect()?, reader_cache_size).await?;
1607
1608 // PRAGMA query_only = ON on reader connection (§5.1.2)
1609 read_conn.execute("PRAGMA query_only = ON", ()).await?;
1610
1611 let migration = migrations::run(&write_conn).await?;
1612
1613 let (highpri_tx, highpri_rx) = mpsc::channel(256);
1614 let (lowpri_tx, lowpri_rx) = mpsc::channel(64);
1615
1616 // Floored after `migrations::run`, so the tables the floor is read from
1617 // are guaranteed to exist.
1618 let clock: Arc<dyn Clock> = match injected {
1619 Some(clock) => {
1620 if let Some(floor) =
1621 crate::util::clock::recorded_at_floor(&read_conn, future_stamps).await?
1622 {
1623 clock.raise_floor(floor);
1624 }
1625 clock
1626 }
1627 None => Arc::new(SystemClock::new(&read_conn, future_stamps).await?),
1628 };
1629 let shared = Arc::new(ActorShared::default());
1630 let writer = tokio::spawn(run_writer_actor(
1631 write_conn,
1632 Arc::clone(&clock),
1633 highpri_rx,
1634 lowpri_rx,
1635 Arc::clone(&shared),
1636 ));
1637
1638 let archive_path = derive_archive_path(path);
1639 let snapshots_dir = derive_snapshots_dir(path);
1640
1641 // **The cadence gets its own connection (Wave 4.1).** It used to share
1642 // `read_conn`, on the reasoning that `libsql::Connection` is an
1643 // Arc-backed handle and R15 makes every extra local connection a cost worth
1644 // not paying for nothing. The cost it was not paying for turned out to be
1645 // real: `reconstruct` brackets a fold with `ATTACH cold … DETACH cold`,
1646 // that region is per-connection state, and it is not synchronised. Two
1647 // folds on one connection can therefore interleave so that one DETACHes
1648 // the handle the other is mid-fold on.
1649 //
1650 // Recorded in §8.5 as a hazard rather than a defect because it **did not
1651 // reproduce**: 200 concurrent reconstructions against a 1 ms cadence with
1652 // an archive present produced zero errors, since the cadence anchors at
1653 // `MAX(recorded_at)` and so almost always takes the hot path. Narrow, and
1654 // real — a write landing between `log_head` and the fold opens it.
1655 //
1656 // Separate connections remove the interleaving rather than ordering it,
1657 // which is why this is preferred to a mutex around the region: there is
1658 // no shared state left to race on, and nothing to remember to hold. The
1659 // R15 objection does not apply — that fault is about *concurrent* opens,
1660 // and this is one more sequential open during `open()`.
1661 let (cadence_stop, cadence) = match cadence {
1662 Some(cadence) => {
1663 let cadence_conn = configure(db.connect()?, reader_cache_size).await?;
1664 cadence_conn.execute("PRAGMA query_only = ON", ()).await?;
1665 let (tx, rx) = tokio::sync::watch::channel(false);
1666 let handle = tokio::spawn(snapshot::run_cadence(
1667 cadence_conn,
1668 snapshots_dir.clone(),
1669 archive_path.clone(),
1670 cadence,
1671 rx,
1672 Arc::clone(&shared) as Arc<dyn snapshot::CommittedTurns>,
1673 ));
1674 (Some(tx), Some(handle))
1675 }
1676 None => (None, None),
1677 };
1678
1679 let handle = Self {
1680 db,
1681 path: path.to_path_buf(),
1682 read_conn,
1683 highpri_tx,
1684 lowpri_tx,
1685 clock,
1686 archive_path,
1687 snapshots_dir,
1688 schema_version: migrations::current_version(),
1689 reader_cache_size,
1690 diagnostic_conn: tokio::sync::Mutex::new(None),
1691 writer: Some(writer),
1692 cadence_stop,
1693 cadence,
1694 closed: false,
1695 shared,
1696 };
1697
1698 // **Re-anchor after a migration (Wave 4.4).**
1699 //
1700 // D-043 makes a `SCHEMA_VERSION` bump invalidate every snapshot on disk,
1701 // which is correct — a snapshot is a serialised `MaterializedState` and a
1702 // schema change can change what that means. What was missing is the other
1703 // half: nothing wrote a replacement, so the first `reconstruct` after an
1704 // upgrade skipped every file as incompatible and folded from genesis. On
1705 // a database with a large log that is the difference between reading one
1706 // snapshot and folding the whole history, and the only trace was a
1707 // `warn!` per skipped file.
1708 //
1709 // Written here rather than left to the cadence because the cadence fires
1710 // on log *growth* (D-053): an upgraded database that is then read but not
1711 // written would never re-anchor at all.
1712 //
1713 // Failure is logged, not returned. A missing anchor costs time and no
1714 // information — snapshots are derivative under Doctrine VI — so refusing
1715 // to open a database because its optimisation could not be rebuilt would
1716 // trade a real capability for a performance one.
1717 //
1718 // Gated on the cadence being enabled, as well as on an actual upgrade:
1719 // `open_with_cadence(None)` means *this handle writes no snapshots except
1720 // at close()*, and a one-off write at open would contradict that for a
1721 // caller who asked for the quiet mode precisely to control when files
1722 // appear. They still get an anchor from `close()`.
1723 if migration.upgraded() && handle.cadence.is_some() {
1724 let ts = handle.clock.now();
1725 let archive = crate::temporal::archive::archive_present(&handle.archive_path)
1726 .then_some(handle.archive_path.as_path());
1727 match snapshot::write_final(&handle.read_conn, &handle.snapshots_dir, &ts, archive)
1728 .await
1729 {
1730 Ok(path) => tracing::info!(
1731 "schema moved v{} -> v{}; re-anchored snapshots at {:?}",
1732 migration.from,
1733 migration.to,
1734 path
1735 ),
1736 Err(e) => tracing::warn!(
1737 "schema moved v{} -> v{} but the re-anchor failed: {e}. \
1738 Reconstruction stays correct and folds from genesis until the \
1739 cadence writes one.",
1740 migration.from,
1741 migration.to
1742 ),
1743 }
1744 }
1745
1746 Ok(handle)
1747 }
1748
1749 /// Read connection handle for queries, traversals, and folds.
1750 pub fn read_conn(&self) -> &libsql::Connection {
1751 &self.read_conn
1752 }
1753
1754 /// The file this handle opened.
1755 pub fn path(&self) -> &Path {
1756 &self.path
1757 }
1758
1759 /// The **OS-level read-only** connection to this database, for diagnostics
1760 /// (§4.7, T5.1, D-091).
1761 ///
1762 /// # Why this exists when `read_conn()` already does
1763 ///
1764 /// Two different things, and the difference is the point:
1765 ///
1766 /// * `read_conn()` returns a shared `&Connection` carrying
1767 /// `PRAGMA query_only = ON`. That pragma is **per-connection and
1768 /// reversible by its holder in one statement**, so it is a guardrail
1769 /// against accident, not a capability boundary. And because it is the
1770 /// connection the crate's own traversals and folds run on, a caller who
1771 /// runs a long reporting query there is competing with all of them.
1772 /// * This returns a connection opened with `SQLITE_OPEN_READ_ONLY`, which is
1773 /// enforced by the engine below the pragma layer, and which nothing inside
1774 /// the crate runs on.
1775 ///
1776 /// # One connection per `Database`, shared between callers (0.15.14, W15.4)
1777 ///
1778 /// Through 0.15.13 this minted a connection per call and the sentence above
1779 /// read *"a **new, independently owned** … connection"*. Review item C-9
1780 /// asked for the file to be opened once per handle instead, and the
1781 /// measurement behind it (`examples/diagnostic_conn_probe.rs`) was sharper
1782 /// than the ask: `connect()` is **51.5 µs of an 82.7 µs call**, and
1783 /// `Builder::…build()` — which every document in this crate called *the
1784 /// open* — is **0.10 µs and opens nothing**, succeeding against a path that
1785 /// does not exist. Caching the handle would have removed 0.10 µs. What
1786 /// ships caches the connection: **82.7 µs → 19.9 µs**, and what remains is
1787 /// the `stat` below, not the connection.
1788 ///
1789 /// **So per-connection state is shared between diagnostic callers.** An
1790 /// `ATTACH`, a `PRAGMA`, a temp table one caller creates is visible to the
1791 /// next — which matters here and nowhere else in this API, because
1792 /// `diagnostic_query` is the one arbitrary-SQL surface the crate exposes.
1793 /// Asserted in `diagnostic_callers_share_one_connection_and_its_state`
1794 /// rather than left to this paragraph.
1795 ///
1796 /// What that costs is isolation between *diagnostic* callers. What it does
1797 /// not cost is the thing D-091 was for: this is still not `read_conn()`, a
1798 /// reporting query here still does not compete with the crate's traversals
1799 /// and folds, and the read-only boundary below is untouched.
1800 ///
1801 /// # What is scrubbed before you get it, and what is not (0.15.15, W15.5)
1802 ///
1803 /// 0.15.14 shipped the sharing and documented it. [D-257] measured what it
1804 /// actually admits, and one of the four is not an isolation nuisance but a
1805 /// correctness hole that leaves this surface entirely: **a leaked `BEGIN`
1806 /// pins a WAL read snapshot.** Measured — 200 writes through the typed
1807 /// surface while a diagnostic caller held an unclosed read transaction —
1808 /// later diagnostic reads answered **1 row instead of 201**, silently, and
1809 /// [`Database::checkpoint`] became a no-op with the WAL stuck at 8.5 MB
1810 /// until the transaction was rolled back. Stale answers on the surface a
1811 /// caller reaches for when they already distrust the typed one, and an
1812 /// unbounded WAL on a method that has nothing to do with diagnostics.
1813 ///
1814 /// So this method scrubs on entry rather than trusting the caller, and the
1815 /// prices are from `examples/diagnostic_hygiene_probe.rs`:
1816 ///
1817 /// | left behind | how it is found | what happens |
1818 /// |---|---|---|
1819 /// | an open transaction | `is_autocommit()`, **0.04 µs** | `ROLLBACK`, 2.4 µs |
1820 /// | a temp table or view | `PRAGMA temp.schema_version` | connection dropped, re-minted at 56.6 µs |
1821 /// | an `ATTACH` | `PRAGMA database_list` | connection dropped, re-minted |
1822 /// | `busy_timeout`, `cache_size` | not detected — restated, 1.0 µs | reset to the crate's values |
1823 /// | any other pragma | **not detected** | **inherited by the next caller** |
1824 ///
1825 /// The two dirt questions are asked as pragmas because the same two asked
1826 /// over `temp.sqlite_master` and `pragma_database_list` cost **7.8 µs**
1827 /// against **2.4 µs**, on a call whose entire warm cost is the `stat`.
1828 ///
1829 /// The last row is the honest residue. SQLite has no cheap enumeration of
1830 /// connection-scoped pragma state, so the crate restates the two pragmas
1831 /// *it* set (D-159's `busy_timeout` above all — a caller who sets it to 0
1832 /// would otherwise remove the 5 s margin from every later diagnostic call)
1833 /// and leaves the rest. A caller who sets `case_sensitive_like` or
1834 /// `recursive_triggers` changes what **later diagnostic queries on this
1835 /// handle** see, and nothing else: the crate's own readers and writer are
1836 /// different connections, so no typed answer can move. That is a smaller
1837 /// blast radius than 0.15.14 had and a larger one than zero, and it is
1838 /// written down rather than rounded off.
1839 ///
1840 /// That paragraph is about pragmas that belong to a *connection*, which is
1841 /// what "residue" means and what the scrub is for. **Not every pragma
1842 /// reachable here is one**, and the section below is the exception —
1843 /// measured after D-257 claimed this one "cannot change any typed answer"
1844 /// without checking (0.15.16, [D-258]).
1845 ///
1846 /// **Scrubbed on entry, not on exit**, because there is no exit: this
1847 /// returns a `Connection` clone the caller keeps for as long as it likes,
1848 /// and the crate is never told they are done. Entry is the one place that
1849 /// covers both this method and `diagnostic_query`. The consequence is that
1850 /// a caller who leaks a transaction and never calls again holds the pin
1851 /// until the handle drops — so the Python binding, which *does* know when
1852 /// a query is over, scrubs on exit as well.
1853 ///
1854 /// **Measured on libSQL 0.9.30 rather than assumed**
1855 /// (`examples/readonly_open_probe.rs`), against a live WAL database with the
1856 /// write actor running:
1857 ///
1858 /// | | `read_conn()` | `diagnostic_conn()` |
1859 /// |---|---|---|
1860 /// | `SELECT`, `EXPLAIN QUERY PLAN` | allowed | allowed |
1861 /// | `INSERT` | refused | refused |
1862 /// | `PRAGMA query_only = OFF` | **allowed** | allowed |
1863 /// | `INSERT` after that | **allowed** | **refused** |
1864 /// | `ATTACH` an existing file | allowed | allowed |
1865 /// | `INSERT` into the attachment | refused¹ | **refused** |
1866 /// | `ATTACH` a path that does not exist | — | refused (`SQLITE_CANTOPEN`) |
1867 ///
1868 /// The third and fourth rows are the whole difference: turning the pragma
1869 /// off restores writes on `read_conn()` and does not here. That is what
1870 /// "boundary rather than guardrail" means, and it is now a number rather
1871 /// than a claim.
1872 ///
1873 /// ¹ On `read_conn()` that refusal is `query_only` — the same reversible
1874 /// thing as row 2. On `diagnostic_conn()` it is the open flags, and the
1875 /// probe runs it *after* `query_only = OFF` so that the pragma cannot be
1876 /// what is doing the work.
1877 ///
1878 /// # `ATTACH` is permitted, and does not widen the write boundary
1879 ///
1880 /// Checked because `diagnostic_query` (Python) is the only arbitrary-SQL
1881 /// surface this crate exposes, and an attachment is a second `open` whose
1882 /// flags it does not obviously inherit. It does inherit them: the
1883 /// attachment is read-only, and a nonexistent path is `SQLITE_CANTOPEN`
1884 /// rather than a new file, because `SQLITE_OPEN_CREATE` is dropped for the
1885 /// attachment as it is for `main`. So `SQLITE_OPEN_READ_ONLY` bounds the
1886 /// **connection**, not just the one file it names (0.10.0, W4.3).
1887 ///
1888 /// What it does widen is *reading*: an `ATTACH` can name any file the
1889 /// process can open, so this connection is a read surface over the
1890 /// filesystem, not over this database. That is a property of arbitrary SQL
1891 /// rather than of the flags, and it is unchanged by them.
1892 ///
1893 /// # One pragma here can end the process, and sharing is not why
1894 ///
1895 /// `PRAGMA hard_heap_limit = 1` through this connection leaves the whole
1896 /// **process** unable to use SQLite. Not this connection, not this handle:
1897 /// measured (`tests_py/probes/diagnostic_global_pragmas.py`), the next
1898 /// ordinary write, the next read, `checkpoint()`, `close()`, and opening a
1899 /// *different* database file all fail with `out of memory`, permanently.
1900 ///
1901 /// `SQLITE_OPEN_READ_ONLY` does not stand in the way because setting it is
1902 /// not a write to the database file, and the scrub above does not help
1903 /// because there is nothing left on the connection to scrub: the limit
1904 /// lives in the SQLite library, one per process. **Re-measured with a
1905 /// connection minted per call — the 0.15.13 shape, before any of the
1906 /// sharing this method now does — the outcome is identical.** So this is
1907 /// not a cost of [D-256]'s shared connection and no amount of hygiene
1908 /// addresses it.
1909 ///
1910 /// Six other candidates were measured and are harmless: `soft_heap_limit`
1911 /// (a hint, not a wall), `locking_mode = EXCLUSIVE` (accepted; the writer
1912 /// kept working), `temp_store_directory`, `max_page_count` (clamped, and
1913 /// per-connection), `case_sensitive_like` (the control), and
1914 /// `wal_checkpoint`, which is refused outright because this connection is
1915 /// read-only.
1916 ///
1917 /// It belongs with the `ATTACH` note above rather than with the scrub: both
1918 /// are properties of handing a caller **arbitrary SQL**, not of the flags
1919 /// the connection was opened with. The practical form is one sentence —
1920 /// *`diagnostic_query` is not a safe place to put a string that came from
1921 /// somewhere else* — which `ATTACH` already made true and this makes
1922 /// sharper. Not blocked by refusing statements that look like this one,
1923 /// because matching SQL text is guesswork wearing the costume of a
1924 /// guarantee, and it would do nothing for a Rust caller holding the
1925 /// connection directly ([D-258]).
1926 ///
1927 /// [D-258]: ../../docs/architecture/s13-decision-register.md#d-258
1928 ///
1929 /// # One way this is *more* permissive, which is worth knowing
1930 ///
1931 /// `CREATE TEMP TABLE` **succeeds** here and is refused by `read_conn()`.
1932 /// Temp tables live in a separate temporary database that is writable
1933 /// regardless of how the main one was opened, whereas `query_only` refuses
1934 /// them outright — which is the mechanism [D-050] measured when it removed
1935 /// `TwoPhaseTempTable` for returning `SQLITE_READONLY (8)` on the read
1936 /// connection. So the stronger boundary is not uniformly stronger, and a
1937 /// strategy that needs a temp table has a connection it could run on. That
1938 /// is recorded, not acted on: D-050 removed the strategy for two reasons and
1939 /// this addresses one of them.
1940 ///
1941 /// # Calling this concurrently was R15's shape, and 0.15.14 is why it is not
1942 ///
1943 /// Through 0.15.13 this section said *"this is the one method on `Database`
1944 /// that opens the file … each call is a fresh `libsql::Builder::…build()`,
1945 /// so *N* threads calling it at once are *N* concurrent opens"*. The first
1946 /// half was true and the second was wrong about which call does it:
1947 /// `build()` opens nothing, and `connect()` is the open. The conclusion
1948 /// happened to be right for the wrong reason, which is why it took a
1949 /// measurement to move.
1950 ///
1951 /// **Measured through the unlocked Python binding**, 48 threads on a
1952 /// barrier, 30 runs per arm (`tests_py/probes/r15_diagnostic_path.py`):
1953 ///
1954 /// | arm | bad runs |
1955 /// |---|---|
1956 /// | a connection per call, as before 0.15.14 | **3 / 30** |
1957 /// | the `libsql::Database` handle cached, a connection per call | 2 / 30 |
1958 /// | the `connect()` serialised behind a mutex | 1 / 18 |
1959 /// | **one connection, as shipped** | **0 / 30** |
1960 ///
1961 /// Rows two and three are why the shape that preserved the old contract was
1962 /// not taken: caching the handle leaves the crash because it leaves the
1963 /// `connect()`, and serialising the `connect()` alone does not reach zero —
1964 /// the race is between minting a connection and the *use* of the others,
1965 /// not between two mintings.
1966 ///
1967 /// **There is nothing left on this path to bound**, because after the first
1968 /// call it no longer opens anything. `Database::open` from many threads is
1969 /// still R15's shape and `examples/r15_soak.rs` still reproduces it; this
1970 /// method is no longer a way to reach it. The Python binding keeps its
1971 /// mutex as margin rather than as a measured necessity — see
1972 /// `PyDatabase::diagnostic_rows`.
1973 ///
1974 /// # Errors
1975 ///
1976 /// The file must already exist. `SQLITE_OPEN_READ_ONLY` drops
1977 /// `SQLITE_OPEN_CREATE` with it, so a missing file is `SQLITE_CANTOPEN`
1978 /// rather than a fresh empty database — which is the right failure, and is
1979 /// surfaced as a typed error rather than as libSQL's error 14.
1980 ///
1981 /// The check is a `stat` on **every** call, not only the first, and it is
1982 /// still most of what a warm call costs (18.6 µs of the 22 µs a clean one
1983 /// takes since 0.15.15). It is kept at that price because the alternative
1984 /// is the worst failure a *diagnostic* surface can have: a cached
1985 /// connection whose file has been deleted and replaced answers from the old
1986 /// inode, silently, on the one method a caller reaches for when they
1987 /// already doubt the typed answer.
1988 pub async fn diagnostic_conn(&self) -> Result<libsql::Connection> {
1989 let fail = |reason: String| DbError::DiagnosticConn {
1990 path: self.path.display().to_string(),
1991 reason,
1992 };
1993 // Checked per call rather than once, because it is the documented
1994 // error of *this* method and costs a `stat`. The handle below is
1995 // opened once; the question "is the file there" is asked every time,
1996 // so a caller who deletes the file still gets the typed refusal on the
1997 // next call rather than a connection to an inode nothing can name.
1998 if !self.path.exists() {
1999 return Err(fail(
2000 "the file does not exist, and a read-only open cannot create it".to_string(),
2001 ));
2002 }
2003 let mut slot = self.diagnostic_conn.lock().await;
2004
2005 // Scrub what the last caller left, before this one can inherit it
2006 // (0.15.15, W15.5, D-257). Free on a clean connection: the transaction
2007 // check is a C call at 0.04 us and the two dirt pragmas are 2.4 us,
2008 // against a `stat` of 18.6 that has already happened above.
2009 scrub(&mut slot).await;
2010
2011 match slot.as_ref() {
2012 // Restated per call since 0.15.15 rather than once at mint. The
2013 // pragmas are per-connection, and there is one connection now, so
2014 // 0.15.14 set them once — which was right until the shared
2015 // connection was also something a caller could move them on. No
2016 // dirt check can see a pragma, so the crate restores its own
2017 // instead of detecting that they went: 1.0 us, against a
2018 // `busy_timeout` of 0 outliving the call that set it.
2019 Some(conn) => configure_common(conn, self.reader_cache_size).await?,
2020 None => {
2021 *slot = Some(self.open_diagnostic_conn(&fail).await?);
2022 }
2023 }
2024
2025 Ok(slot
2026 .as_ref()
2027 .expect("the slot was filled above or the open returned Err")
2028 .clone())
2029 }
2030
2031 /// Roll back and discard whatever the last diagnostic caller left behind,
2032 /// without handing a connection out (0.15.15, W15.5, [D-257]).
2033 ///
2034 /// [`Database::diagnostic_conn`] does this on the way *in*, which is the
2035 /// only place the crate can do it: the method returns a `Connection` clone
2036 /// and is never told the caller is finished with it. That covers every
2037 /// caller and leaves one gap — somebody who leaks a transaction and then
2038 /// never calls again holds the WAL read snapshot until the handle drops,
2039 /// which makes [`Database::checkpoint`] a no-op for that whole time.
2040 ///
2041 /// This is for the callers that *do* know when they are done. It costs the
2042 /// scrub and not the `stat` — around 3.5 µs on a clean connection — because
2043 /// it hands nothing back and so has nothing to promise about the file still
2044 /// being there.
2045 ///
2046 /// **The Python binding does not call it**, though the first draft did. A
2047 /// mutation deleting that call left the whole suite green, and the reason is
2048 /// that the gap is not reachable from there: `diagnostic_query` runs one
2049 /// statement, a bare `BEGIN` pins nothing — the snapshot is taken by the
2050 /// first *read* inside the transaction — and any statement that would take
2051 /// it arrives through the same method, whose entry scrub has already rolled
2052 /// the transaction back. The gap is real for a Rust caller holding a clone
2053 /// across both, which is what this method and
2054 /// `scrubbing_releases_the_pin_without_handing_out_a_connection` are for.
2055 ///
2056 /// Infallible by construction: everything it might have reported is
2057 /// something it responds to by discarding the connection, and the next
2058 /// [`Database::diagnostic_conn`] opens a fresh one.
2059 ///
2060 /// [D-257]: ../../docs/architecture/s13-decision-register.md#d-257
2061 pub async fn scrub_diagnostic_conn(&self) {
2062 let mut slot = self.diagnostic_conn.lock().await;
2063 scrub(&mut slot).await;
2064 }
2065
2066 /// The cold half of [`Database::diagnostic_conn`]: the actual open.
2067 ///
2068 /// Reached on the first call and on any call whose predecessor left the
2069 /// connection dirty enough to discard. It costs 56.5 µs against the warm
2070 /// path's scrub, and it is split out for reading rather than for speed:
2071 /// `Box::pin`ning it here — on the theory that carrying `Builder::build()`'s
2072 /// state machine inside the warm path's was what made a clean call 29.8 µs
2073 /// rather than the 21.8 its parts measure — changed nothing at all
2074 /// (0.15.15, W15.5, [D-257]).
2075 ///
2076 /// [D-257]: ../../docs/architecture/s13-decision-register.md#d-257
2077 async fn open_diagnostic_conn(
2078 &self,
2079 fail: &dyn Fn(String) -> DbError,
2080 ) -> Result<libsql::Connection> {
2081 let db = libsql::Builder::new_local(&self.path)
2082 .flags(libsql::OpenFlags::SQLITE_OPEN_READ_ONLY)
2083 .build()
2084 .await
2085 .map_err(|e| fail(e.to_string()))?;
2086 let conn = db.connect().map_err(|e| fail(e.to_string()))?;
2087 // Configured since 0.12.16 (W5.5, D-159). Until then this connection
2088 // ran with SQLite's defaults while every other connection in the
2089 // process ran with the crate's — most consequentially a `busy_timeout`
2090 // of 0 against everyone else's 5 s, on the one surface whose job is to
2091 // answer questions when the typed path is already suspect. Only the
2092 // common half: `SQLITE_OPEN_READ_ONLY` cannot set `journal_mode`, and
2093 // the rest govern writes this connection cannot make.
2094 configure_common(&conn, self.reader_cache_size).await?;
2095 // The `libsql::Database` is dropped here and the connection outlives
2096 // it, which is the ownership libSQL's own API implies: `connect()`
2097 // returns a `Connection` that does not borrow the builder's handle.
2098 Ok(conn)
2099 }
2100
2101 /// Cross-check the snapshot chain against a fold from genesis (§5.5, T5.3,
2102 /// D-092).
2103 ///
2104 /// `write_final` composes onto the previous snapshot, so snapshot *n* is
2105 /// derived from snapshot *n−1* and nothing in the chain ever folds the whole
2106 /// log. An error at any link propagates forward forever and every read
2107 /// agrees with it, because every read descends from it. This is the check
2108 /// that would notice.
2109 ///
2110 /// # When to run it
2111 ///
2112 /// **Not on a schedule this crate chooses.** A genesis fold is precisely the
2113 /// cost snapshots exist to avoid, so running it periodically by default
2114 /// would give every application the bill snapshots were bought to remove —
2115 /// on a database whose log is large enough for snapshots to matter, which is
2116 /// the only kind where this is worth doing. The plan calls it a scheduling
2117 /// problem and it is the caller's schedule: an idle period, a nightly job,
2118 /// or once per *N* anchors, chosen against a log size this crate cannot see.
2119 ///
2120 /// The cadence is deliberately left alone for the same reason — it runs on a
2121 /// connection shared with nothing and a fold there would compete with
2122 /// interactive reads at a moment nobody chose.
2123 ///
2124 /// # It reports; it does not repair
2125 ///
2126 /// A divergence means the snapshots are a wrong **cache**, not that the
2127 /// ledger is corrupt: [Doctrine VI] makes them disposable, so deleting
2128 /// [`Self::snapshots_dir`] restores correctness and costs only speed.
2129 /// Rewriting the file here would destroy the evidence that composition has a
2130 /// defect, which is the only thing this can tell you that you did not
2131 /// already know.
2132 ///
2133 /// Pair it with the actor counters ([`Self::metrics`], D-079) so a
2134 /// divergence found by a scheduled run is visible beside the write latency
2135 /// of the period that produced it.
2136 ///
2137 /// [Doctrine VI]: ../../docs/architecture/s0-s3-foundations.md#doctrine-vi
2138 pub async fn verify_snapshot_chain(&self, ts: &str) -> Result<crate::temporal::ChainCheck> {
2139 let archive = crate::temporal::archive::archive_present(&self.archive_path)
2140 .then_some(self.archive_path.as_path());
2141 crate::temporal::verify_snapshot_chain(&self.read_conn, ts, archive, &self.snapshots_dir)
2142 .await
2143 }
2144
2145 /// Check the newest link of the snapshot chain (0.15.19, review C-18).
2146 ///
2147 /// The affordable half of [`Self::verify_snapshot_chain`]: re-derive the
2148 /// newest snapshot from the one before it and compare, which is one
2149 /// anchored delta rather than a fold from genesis. `Ok(None)` when there
2150 /// are not two snapshots yet.
2151 ///
2152 /// The snapshot cadence already runs this after every anchor it writes and
2153 /// logs a divergence at `warn`, so a caller reaching for it directly is
2154 /// usually one that wants the [`crate::temporal::ChainCheck`] itself — the
2155 /// disagreeing ids — rather than a yes or no.
2156 ///
2157 /// **It reports; it does not repair.** A snapshot is derivative
2158 /// (Doctrine VI), so the repair is to delete the snapshot directory, which
2159 /// is the caller's call and one line. What this cannot tell you is whether
2160 /// the chain went wrong further back than one link; that is what
2161 /// [`Self::verify_snapshot_chain`] is for, and why it stays.
2162 pub async fn verify_last_link(&self) -> Result<Option<crate::temporal::ChainCheck>> {
2163 let archive = crate::temporal::archive::archive_present(&self.archive_path)
2164 .then_some(self.archive_path.as_path());
2165 crate::temporal::verify_last_link(&self.read_conn, archive, &self.snapshots_dir).await
2166 }
2167
2168 /// The clock every write is stamped with (§5.1.1).
2169 pub fn clock(&self) -> &Arc<dyn Clock> {
2170 &self.clock
2171 }
2172
2173 /// Schema version this handle opened against.
2174 pub fn schema_version(&self) -> u32 {
2175 self.schema_version
2176 }
2177
2178 /// Cold database path, derived by convention from the main file.
2179 pub fn archive_path(&self) -> &Path {
2180 &self.archive_path
2181 }
2182
2183 /// Snapshot directory, derived by convention from the main file.
2184 pub fn snapshots_dir(&self) -> &Path {
2185 &self.snapshots_dir
2186 }
2187
2188 /// What the write actor has done since this handle was opened (T1.4, D-079).
2189 ///
2190 /// Requires the `metrics` feature. The counters are per-handle and start at
2191 /// zero on `open()` — they are not read from the database, because the thing
2192 /// being measured is *this process's* actor and merging two processes'
2193 /// histograms would produce a number about neither.
2194 ///
2195 /// The intended first question is [`crate::metrics::MetricsSnapshot::budget_violations`]:
2196 ///
2197 /// ```no_run
2198 /// # async fn f(db: ¯ame::Database) {
2199 /// # #[cfg(feature = "metrics")] {
2200 /// for k in db.metrics().budget_violations() {
2201 /// eprintln!("{} broke the 3 ms bound {} times", k.kind, k.over_budget);
2202 /// }
2203 /// # }
2204 /// # }
2205 /// ```
2206 ///
2207 /// Reading this does not stop the actor — see
2208 /// [`crate::metrics::ActorMetrics::snapshot`] for what that costs in
2209 /// consistency, and why the trade goes that way.
2210 #[cfg(feature = "metrics")]
2211 pub fn metrics(&self) -> crate::metrics::MetricsSnapshot {
2212 self.shared.metrics.snapshot()
2213 }
2214
2215 /// The underlying libSQL database, for callers that need their own connection.
2216 ///
2217 /// # Actor containment is a convention above this line, not a guarantee
2218 ///
2219 /// **Kept public, and the honest statement of what that costs (Wave 4.3).**
2220 /// §5.1 says the write actor is the sole writer, and two mechanisms make that
2221 /// true of the handle: every write method goes through a channel, and
2222 /// [`Self::read_conn`] carries `PRAGMA query_only = ON`. **Nothing protects a
2223 /// connection obtained from here.** A caller can open one, write to `links`
2224 /// directly, and the actor will not know — the triggers still fire and the
2225 /// ledger stays internally consistent, but the single-writer property that
2226 /// [`crate::CHUNK_BUDGET`]'s latency argument rests on is gone, and so is the
2227 /// serialisation the overlap guard (D-060) relies on.
2228 ///
2229 /// This is the same shape as the limit stated in §4.2 for that guard, and it
2230 /// is one fact rather than two: **the storage layer permits what this API
2231 /// refuses.** Making it private would not change that — the database file is
2232 /// reachable by any SQLite client on the machine — it would only remove the
2233 /// supported way to do the thing, which is how escape hatches become
2234 /// `unsafe`-adjacent folklore.
2235 ///
2236 /// The free functions [`crate::register_model`] and
2237 /// [`crate::upsert_embedding`] take a bare connection for the same reason and
2238 /// carry the same caveat; prefer [`Self::register_model`] and
2239 /// [`Self::upsert_embeddings`], which go through the actor.
2240 ///
2241 /// # The legitimate-use list is now one item long (T5.1, D-091)
2242 ///
2243 /// It used to read: `EXPLAIN QUERY PLAN` and other diagnostics, read-only
2244 /// reporting queries wanting their own connection rather than sharing the
2245 /// reader, and provoking a guard in a test. The first two are exactly what
2246 /// [`Self::diagnostic_conn`] now does, and it does them behind an OS-level
2247 /// read-only open rather than on a handle that can write. **Use that.**
2248 ///
2249 /// What is left is the one use that genuinely requires write access through
2250 /// a connection the actor does not own: *provoking a guard* — writing the
2251 /// state §4.7 says the storage layer permits and this API refuses, so a test
2252 /// can assert the gap is still where the document says it is. That is the
2253 /// only thing this crate's own suite uses it for.
2254 ///
2255 /// # Why `#[doc(hidden)]` and not a `raw-access` feature
2256 ///
2257 /// T5.1 offers either. The feature is the stronger declaration — it shows up
2258 /// in the consumer's `Cargo.toml`, where a reviewer sees it — and it was
2259 /// **not** taken, for a reason specific to what uses this:
2260 ///
2261 /// Cargo features are additive and cannot be *required* by a test target
2262 /// except through `required-features`, which makes a plain `cargo test`
2263 /// **skip** that binary silently. The binaries that call this are
2264 /// `storage_boundary_tests` and `wave1_regression_tests` — the §4.7
2265 /// tripwires, whose entire job is to fail when a documented gap moves. Gating
2266 /// them behind a feature would mean the ordinary `cargo test` stopped running
2267 /// the tests that enforce the section this item is about, to make a
2268 /// declaration about a hatch. That trade is the wrong way round, and it is
2269 /// the same failure the project already names: a suite that quietly does less
2270 /// than it appears to.
2271 ///
2272 /// So the hatch stays reachable and stops being *discoverable*: it is absent
2273 /// from the docs, and the documented path for every non-write use is
2274 /// [`Self::diagnostic_conn`]. [D-068] is unchanged — removing it would buy
2275 /// the appearance of a guarantee, since the file is reachable by any SQLite
2276 /// client on the machine.
2277 ///
2278 /// [D-068]: ../../docs/architecture/s13-decision-register.md#d-068
2279 // convention (D-068/D-091): `raw()` is #[doc(hidden)] and is NOT exposed by
2280 // any binding. Everything above this line is invisible on docs.rs and
2281 // invisible to a contributor reading the Python surface list, which is where
2282 // the decision to expose it would actually be taken — hence this sentinel and
2283 // its twin in `bindings/python/src/lib.rs` (0.10.0, W4.10). The documented
2284 // path for every non-write use is `diagnostic_conn`.
2285 #[doc(hidden)]
2286 pub fn raw(&self) -> &libsql::Database {
2287 &self.db
2288 }
2289
2290 // -- write surface (§5.1, Appendix A) --
2291 //
2292 // Every method here validates and canonicalises before the value crosses the
2293 // channel, so a bad edge type or a second-precision timestamp is a typed
2294 // error at the call site rather than an engine `CHECK` failure surfacing
2295 // from the far side of an actor with no context attached.
2296 //
2297 // NOTE (§5.1.8, D-028): awaiting one of these waits on a Rust channel, not
2298 // in SQLite, so `busy_timeout` does not bound it. During an in-flight
2299 // `rebuild_current` or `archive` the caller stalls for that transaction's
2300 // duration. Wrap in `tokio::time::timeout` if you need a bound — but a
2301 // timeout is not a cancellation: the command stays queued and commits when
2302 // the actor reaches it.
2303
2304 /// Assert an edge (Doctrine III: a new row, never an update).
2305 ///
2306 /// # One row costs a transaction, so N rows cost N transactions
2307 ///
2308 /// This is the correct method for a caller who genuinely has one edge, and
2309 /// it is the wrong one in a loop. Each call is its own transaction and pays
2310 /// the ~0.8 ms per-transaction floor (D-090) whole, so a thousand edges
2311 /// asserted one at a time spend roughly **0.8 s in transaction overhead
2312 /// alone** — before any of the work — and mint a thousand distinct
2313 /// `recorded_at` stamps for what the caller probably means as one act.
2314 ///
2315 /// There are two bulk forms and the difference between them is the one to
2316 /// get right:
2317 ///
2318 /// - [`Self::bulk_import`] is **chunked** against [`CHUNK_BUDGET`] and
2319 /// atomic per chunk. It amortises the transaction floor across the batch
2320 /// while still yielding to interactive work at every chunk boundary. This
2321 /// is the one a loop should almost always become.
2322 /// - [`Self::write_bulk_atomic`] is one transaction under one stamp and is
2323 /// **the one write with no latency bound** — the hold is a function of
2324 /// `edges.len()`, tabulated in its own docs, and is time every other
2325 /// writer spends waiting. Reach for it when the batch is genuinely one
2326 /// act that must not be observable half-applied, not for speed.
2327 ///
2328 /// The choice is the caller's and neither form is deprecated. Doctrine III
2329 /// makes "one act, one stamp" a semantic claim rather than a performance
2330 /// one, and only the caller knows whether their thousand edges are one act.
2331 pub async fn assert_edge(&self, edge: EdgeAssertion) -> Result<()> {
2332 let edge = edge.normalized()?;
2333 self.high(|responder| HighPriCommand::AssertEdge { edge, responder })
2334 .await
2335 }
2336
2337 /// Close an open interval by asserting its replacement (Doctrine III).
2338 pub async fn retire_edge(
2339 &self,
2340 source: impl Into<String>,
2341 target: impl Into<String>,
2342 edge_type: impl Into<String>,
2343 valid_from: &str,
2344 valid_to: &str,
2345 ) -> Result<()> {
2346 let edge_type = edge_type.into();
2347 crate::graph::edge::validate_edge_type(&edge_type)?;
2348 let valid_from = timestamp::normalize(valid_from)?;
2349 let valid_to = timestamp::normalize(valid_to)?;
2350 let (source, target) = (source.into(), target.into());
2351
2352 self.high(|responder| HighPriCommand::RetireEdge {
2353 source,
2354 target,
2355 edge_type,
2356 valid_from,
2357 valid_to,
2358 branch: None,
2359 responder,
2360 })
2361 .await
2362 }
2363
2364 /// Retire an edge **on a lineage**, which is a different write (0.14.8).
2365 ///
2366 /// The `_on` suffix is the crate's established spelling for the
2367 /// branch-taking variant of a call whose trunk form predates branching —
2368 /// [`query_as_of_edges_on`](crate::temporal::query_as_of_edges_on) is the
2369 /// other one. A sixth positional `Option<BranchId>` on
2370 /// [`Self::retire_edge`] would have made every existing call site read as
2371 /// though it had made a lineage decision it never made.
2372 ///
2373 /// # This closes a row; it does not close *the* row
2374 ///
2375 /// Retiring an edge the branch **inherited** writes the branch's own row at
2376 /// the ancestor's key, carrying the closed interval and this lineage's id.
2377 /// The ancestor's row is untouched, and the read prefers the nearer one, so
2378 /// the edge is gone from this lineage's view and unchanged in its parent's.
2379 /// That is **shadow retirement**, and it is the only retirement across
2380 /// lineages that does not commit the parent corruption
2381 /// [Doctrine III](../../docs/architecture/s0-s3-foundations.md#doctrine-iii)
2382 /// forbids — which is not a rule this method obeys but a shape the ledger
2383 /// cannot express: `links` is append-only and no statement in this crate
2384 /// closes a row in place.
2385 ///
2386 /// `weight` and `properties` are carried over from the visible row rather
2387 /// than restated, which is what makes this a retirement rather than a new
2388 /// assertion that happens to be closed.
2389 ///
2390 /// # Errors
2391 ///
2392 /// - [`DbError::UnknownBranch`] when `branch` is not registered.
2393 /// - [`DbError::NotFound`] when this lineage can see no open row at that
2394 /// `valid_from`. On a branch that includes *never inherited it* and
2395 /// *inherited it and already shadowed it*, which are one answer here
2396 /// because they are one answer to the question asked: there is nothing
2397 /// at that key to retire.
2398 pub async fn retire_edge_on(
2399 &self,
2400 source: impl Into<String>,
2401 target: impl Into<String>,
2402 edge_type: impl Into<String>,
2403 valid_from: &str,
2404 valid_to: &str,
2405 branch: crate::branch::BranchId,
2406 ) -> Result<()> {
2407 let edge_type = edge_type.into();
2408 crate::graph::edge::validate_edge_type(&edge_type)?;
2409 let valid_from = timestamp::normalize(valid_from)?;
2410 let valid_to = timestamp::normalize(valid_to)?;
2411 let (source, target) = (source.into(), target.into());
2412
2413 self.high(|responder| HighPriCommand::RetireEdge {
2414 source,
2415 target,
2416 edge_type,
2417 valid_from,
2418 valid_to,
2419 branch: Some(branch),
2420 responder,
2421 })
2422 .await
2423 }
2424
2425 /// Insert or update a concept.
2426 ///
2427 /// # One row costs a transaction
2428 ///
2429 /// The same trade [`Self::assert_edge`] describes, for the same reason and
2430 /// with the same ~0.8 ms floor (D-090): correct for one concept, wrong in a
2431 /// loop. [`Self::write_concepts`] takes a `Vec` and commits it as one
2432 /// transaction under one stamp.
2433 ///
2434 /// There is no atomic-across-chunks concept path and none is needed to make
2435 /// the choice: `write_concepts` is chunked against [`CHUNK_BUDGET`] and
2436 /// atomic per chunk, so a large `Vec` is cooperative rather than a stall.
2437 /// The responsiveness argument for writing one row at a time therefore does
2438 /// not apply — the bulk form already yields at every chunk boundary.
2439 pub async fn upsert_concept(&self, concept: ConceptUpsert) -> Result<()> {
2440 let concept = concept.normalized()?;
2441 self.high(|responder| HighPriCommand::UpsertConcept { concept, responder })
2442 .await
2443 }
2444
2445 /// A handle on one lineage (§15.4, 0.14.9, [D-226]).
2446 ///
2447 /// Takes `&Arc<Self>` rather than `&self` because the view holds the handle
2448 /// and must not be able to end it: `close` takes `self` by value and an
2449 /// `Arc` cannot surrender that while a clone survives, so the restriction
2450 /// is structural rather than documented. Sharing the handle is already
2451 /// `Arc<Database>` (§5.1.11), so this asks for nothing a caller did not
2452 /// have.
2453 ///
2454 /// Does no I/O and cannot fail. Whether the lineage is *registered* is
2455 /// asked by every operation on the view, which is where
2456 /// [`DbError::UnknownBranch`] names it.
2457 ///
2458 /// [D-226]: ../../docs/architecture/s13-decision-register.md#d-226
2459 pub fn view(
2460 self: &std::sync::Arc<Self>,
2461 branch: crate::branch::BranchId,
2462 ) -> crate::branch::BranchView {
2463 crate::branch::BranchView::new(std::sync::Arc::clone(self), branch)
2464 }
2465
2466 /// Cut a new lineage from an existing one (§15.2, §15.4).
2467 ///
2468 /// # A fork is O(1) in rows written
2469 ///
2470 /// One row in `branches`, and nothing else. No ledger table is read, copied
2471 /// or touched: a branch inherits its parent's history by *resolution at
2472 /// read* rather than by owning a copy of it, which is what
2473 /// [`TraversalBuilder::on_branch`](crate::graph::TraversalBuilder::on_branch)
2474 /// resolves and 0.14.6 bounds by the fork point. The cost of that choice is
2475 /// on the read side and is measured — [D-220] for the resolution, [D-223]
2476 /// for the cutoff — and the cost of the alternative would be here, as an
2477 /// O(rows) fork and storage multiplied by branch count (§15.3, option 3).
2478 ///
2479 /// # The fork point is *now*, and that is a bound on this release rather
2480 /// than on the design
2481 ///
2482 /// `forked_at` is stamped from the same clock as every other write, so the
2483 /// new lineage sees its parent's history up to this instant. Forking from a
2484 /// *past* instant is a coherent thing to want and the schema has always
2485 /// allowed it — `branches` carries `forked_at` and `created_at` as separate
2486 /// columns under `CHECK (forked_at <= created_at)` — but it is not in this
2487 /// release and is additive when it is.
2488 ///
2489 /// # What this lineage can do
2490 ///
2491 /// It can be **read**: every traversal entry point takes a branch, and on a
2492 /// forked ledger the read resolves along the ancestry and stops at the fork
2493 /// point. Since 0.14.8 it can also be **written** — [`EdgeAssertion`] and
2494 /// [`ConceptUpsert`] carry a lineage, and [`Self::retire_edge_on`] shadows
2495 /// an inherited edge (D-225). Through 0.14.7 they did not, and a caller who
2496 /// forked and then called `assert_edge` got a successful write **on the
2497 /// trunk**; that is fixed rather than documented now.
2498 ///
2499 /// What a branch still may not do is **restate an inherited concept**.
2500 /// `concepts` is keyed by identity, so that is refused as
2501 /// [`DbError::CrossLineage`] — see [`ConceptUpsert::branch`]. Edges are the
2502 /// thing a lineage may hold its own belief about, and superseding one is a
2503 /// row written *beside* the ancestor's rather than over it.
2504 ///
2505 /// # Errors
2506 ///
2507 /// - [`DbError::UnknownBranch`] when `from` is not registered. Named rather
2508 /// than left to the foreign key, because the caller asked about a branch.
2509 /// - [`DbError::BranchExists`] when `name` is taken — including `"main"`,
2510 /// which every database has from its first migration.
2511 /// - [`DbError::ForkPrecedesParent`] when the clock would place this fork
2512 /// point before the parent's *own* — not before the parent's
2513 /// `created_at`, which is what the schema comment promised until 0.14.7
2514 /// and is not checkable: the trunk's `created_at` is stamped during
2515 /// migration from the wall clock, before an injected clock exists, so
2516 /// that rule refuses every fork on every `FakeClock` database (D-224).
2517 /// Reachable with [`FakeClock`](crate::util::FakeClock), and the one
2518 /// refusal here that no `CHECK` could have made — it is cross-row, and a
2519 /// `CHECK` sees one row.
2520 ///
2521 /// # Example
2522 ///
2523 /// ```no_run
2524 /// # use macrame::prelude::*;
2525 /// # async fn f(db: &Database) -> Result<()> {
2526 /// let alt = db.fork(BranchId::new("turn/17/alt/1")?, BranchId::main()).await?;
2527 /// let seen = TraversalBuilder::new("socrates")
2528 /// .on_branch(alt.id.clone())
2529 /// .execute_ids(db.read_conn(), "2026-08-29T00:00:00.000000Z")
2530 /// .await?;
2531 /// # let _ = seen;
2532 /// # Ok(())
2533 /// # }
2534 /// ```
2535 ///
2536 /// [D-220]: ../../docs/architecture/s13-decision-register.md#d-220
2537 /// [D-223]: ../../docs/architecture/s13-decision-register.md#d-223
2538 pub async fn fork(
2539 &self,
2540 name: crate::branch::BranchId,
2541 from: crate::branch::BranchId,
2542 ) -> Result<crate::branch::Branch> {
2543 self.high(|responder| HighPriCommand::Fork {
2544 name,
2545 parent: from,
2546 responder,
2547 })
2548 .await
2549 }
2550
2551 /// Every lineage the ledger knows about, trunk first (§15.4).
2552 ///
2553 /// Read through [`Self::read_conn`] rather than the write actor, which is
2554 /// the difference between this and [`Self::fork`] and is deliberate:
2555 /// `branches` is append-only, so the only way this listing can be stale is
2556 /// by missing a branch created after it was taken, and a caller who wanted
2557 /// to know about that branch would have had to create it. Queueing a read
2558 /// behind the write actor would make listing branches wait on a bulk import
2559 /// for no answer it could change.
2560 ///
2561 /// A database that has never forked returns exactly one row: the trunk,
2562 /// with no parent and no fork point.
2563 pub async fn branches(&self) -> Result<Vec<crate::branch::Branch>> {
2564 crate::branch::list(self.read_conn()).await
2565 }
2566
2567 /// The beliefs `a` holds that `b` does not (§15.4, 0.14.11, D-228).
2568 ///
2569 /// One [`Divergence`](crate::branch::Divergence) per edge key the two
2570 /// lineages disagree about, in key order: `b` holds no belief about it, or
2571 /// holds one with a different interval or weight. Not symmetric —
2572 /// `diff(b, a)` is the other half, and composing the two is *two* snapshots
2573 /// even though each is one.
2574 ///
2575 /// Read through the read connection rather than the actor, like
2576 /// [`Self::branches`], and taken at one snapshot rather than two: see
2577 /// `graph::lineage::diff_sql` for why that decides the shape of the query.
2578 ///
2579 /// There is no instant parameter. A diff filtered to a valid-time instant
2580 /// cannot report the one divergence that is *about* an instant having
2581 /// passed — a branch that retired an edge its parent still holds open — so
2582 /// this compares the whole of both views.
2583 ///
2584 /// # Errors
2585 ///
2586 /// [`DbError::UnknownBranch`], naming whichever of the two is not
2587 /// registered, and `a` first when neither is.
2588 pub async fn diff(
2589 &self,
2590 a: &crate::branch::BranchId,
2591 b: &crate::branch::BranchId,
2592 ) -> Result<Vec<crate::branch::Divergence>> {
2593 crate::branch::diff(self.read_conn(), a, b).await
2594 }
2595
2596 /// Assert many edges in one transaction under one stamp (D-014).
2597 ///
2598 /// # This is the one write with no latency bound, and here is what it costs
2599 ///
2600 /// The batch is one act under one `recorded_at`, so it cannot be chunked —
2601 /// splitting it is the thing this method exists not to do. That makes the
2602 /// actor's hold a function of `edges.len()`, and until now the only
2603 /// statement of that anywhere was the prose "uncapped" in
2604 /// [`CHUNK_BUDGET`]'s table. A caller who stalls every other writer for
2605 /// eight seconds should have been able to predict it from the signature.
2606 ///
2607 /// Measured on libSQL 0.9.30 (T1.3, D-081), holding the actor for:
2608 ///
2609 /// | rows | hold |
2610 /// |---|---|
2611 /// | 500 | ~34 ms |
2612 /// | 2,000 | ~155 ms |
2613 /// | 10,000 | ~1.0 s |
2614 /// | 20,000 | ~2.6 s |
2615 ///
2616 /// [`estimated_bulk_hold`] is that curve as a function, and this method
2617 /// emits a `tracing::warn!` when it predicts more than
2618 /// [`BULK_ATOMIC_WARN_HOLD`]. **The estimate is a shape, not a promise** —
2619 /// see [`estimated_bulk_hold`] for what it is calibrated against and where
2620 /// it will be wrong.
2621 ///
2622 /// A caller who needs the latency bound and not the atomicity wants
2623 /// [`Self::bulk_import`], which is the same write chunked and explicitly not
2624 /// atomic overall (D-011).
2625 pub async fn write_bulk_atomic(&self, edges: Vec<EdgeAssertion>) -> Result<usize> {
2626 let estimate = estimated_bulk_hold(&edges);
2627 if estimate > BULK_ATOMIC_WARN_HOLD {
2628 // Warned here rather than in the actor, and before the send: this is
2629 // the caller's own task, so the log line lands with their span
2630 // attached and names the call site that chose the batch size. By the
2631 // time the actor has it, the only context left is "a large batch".
2632 tracing::warn!(
2633 rows = edges.len(),
2634 estimated_hold_ms = estimate.as_millis() as u64,
2635 "write_bulk_atomic will hold the write actor for roughly \
2636 {estimate:?} — it is atomic by contract (D-014) and cannot be \
2637 chunked. Every other writer waits that long. Use bulk_import \
2638 if the batch does not need to be all-or-nothing."
2639 );
2640 }
2641
2642 let edges = normalize_all(edges)?;
2643 self.high(|responder| HighPriCommand::WriteBulkAtomic { edges, responder })
2644 .await
2645 }
2646
2647 /// Move the WAL back into the main database file (§4.5, F-30, 0.12.13,
2648 /// W5.2, D-156).
2649 ///
2650 /// Runs `PRAGMA wal_checkpoint(FULL)` and then `(TRUNCATE)` on the write
2651 /// connection, as one actor turn, and returns what SQLite reported. **Read
2652 /// [`CheckpointReport::busy`]** — a checkpoint that could not run is an
2653 /// `Ok` whose WAL is still there.
2654 ///
2655 /// Two passes rather than one because **a truncating checkpoint cannot
2656 /// report its own work**: the counts describe the WAL *after* the
2657 /// operation, and after a truncation there is nothing left to describe, so
2658 /// `TRUNCATE` alone answers `busy=0, log=0, checkpointed=0` on success —
2659 /// indistinguishable from having done nothing. `FULL` supplies the frame
2660 /// count and `TRUNCATE` resets the file; `busy` is the union of the two.
2661 ///
2662 /// # When a caller needs this
2663 ///
2664 /// Three cases, and only three:
2665 ///
2666 /// - **Before copying the database file elsewhere.** In WAL mode the `.db`
2667 /// file alone is not the database; recent commits live in the `-wal`. A
2668 /// complete checkpoint is what makes the main file self-contained.
2669 /// - **At the end of a bulk load that turned the automatic checkpointer
2670 /// off.** That is the pairing this method exists for — see
2671 /// [`Tuning::wal_autocheckpoint`]. Disabling autocheckpoint without
2672 /// calling this leaves a WAL that grows for the life of the process.
2673 /// - **Before a long idle period**, to give back the disk.
2674 ///
2675 /// Nobody else should call it on a timer. SQLite checkpoints automatically
2676 /// every 1,000 pages and that default is not changed by this method
2677 /// existing; a periodic explicit checkpoint on top of it buys nothing and
2678 /// takes the write lock to do so.
2679 ///
2680 /// # It takes the write lock, and it is budget-exempt
2681 ///
2682 /// The hold is a function of how many frames have accumulated, which is a
2683 /// function of how long since the last checkpoint — not of anything passed
2684 /// in. It is on [`CHUNK_BUDGET`]'s exemption table for that reason, and it
2685 /// is the one entry there that is not a transaction: there is no smaller
2686 /// unit to chunk into, because the operation *is* the copy.
2687 pub async fn checkpoint(&self) -> Result<CheckpointReport> {
2688 self.high(|responder| HighPriCommand::Checkpoint { responder })
2689 .await
2690 }
2691
2692 /// Rebuild `links_current` from `links` and verify zero drift (§5.8).
2693 ///
2694 /// One transaction holding the write lock for its whole duration, because
2695 /// [D-023] will not let the `DELETE` and the `INSERT` be split: a reader
2696 /// landing between them would see a graph with no edges and no error.
2697 /// [`Self::rebuild_current_chunked`] is the same result with a different
2698 /// latency profile, and is what a populated database wants.
2699 ///
2700 /// The report's `drift_after` is the audit run inside the same transaction,
2701 /// so a repair that did not converge is reported by the call that made it
2702 /// rather than by the next one to look.
2703 ///
2704 /// [D-023]: ../docs/architecture/s13-decision-register.md#d-023
2705 pub async fn rebuild_current(&self) -> Result<RebuildReport> {
2706 self.high(|responder| HighPriCommand::RebuildCurrent { responder })
2707 .await
2708 }
2709
2710 /// Rebuild `links_current` beside itself, in chunks (§5.8, T1.2, D-082).
2711 ///
2712 /// Same result as [`Self::rebuild_current`], different latency profile.
2713 /// `rebuild_current` is one transaction holding the write lock for its whole
2714 /// duration, because D-023 will not let the `DELETE` and the `INSERT` be
2715 /// split: a reader landing between them sees a graph with no edges and no
2716 /// error. This builds the replacement in a shadow table instead — the live
2717 /// table stays live and trigger-maintained throughout — and swaps it in at
2718 /// the end.
2719 ///
2720 /// Each step is its own actor turn, so an interactive assertion can jump the
2721 /// queue between chunks. That is the whole of the improvement, and it is why
2722 /// the loop is here rather than inside the actor's arm (the same reasoning
2723 /// as [`Self::archive_windowed`] and [`Self::bulk_import`]).
2724 ///
2725 /// # What the swap still costs
2726 ///
2727 /// Not microseconds. Index names are global and SQLite has no `ALTER INDEX
2728 /// … RENAME`, so the shadow cannot be built carrying `links_current`'s index
2729 /// names while `links_current` still holds them — and building it under
2730 /// other names would leave the table permanently indexed under names absent
2731 /// from [`CREATE_INDICES`](crate::schema::ddl::CREATE_INDICES), so the next
2732 /// migration would create a second copy of each.
2733 /// `DROP TABLE` frees the names, so the swap transaction is where
2734 /// the three indexes get built. What the chunking moves off the lock is the
2735 /// **projection** — the window function over all of `links` — which is the
2736 /// O(E log E) term.
2737 ///
2738 /// # When this returns an error rather than a repair
2739 ///
2740 /// [`DbError::RebuildInterrupted`] means an archive committed while the
2741 /// shadow was being built. Its deletions are invisible to a catch-up pass
2742 /// keyed on `recorded_at` — a deleted row has no `recorded_at` left to find
2743 /// it by — so the work is discarded rather than swapped in. `links_current`
2744 /// is untouched and the call can simply be retried.
2745 ///
2746 /// Use [`Self::rebuild_current`] when the repair must be one atomic act, or
2747 /// when nothing else is contending for the actor and the extra turns are
2748 /// pure overhead.
2749 pub async fn rebuild_current_chunked(&self) -> Result<RebuildReport> {
2750 use crate::integrity::{ShadowOutcome, ShadowStep};
2751
2752 // Each `else` arm is unreachable: the actor maps each step to its own
2753 // outcome variant. Written as a refutable pattern rather than an
2754 // `unwrap` so that adding a step cannot turn a mismatch into a panic on
2755 // the write path — and `WriterDroppedResponder` is the honest name for
2756 // "the actor answered with something this cannot use".
2757 let ShadowOutcome::Started { build_start, epoch } =
2758 self.shadow_step(ShadowStep::Begin).await?
2759 else {
2760 return Err(DbError::WriterDroppedResponder);
2761 };
2762
2763 let mut after: Option<String> = None;
2764 loop {
2765 let ShadowOutcome::Filled { last } = self
2766 .shadow_step(ShadowStep::Fill {
2767 after: after.take(),
2768 })
2769 .await?
2770 else {
2771 return Err(DbError::WriterDroppedResponder);
2772 };
2773 match last {
2774 Some(last) => after = Some(last),
2775 None => break,
2776 }
2777 }
2778
2779 let ShadowOutcome::Swapped { rows } = self
2780 .shadow_step(ShadowStep::Swap { build_start, epoch })
2781 .await?
2782 else {
2783 return Err(DbError::WriterDroppedResponder);
2784 };
2785
2786 Ok(RebuildReport {
2787 rows_rebuilt: rows,
2788 // Not audited. The chunked path's whole argument is that the
2789 // expensive work happens off the lock, and `audit_current` is two
2790 // `EXCEPT` passes over the projection — the cost D-077 removed from
2791 // the archive for the same reason. A caller who wants the check has
2792 // `audit_current` on the read connection, where it costs nobody the
2793 // write lock.
2794 drift_after: 0,
2795 })
2796 }
2797
2798 /// Run one step of a chunked rebuild, for a caller doing its own scheduling.
2799 ///
2800 /// [`Self::rebuild_current_chunked`] is this in a loop and is what almost
2801 /// everyone wants. This exists because that loop offers no seam: it drives
2802 /// `Begin`, then `Fill` to exhaustion, then `Swap`, and a caller who needs to
2803 /// do something *between* steps — pace them against a frame budget, abandon
2804 /// a rebuild that has run long enough, or provoke the archive interlock in a
2805 /// test — cannot get in.
2806 ///
2807 /// The obligation that comes with it: `epoch` from
2808 /// [`ShadowOutcome::Started`](crate::integrity::ShadowOutcome) must be handed
2809 /// back to [`ShadowStep::Swap`](crate::integrity::ShadowStep), or the
2810 /// archive interlock is defeated and a stale projection can be swapped in.
2811 /// The looping version cannot get that wrong; this one can.
2812 pub async fn shadow_step(
2813 &self,
2814 step: crate::integrity::ShadowStep,
2815 ) -> Result<crate::integrity::ShadowOutcome> {
2816 self.low(|responder| LowPriCommand::ShadowRebuild { step, responder })
2817 .await
2818 }
2819
2820 /// Import edges on the background channel, chunked (D-011).
2821 ///
2822 /// Atomic *per chunk*, not overall: a failure partway leaves earlier chunks
2823 /// committed. That is the tradeoff [`chunk_rows`] documents — use
2824 /// [`Database::write_bulk_atomic`] when the batch must be all-or-nothing.
2825 ///
2826 /// Chunked adaptively, at most [`chunk_rows::EDGES`] rows at a time: that
2827 /// constant is where the loop starts and the largest chunk it will send, and
2828 /// each chunk's measured hold sizes the next against [`CHUNK_BUDGET`]. It is
2829 /// also faster in total than the larger chunks this used through 0.5.5
2830 /// (D-058).
2831 ///
2832 /// A consequence worth planning for: the chunk boundaries — and so the
2833 /// `recorded_at` stamps this import writes — depend on how fast the machine
2834 /// was, not only on how many edges were passed (§5.1.6).
2835 ///
2836 /// Returns [`BulkInterrupted`] rather than [`DbError`] on failure, because
2837 /// a path that is not all-or-nothing owes its caller the count of what
2838 /// landed (0.13.8, W7.6). `?` into a `Result<_, DbError>` still compiles
2839 /// and drops the count, which is the caller's decision to take.
2840 ///
2841 /// [`Self::bulk_import_with`] adds cancellation and per-chunk progress.
2842 pub async fn bulk_import(&self, edges: Vec<EdgeAssertion>) -> BulkResult<usize> {
2843 self.bulk_import_with(edges, BulkControl::new()).await
2844 }
2845
2846 /// [`Self::bulk_import`] with cancellation and progress (0.13.8, W7.6,
2847 /// D-181).
2848 ///
2849 /// The chunk boundaries this path already has are what make both possible:
2850 /// the loop is between transactions several times a second, which is where
2851 /// a token can be read and a callback run without holding anything.
2852 pub async fn bulk_import_with(
2853 &self,
2854 edges: Vec<EdgeAssertion>,
2855 control: BulkControl,
2856 ) -> BulkResult<usize> {
2857 let edges = normalize_all(edges).map_err(before_any_chunk)?;
2858 self.low_chunked(edges, chunk_rows::EDGES, control, |chunk, responder| {
2859 LowPriCommand::BulkImportChunk { chunk, responder }
2860 })
2861 .await
2862 }
2863
2864 /// Upsert many **concepts** on the background channel, chunked (D-011).
2865 ///
2866 /// This is the bulk concept path, and every row it writes is a ledger write:
2867 /// it versions the concept and lands in `transaction_log`. Derived analytics
2868 /// output does not belong here — see
2869 /// [`Database::write_analytics_annotations`] and D-041.
2870 ///
2871 /// Called `write_annotations` through 0.5.6, from when the two writes were
2872 /// one call. D-041 split them and the name stayed on the wrong one for three
2873 /// releases, so the crate had a `write_annotations` that wrote concepts
2874 /// sitting beside a `write_analytics_annotations` that wrote annotations
2875 /// (D-075).
2876 ///
2877 /// Chunked, so it returns [`BulkInterrupted`] and its `written` count on
2878 /// failure (0.13.8, W7.6); [`Self::write_concepts_with`] adds cancellation
2879 /// and progress.
2880 pub async fn write_concepts(&self, concepts: Vec<ConceptUpsert>) -> BulkResult<usize> {
2881 self.write_concepts_with(concepts, BulkControl::new()).await
2882 }
2883
2884 /// [`Self::write_concepts`] with cancellation and progress (0.13.8, W7.6).
2885 pub async fn write_concepts_with(
2886 &self,
2887 concepts: Vec<ConceptUpsert>,
2888 control: BulkControl,
2889 ) -> BulkResult<usize> {
2890 let concepts: Vec<ConceptUpsert> = concepts
2891 .into_iter()
2892 .map(ConceptUpsert::normalized)
2893 .collect::<Result<_>>()
2894 .map_err(before_any_chunk)?;
2895 self.low_chunked(
2896 concepts,
2897 chunk_rows::CONCEPTS,
2898 control,
2899 |chunk, responder| LowPriCommand::WriteConceptsChunk { chunk, responder },
2900 )
2901 .await
2902 }
2903
2904 /// State as believed at `ts` (§5.5, D-026, D-049).
2905 ///
2906 /// A read: it runs on `read_conn` and never touches the Write Actor, so a
2907 /// reconstruction and a full-speed write-back do not slow each other.
2908 ///
2909 /// Prefer this to calling [`crate::temporal::reconstruct`] directly. The
2910 /// free function takes the archive path and the snapshot directory as
2911 /// arguments, and a caller who passes `None` for the second gets a correct
2912 /// answer that folds the whole log every time — the composition is opt-in
2913 /// at that layer and easy to leave off by accident. Here both come from the
2914 /// handle, so the fast path is the default one.
2915 pub async fn reconstruct(&self, ts: &str) -> Result<crate::temporal::MaterializedState> {
2916 let ts = timestamp::normalize(ts)?;
2917 crate::temporal::reconstruct(
2918 &self.read_conn,
2919 &ts,
2920 Some(&self.archive_path),
2921 Some(&self.snapshots_dir),
2922 )
2923 .await
2924 }
2925
2926 /// State at `ts` as `branch` saw it (0.15.17, [D-259], review C-10).
2927 ///
2928 /// [`Self::reconstruct`] with the ancestry resolved: each ancestor bounded
2929 /// at its fork point, one belief per edge key from the nearest lineage
2930 /// holding it. See [`crate::temporal::reconstruct_on`] for how it is
2931 /// assembled, what it costs, and the two things it does **not** do —
2932 /// concepts are not resolved by lineage, and the result must not be saved
2933 /// as a snapshot.
2934 ///
2935 /// A read, on `read_conn`, like [`Self::reconstruct`]. The archive path and
2936 /// the snapshot directory come from the handle, so snapshot composition is
2937 /// on by default for each of the folds this runs.
2938 ///
2939 /// [D-259]: ../../docs/architecture/s13-decision-register.md#d-259
2940 pub async fn reconstruct_on(
2941 &self,
2942 ts: &str,
2943 branch: &str,
2944 ) -> Result<crate::temporal::MaterializedState> {
2945 let ts = timestamp::normalize(ts)?;
2946 crate::temporal::reconstruct_on(
2947 &self.read_conn,
2948 &ts,
2949 branch,
2950 Some(&self.archive_path),
2951 Some(&self.snapshots_dir),
2952 )
2953 .await
2954 }
2955
2956 /// `branch`'s ancestry, nearest first, each with its fork-point cutoff.
2957 ///
2958 /// The input [`crate::temporal::resolve_beliefs`] takes. Resolved from
2959 /// `branches` in Rust since 0.15.17 ([D-259]) — the walk is a few
2960 /// microseconds and the table is tiny and append-only, so this is a read
2961 /// like any other rather than something to cache.
2962 ///
2963 /// The trunk of an unforked database answers with one row and no cutoff,
2964 /// which is its true ancestry. A lineage that is not registered is refused
2965 /// by name with [`DbError::UnknownBranch`].
2966 ///
2967 /// [D-259]: ../../docs/architecture/s13-decision-register.md#d-259
2968 pub async fn ancestry(&self, branch: &str) -> Result<Vec<crate::branch::Ancestor>> {
2969 let lineages = crate::graph::lineage::Lineages::load(&self.read_conn).await?;
2970 // Checked before it is walked: `resolve` answers for a name it has never
2971 // seen with a one-row ancestry, which is the right answer for a root and
2972 // the D-069 wrong-looking-right answer for a typo.
2973 lineages.shape(branch)?;
2974 Ok(lineages.ancestry(branch))
2975 }
2976
2977 /// Every edge one [`ReadPlan`] names (0.15.9, W13.4, [D-251]).
2978 ///
2979 /// The whole projection filtered to the plan's instants and lineage —
2980 /// topology only, no start node, and no budget on the answer. On a large
2981 /// ledger that is a large `Vec`; [`Self::load_subgraph`] is the bounded
2982 /// neighbourhood read and [`crate::graph::TraversalBuilder`] is the
2983 /// anchored one.
2984 ///
2985 /// # What this can express that nothing else could
2986 ///
2987 /// [`crate::temporal::query_as_of_edges_on`] is the same read at a
2988 /// valid-time instant, and it takes no transaction-time one: before this
2989 /// release, *"which edges did we believe existed, as of March, as they
2990 /// stood in January"* had exactly two answers available — walk it from a
2991 /// start node, or fold the entire log with [`Self::reconstruct`] and filter
2992 /// the result. The first needs an anchor the question does not have and the
2993 /// second is a different order of work. The fold this uses is the
2994 /// traversal's own, so the bitemporal cell is now readable whole at the
2995 /// cost of reading it.
2996 ///
2997 /// The two functions share one statement, which is why neither can drift
2998 /// from the other; `query_as_of_edges_on` is this with `recorded` unset and
2999 /// the lineage dropped from each row.
3000 ///
3001 /// # Errors
3002 ///
3003 /// [`DbError::UnknownBranch`] naming a
3004 /// lineage that was never registered — refused rather than answered for the
3005 /// trunk, for `graph::lineage::Lineages::shape`'s reason.
3006 /// [`DbError::RecordedInstantUnreachable`]
3007 /// when [`ReadPlan::recorded`] is below what the hot log still covers
3008 /// ([D-247](../../docs/architecture/s13-decision-register.md#d-247)).
3009 /// [`DbError::InvalidTimestamp`] for a
3010 /// stamp that is not canonical, from the same normaliser every other read
3011 /// uses — a plan is inert and validates nothing, so this is where a
3012 /// malformed instant is noticed.
3013 ///
3014 /// [D-251]: ../../docs/architecture/s13-decision-register.md#d-251
3015 pub async fn edges(&self, plan: ReadPlan) -> Result<Vec<crate::temporal::EdgeBelief>> {
3016 // `None` is now, and now is this handle's clock rather than the
3017 // system's: a database opened on a `FakeClock` reads at the instant it
3018 // is writing at, which is the whole reason the clock is a handle
3019 // property (§5.1.1).
3020 let valid = match plan.valid.as_deref() {
3021 Some(ts) => timestamp::normalize(ts)?,
3022 None => self.clock.now(),
3023 };
3024 let recorded = plan
3025 .recorded
3026 .as_deref()
3027 .map(timestamp::normalize)
3028 .transpose()?;
3029 crate::plan::edges_at(
3030 &self.read_conn,
3031 &valid,
3032 recorded.as_deref(),
3033 plan.branch_name(),
3034 plan.limit,
3035 )
3036 .await
3037 }
3038
3039 /// Create a model's embedding table and DiskANN index (§5.9, D-048).
3040 ///
3041 /// Idempotent: registering a model that already exists at the same
3042 /// dimension succeeds, and at a different dimension fails with
3043 /// [`DbError::DimMismatch`] naming both, rather than no-opping through
3044 /// `IF NOT EXISTS` and leaving the caller believing the dimension they
3045 /// asked for is the one in force.
3046 ///
3047 /// This issues DDL, which everywhere else in the crate is the migration
3048 /// runner's exclusive business (D-032). The exception is bounded and
3049 /// deliberate: a model's table is created once, by an explicit call, and
3050 /// the alternative — a caller-supplied write connection — is the very thing
3051 /// the Write Actor exists to make impossible.
3052 ///
3053 /// # Latency
3054 ///
3055 /// One small transaction, but it queues like any other write: see §5.1.8.
3056 pub async fn register_model(&self, model: &ModelName, dim: usize) -> Result<()> {
3057 let model = model.clone();
3058 self.high(|responder| HighPriCommand::RegisterModel {
3059 model,
3060 dim,
3061 responder,
3062 })
3063 .await
3064 }
3065
3066 /// Store or replace vectors for `model`, chunked (§5.9, D-011, D-048).
3067 ///
3068 /// The write path for embeddings. Before 0.5.4 there was none:
3069 /// [`crate::vector::upsert_embedding`] takes a raw connection, `read_conn`
3070 /// is `query_only`, and the write connection lives inside the actor — so an
3071 /// application could search vectors it had no way to store.
3072 ///
3073 /// Low priority and chunked at [`chunk_rows::EMBEDDINGS`], because embedding
3074 /// is bulk derived work: a 50,000-vector backfill must yield to an
3075 /// interactive assertion at every chunk boundary. That constant is the
3076 /// smallest of the four by a wide margin — DiskANN index maintenance makes an
3077 /// embedding the most expensive row in the system (D-058). Atomic per chunk, not overall, which
3078 /// is the same trade [`Database::bulk_import`] makes and is safer here than
3079 /// there — an embedding is derived (Doctrine VII), so a partially written
3080 /// batch is recoverable by re-embedding.
3081 ///
3082 /// Fails with [`DbError::ModelNotRegistered`] if `model` has no table, and
3083 /// [`DbError::DimMismatch`] if a vector's length is not the declared
3084 /// dimension. The dimension is read from the schema once per chunk (D-037):
3085 /// the crate keeps no registry of its own to fall out of date.
3086 ///
3087 /// Chunked, so it returns [`BulkInterrupted`] and its `written` count on
3088 /// failure (0.13.8, W7.6). A 50,000-vector backfill is the longest-running
3089 /// write the crate has, which makes it the one most likely to be cancelled
3090 /// — [`Self::upsert_embeddings_with`] is how.
3091 pub async fn upsert_embeddings(
3092 &self,
3093 model: &ModelName,
3094 rows: Vec<(String, Vec<f32>)>,
3095 ) -> BulkResult<usize> {
3096 self.upsert_embeddings_with(model, rows, BulkControl::new())
3097 .await
3098 }
3099
3100 /// [`Self::upsert_embeddings`] with cancellation and progress (0.13.8,
3101 /// W7.6).
3102 pub async fn upsert_embeddings_with(
3103 &self,
3104 model: &ModelName,
3105 rows: Vec<(String, Vec<f32>)>,
3106 control: BulkControl,
3107 ) -> BulkResult<usize> {
3108 self.low_chunked(rows, chunk_rows::EMBEDDINGS, control, |chunk, responder| {
3109 LowPriCommand::UpsertEmbeddingChunk {
3110 model: model.clone(),
3111 chunk,
3112 responder,
3113 }
3114 })
3115 .await
3116 }
3117
3118 /// Reconstruct the concept-text search index from the ledger (§5.9, D-036).
3119 ///
3120 /// The FTS index is derivative: D-036 promises every derivative table can be
3121 /// rebuilt from the ledger tables, and this is that promise made callable
3122 /// for `concepts_fts`. Needed after a restore that skipped the shadow
3123 /// tables, or if the index is ever suspected of drifting from the text —
3124 /// and, as a matter of policy, cheaper to run than to reason about.
3125 ///
3126 /// The work is `INSERT INTO concepts_fts(concepts_fts) VALUES('rebuild')`,
3127 /// which is FTS5's own operation over the content table, so this is not a
3128 /// second implementation of the sync triggers that could disagree with them.
3129 pub async fn rebuild_fts(&self) -> Result<()> {
3130 self.low(|responder| LowPriCommand::RebuildFts { responder })
3131 .await
3132 }
3133
3134 /// Refresh the query planner's statistics (0.12.4, [D-149]).
3135 ///
3136 /// Runs `ANALYZE`, which writes `sqlite_stat1`. **Before 0.12.4 nothing in
3137 /// this crate ever did**, so the planner costed every query against SQLite's
3138 /// built-in defaults — assume ~1M rows, assume each bound equality column
3139 /// divides by ten. That estimate is structural: it depends on how many
3140 /// columns a query binds, not on what the table contains.
3141 ///
3142 /// Which is this schema's own worst defect restated. D-042, D-059 and D-064
3143 /// are three occasions where *a covering index captured a query because it
3144 /// contained the columns, not because it discriminated*, and two of the four
3145 /// declared indices lead on the same column. Statistics are what let the
3146 /// planner tell them apart by measurement instead of by shape.
3147 ///
3148 /// # Cost, and why it is bounded
3149 ///
3150 /// This is a write and it takes the write lock. `PRAGMA analysis_limit`
3151 /// (set per connection, see [`ddl::ANALYSIS_LIMIT`]) caps the rows examined
3152 /// per index. It is scheduled as low-priority work and will not preempt an
3153 /// interactive assertion.
3154 ///
3155 /// **The bound is a constant factor, not an independence** (0.12.23,
3156 /// D-166). This rustdoc said the hold "scales with the number of indices —
3157 /// four — and not with the size of `links_current`", which is measurably
3158 /// wrong: the pragma is worth 3–4× and what remains still grows with the
3159 /// table. Measured, `examples/analyze_hold.rs`: **5.26 ms at 10,000 edges,
3160 /// 19.1 ms at 40,000**, against a 3 ms [`crate::CHUNK_BUDGET`].
3161 ///
3162 /// So this call **misses the budget by ~6× on a moderately sized ledger**,
3163 /// and [`crate::metrics::CommandKind::Analyze`] is deliberately not among
3164 /// the budget-exempt kinds — `metrics().budget_violations()` names it. That
3165 /// is the honest position: the work is low priority and preemptible between
3166 /// commands, but it is one indivisible statement and cannot be chunked, so
3167 /// the hold is what it is. Prefer [`optimize`], which does nothing when
3168 /// nothing has moved.
3169 ///
3170 /// **Since 0.13.24 the counter is this call and not also [`optimize`]**
3171 /// (W10.5, [D-197]). The two shared `CommandKind::Analyze` until then, which
3172 /// is why an `analyze` row in `budget_violations()` used to be unreadable:
3173 /// it could have been an explicit call or a handle close.
3174 ///
3175 /// # When to call it
3176 ///
3177 /// After a bulk import, and after anything that changes a table's shape by
3178 /// an order of magnitude. Prefer [`optimize`] for routine upkeep: it does
3179 /// nothing when nothing has moved, and this does the work unconditionally.
3180 ///
3181 /// Statistics are derived state in the sense Doctrine VI means it — deleting
3182 /// `sqlite_stat1` costs plan quality and no information, and this call
3183 /// rebuilds it.
3184 ///
3185 /// [D-149]: ../docs/architecture/s13-decision-register.md#d-149
3186 /// [`ddl::ANALYSIS_LIMIT`]: crate::schema::ddl::ANALYSIS_LIMIT
3187 /// [`optimize`]: Database::optimize
3188 pub async fn analyze(&self) -> Result<()> {
3189 self.low(|responder| LowPriCommand::Analyze {
3190 incremental: false,
3191 responder,
3192 })
3193 .await
3194 }
3195
3196 /// Re-analyse only what has gone stale (0.12.4, [D-149]).
3197 ///
3198 /// `PRAGMA optimize`. SQLite tracks how far each table has drifted since its
3199 /// last analysis and re-analyses only where it believes the statistics no
3200 /// longer hold — so this is a no-op on an idle database and the full cost of
3201 /// [`analyze`] on one that has changed completely.
3202 ///
3203 /// That property is the whole point: it is safe to call on a schedule, where
3204 /// [`analyze`] is not. `close()` runs it, so a process that opens, works and
3205 /// closes keeps its statistics current without anybody arranging it.
3206 ///
3207 /// # What it costs, measured, and the threshold it applies rather than takes
3208 /// (0.13.24, W10.5, [D-197])
3209 ///
3210 /// `examples/optimize_hold.rs`, on a 40,000-edge ledger: **10.7 ms the
3211 /// first time on a database that has never been analysed** — there is
3212 /// nothing incremental about the first call — and **90–220 µs every time
3213 /// after**, well inside [`crate::CHUNK_BUDGET`].
3214 ///
3215 /// **The staleness test is SQLite's and it is a ratio, not a row count.**
3216 /// Measured by reading `sqlite_stat1` across the call rather than by timing
3217 /// it: growth of 2× and 5× both left the statistics **untouched**, and
3218 /// only at 25× did it re-analyse — for a 460 ms hold. So this is not a
3219 /// cheaper `analyze()` and calling it after a bulk load is not a way to
3220 /// refresh statistics the load invalidated: below the ratio it declines,
3221 /// and above it it costs what [`analyze`] costs. It reports as
3222 /// [`crate::metrics::CommandKind::Optimize`] since 0.13.24, which is what
3223 /// makes those two outcomes distinguishable in the metrics at all.
3224 ///
3225 /// [D-197]: ../docs/architecture/s13-decision-register.md#d-197
3226 ///
3227 /// [D-149]: ../docs/architecture/s13-decision-register.md#d-149
3228 /// [`analyze`]: Database::analyze
3229 pub async fn optimize(&self) -> Result<()> {
3230 self.low(|responder| LowPriCommand::Analyze {
3231 incremental: true,
3232 responder,
3233 })
3234 .await
3235 }
3236
3237 // **There is deliberately no `verify_fts()` (§5.9, D-071).**
3238 //
3239 // `rebuild_fts` is the repair with no way to ask whether it is needed, and
3240 // Wave 5 set out to add the missing half. FTS5 offers `'integrity-check'`,
3241 // which looked like exactly the engine-provided answer this crate prefers.
3242 // It is not: on libSQL 0.9.30 it verifies the index's *internal* consistency
3243 // and not its agreement with the content table. Measured — after
3244 // `'delete-all'` the index matches nothing where it matched ten rows, and
3245 // both `'integrity-check'` and `'integrity-check', 0` still report success.
3246 //
3247 // A `verify_fts()` on that footing would answer "healthy" for an empty
3248 // index, which is worse than having no method at all: it is the shape of
3249 // defect AC, a function that looks like it checks something and does not.
3250 // `an_emptied_fts_index_still_passes_integrity_check` pins the limitation so
3251 // that if a later libSQL fixes it, the test fails and says so.
3252
3253 /// Write derived analytics results on the background channel, chunked
3254 /// (§5.4, D-041).
3255 ///
3256 /// Rows go to `analytics_annotations`, which has no log trigger, so nothing
3257 /// written here reaches `transaction_log` and nothing here versions a
3258 /// concept. Rerunning an algorithm replaces the previous pass rather than
3259 /// recording that the world changed.
3260 ///
3261 /// Low priority and chunked at up to [`chunk_rows::ANNOTATIONS`] — the
3262 /// largest ceiling of the four, because this is the only bulk table carrying
3263 /// no triggers at all
3264 /// and its rows are correspondingly cheap (D-058) — so a 50,000-label Louvain
3265 /// save yields to interactive writes at every chunk boundary and carries the
3266 /// per-chunk fidelity boundary of §5.1.6 — a partially written pass is
3267 /// recoverable by rerunning, which is the property that makes derived state
3268 /// safe to write this way and assertions not.
3269 ///
3270 /// Chunked, so it returns [`BulkInterrupted`] and its `written` count on
3271 /// failure (0.13.8, W7.6); [`Self::write_analytics_annotations_with`] adds
3272 /// cancellation and progress.
3273 pub async fn write_analytics_annotations(
3274 &self,
3275 annotations: Vec<Annotation>,
3276 ) -> BulkResult<usize> {
3277 self.write_analytics_annotations_with(annotations, BulkControl::new())
3278 .await
3279 }
3280
3281 /// [`Self::write_analytics_annotations`] with cancellation and progress
3282 /// (0.13.8, W7.6).
3283 pub async fn write_analytics_annotations_with(
3284 &self,
3285 annotations: Vec<Annotation>,
3286 control: BulkControl,
3287 ) -> BulkResult<usize> {
3288 self.low_chunked(
3289 annotations,
3290 chunk_rows::ANNOTATIONS,
3291 control,
3292 |chunk, responder| LowPriCommand::WriteAnalyticsChunk { chunk, responder },
3293 )
3294 .await
3295 }
3296
3297 /// Move closed intervals and superseded log rows older than `cutoff` to the
3298 /// cold database (§5.7, D-012).
3299 pub async fn archive(&self, cutoff: &str) -> Result<ArchiveReport> {
3300 let cutoff = timestamp::normalize(cutoff)?;
3301 let archive_path = self.archive_path.clone();
3302 self.low(|responder| LowPriCommand::Archive {
3303 cutoff,
3304 archive_path,
3305 responder,
3306 })
3307 .await
3308 }
3309
3310 /// Forget one lineage: move its whole ledger to the cold database and
3311 /// remove the lineage record (0.14.13, §15.4, D-230).
3312 ///
3313 /// The abandonment arm. A conversation tree discards most of what it grows,
3314 /// and [`Self::archive`] cannot reclaim it: that arm is indexed by *time*,
3315 /// so archiving an abandoned branch's recent history means archiving the
3316 /// trunk's recent history with it.
3317 ///
3318 /// **Everything the lineage holds moves in one transaction** — its `links`,
3319 /// its `concepts`, its `transaction_log` entries and its `branches` row —
3320 /// and afterwards the name is unknown: every read and write naming it
3321 /// raises [`DbError::UnknownBranch`]. That is the design's whole shape, and
3322 /// `temporal::archive::archive_branch` records why it has no smaller
3323 /// version.
3324 ///
3325 /// # It refuses more than it accepts, on purpose
3326 ///
3327 /// - The trunk, and a name that is not registered
3328 /// ([`DbError::UnknownBranch`]).
3329 /// - A branch with **descendants**: they read through it, so archiving it
3330 /// would delete rows they still believe.
3331 /// - A branch whose **concepts another lineage's hot link names**. The road
3332 /// map assumed an abandoned branch's rows were "a contiguous archivable
3333 /// set by construction"; a concept is keyed by identity across the whole
3334 /// ledger (D-214), so they are not, and this refusal is what makes them
3335 /// contiguous in the cases it accepts.
3336 ///
3337 /// All but the first return [`DbError::BranchNotArchivable`] with a reason.
3338 ///
3339 /// The lineage record lands in `cold.branches` with an `archived_at`, so a
3340 /// cold row's `branch_id` still resolves to something — in the cold file,
3341 /// which is now the only place it does.
3342 pub async fn archive_branch(&self, branch: crate::branch::BranchId) -> Result<ArchiveReport> {
3343 let branch = branch.as_str().to_string();
3344 let archive_path = self.archive_path.clone();
3345 self.low(|responder| LowPriCommand::ArchiveBranch {
3346 branch,
3347 archive_path,
3348 responder,
3349 })
3350 .await
3351 }
3352
3353 /// Move the named concepts back from the cold database into the hot tables
3354 /// (§2.3, C3).
3355 ///
3356 /// Rehydration is a **physical move back, not a write**: it mints no
3357 /// transaction-time facts and is invisible to both clocks. An id that is not
3358 /// in the cold file is skipped rather than being an error — the caller
3359 /// generally has a list from a cold-side query, and a partially-stale list is
3360 /// the normal case rather than a mistake. The report says how many actually
3361 /// moved.
3362 ///
3363 /// See [`RehydrateReport::rowids_reassigned`] for the one way a rehydrated
3364 /// row can differ from the row that was archived.
3365 pub async fn rehydrate(&self, ids: &[&str]) -> Result<RehydrateReport> {
3366 let ids: Vec<String> = ids.iter().map(|s| (*s).to_string()).collect();
3367 let archive_path = self.archive_path.clone();
3368 self.low(|responder| LowPriCommand::Rehydrate {
3369 ids,
3370 archive_path,
3371 responder,
3372 })
3373 .await
3374 }
3375
3376 /// Archive up to `cutoff` as a sequence of sessions, each covering at most
3377 /// `window` of **transaction** time (T1.1, D-080).
3378 ///
3379 /// `archive(cutoff)` is one transaction whose size is set by how long it has
3380 /// been since the last one, which makes it the least bounded of the three
3381 /// operations exempt from [`CHUNK_BUDGET`] — its hold is a function of
3382 /// operational history rather than of anything a caller chose. This runs the
3383 /// same work as *N* complete sessions, each with its own marker, horizon row
3384 /// and rebuild, and returns one [`ArchiveReport`] per session in order.
3385 ///
3386 /// # D-012 is satisfied per session, and that is what it requires
3387 ///
3388 /// The atomicity D-012 demands is that copy-then-delete never be split — a
3389 /// crash between the phases duplicates or loses rows. *N* small sessions
3390 /// satisfy that exactly as one large one does. The obligation windowing adds
3391 /// is that a partial run leave a coherent intermediate state, which it does:
3392 /// each session commits a valid horizon, so a failure at window *k* leaves a
3393 /// database archived up to boundary *k−1* and nothing in between. **The
3394 /// sequence is not atomic and does not claim to be** — on error, the reports
3395 /// for the sessions that did commit are lost with it, but their effect is
3396 /// not, and re-running with the same `cutoff` completes the job.
3397 ///
3398 /// # Each session is its own actor turn, and that is the entire point
3399 ///
3400 /// This loop lives here, on the handle, rather than inside the actor's
3401 /// `Archive` arm. Putting it there would have produced *N* small
3402 /// transactions inside **one** hold, which shrinks the transaction and
3403 /// changes the latency not at all: the actor is single-threaded, so nothing
3404 /// else writes until its turn returns regardless of how many `COMMIT`s the
3405 /// turn contains. Sending *N* commands returns the actor to its `select!`
3406 /// between sessions, which is where an interactive assertion gets to jump
3407 /// the queue — and it is high-priority, so it does.
3408 ///
3409 /// The same reasoning is why [`Self::bulk_import`] chunks here and not
3410 /// there, and it is the trap T1.2 names for `CREATE TABLE … AS SELECT`.
3411 ///
3412 /// # Choosing a window
3413 ///
3414 /// The bound is on *transaction* time, so the session count is set by how
3415 /// far back the hot file goes, not by how much it holds. A window is
3416 /// rejected rather than clamped if it would need more than
3417 /// [`MAX_ARCHIVE_SESSIONS`] sessions — see [`DbError::ArchiveWindow`].
3418 ///
3419 /// Windows containing nothing archivable are cheap but not free: each still
3420 /// opens a transaction and writes a horizon row. What they no longer do is
3421 /// re-project `links_current`, which `archive_session` now skips when its
3422 /// `DELETE` removed no rows — without that, windowing costs *more* in total
3423 /// than not windowing, because the repair term scales with the surviving
3424 /// table and not with the batch (D-077).
3425 pub async fn archive_windowed(
3426 &self,
3427 cutoff: &str,
3428 window: std::time::Duration,
3429 ) -> Result<Vec<ArchiveReport>> {
3430 let cutoff = timestamp::normalize(cutoff)?;
3431 let boundaries = self.archive_boundaries(&cutoff, window).await?;
3432
3433 let mut reports = Vec::with_capacity(boundaries.len());
3434 for boundary in boundaries {
3435 let archive_path = self.archive_path.clone();
3436 reports.push(
3437 self.low(|responder| LowPriCommand::Archive {
3438 cutoff: boundary,
3439 archive_path,
3440 responder,
3441 })
3442 .await?,
3443 );
3444 }
3445 Ok(reports)
3446 }
3447
3448 /// The cutoffs [`Self::archive_windowed`] will run, ascending, ending at
3449 /// `cutoff` exactly.
3450 ///
3451 /// Read on `read_conn`, not on the actor: this is two `MIN`s and the actor
3452 /// has no reason to hold its lock for them.
3453 ///
3454 /// The lower end comes from the data rather than from the clock. Stepping
3455 /// from some fixed epoch would make the session count a function of the
3456 /// calendar — a database opened yesterday would still be asked to archive
3457 /// 1970 — whereas the oldest `recorded_at` actually present is the earliest
3458 /// boundary that can contain anything.
3459 async fn archive_boundaries(
3460 &self,
3461 cutoff: &str,
3462 window: std::time::Duration,
3463 ) -> Result<Vec<String>> {
3464 // A single session at `cutoff` is exactly `archive(cutoff)`, and it is
3465 // the right answer for an empty hot file: it still writes the horizon
3466 // row, so windowed and unwindowed runs leave the same observable state.
3467 let Some(oldest) = self.oldest_hot_stamp(cutoff).await? else {
3468 return Ok(vec![cutoff.to_string()]);
3469 };
3470
3471 let start = timestamp::parse(&oldest)?;
3472 let end = timestamp::parse(cutoff)?;
3473 let Ok(span) = end.duration_since(start) else {
3474 // Everything in the hot file is at or after the cutoff, so there is
3475 // nothing in range to divide.
3476 return Ok(vec![cutoff.to_string()]);
3477 };
3478
3479 if window.is_zero() {
3480 return Err(DbError::ArchiveWindow {
3481 window,
3482 reason: "a zero-length window never advances past the first boundary".into(),
3483 });
3484 }
3485
3486 // `div_ceil` on nanos: a span of 90 minutes in 60-minute windows is two
3487 // sessions, not one. `as_nanos` is u128, so neither the division nor the
3488 // span can overflow for any timestamp this crate can store.
3489 let sessions = span.as_nanos().div_ceil(window.as_nanos());
3490 if sessions > MAX_ARCHIVE_SESSIONS as u128 {
3491 return Err(DbError::ArchiveWindow {
3492 window,
3493 reason: format!(
3494 "a span of {span:?} would need {sessions} sessions (limit \
3495 {MAX_ARCHIVE_SESSIONS}); widen the window"
3496 ),
3497 });
3498 }
3499
3500 let mut boundaries = Vec::with_capacity(sessions as usize);
3501 for k in 1..sessions {
3502 boundaries.push(timestamp::format(start + window * k as u32));
3503 }
3504 // The last boundary is `cutoff` itself and not `start + n*window`, which
3505 // would overshoot and archive rows the caller excluded.
3506 boundaries.push(cutoff.to_string());
3507 Ok(boundaries)
3508 }
3509
3510 /// Oldest `recorded_at` below `cutoff` in either hot table, or `None`.
3511 async fn oldest_hot_stamp(&self, cutoff: &str) -> Result<Option<String>> {
3512 let mut oldest: Option<String> = None;
3513 for table in ["links", "transaction_log"] {
3514 let found: Option<String> = self
3515 .read_conn
3516 .query(
3517 &format!("SELECT MIN(recorded_at) FROM {table} WHERE recorded_at < ?1"),
3518 libsql::params![cutoff],
3519 )
3520 .await?
3521 .next()
3522 .await?
3523 .and_then(|row| row.get(0).ok());
3524 if let Some(found) = found {
3525 if oldest.as_ref().is_none_or(|o| found < *o) {
3526 oldest = Some(found);
3527 }
3528 }
3529 }
3530 Ok(oldest)
3531 }
3532
3533 /// Send a high-priority command and wait for its answer.
3534 ///
3535 /// The two error mappings here are the whole reason this helper exists.
3536 /// `send` failing means the actor is gone — `WriterUnavailable`. The
3537 /// responder being dropped without an answer means the actor took the
3538 /// command and never replied — `WriterDroppedResponder`, which is a bug in
3539 /// the actor rather than a condition the caller can retry. Both variants
3540 /// existed in `error.rs` from 0.4.5 and neither was ever constructed, so a
3541 /// dead actor and a hung one were both just a caller waiting forever.
3542 async fn high<T>(
3543 &self,
3544 make: impl FnOnce(oneshot::Sender<Result<T>>) -> HighPriCommand,
3545 ) -> Result<T> {
3546 let (tx, rx) = oneshot::channel();
3547 self.highpri_tx
3548 .send(make(tx))
3549 .await
3550 .map_err(|_| DbError::WriterUnavailable)?;
3551 rx.await.map_err(|_| DbError::WriterDroppedResponder)?
3552 }
3553
3554 /// Send each chunk in turn and sum the counts — the shape all four bulk
3555 /// paths share (T3.4, D-086).
3556 ///
3557 /// # This is sequential on purpose, and the purpose is a measurement
3558 ///
3559 /// T3.4 proposed pipelining: send *k* chunks ahead so the actor never finds
3560 /// an empty queue. The reasoning is that awaiting each chunk before building
3561 /// the next leaves the actor idle for a channel round trip every time, which
3562 /// on a 1M-edge import is ~11,000 idle gaps.
3563 ///
3564 /// Both halves of that are true and the conclusion does not follow. The gaps
3565 /// are real; they are also **four orders of magnitude smaller than the work
3566 /// they interrupt**. A tokio mpsc hop is sub-microsecond and a chunk takes
3567 /// 13–21 ms. Implemented and swept at depths 1, 2, 4, 8 and 16 over 20K and
3568 /// 100K edges: every cell landed within 1% of sequential, in both directions
3569 /// — see `examples/pipeline_diag.rs`, which is kept precisely so this is not
3570 /// re-proposed from the same reasoning.
3571 ///
3572 /// So the pipelining was removed and the deduplication kept. It was not free
3573 /// to hold: with chunks in flight, a failure at chunk `i` no longer leaves a
3574 /// **prefix** committed, because `i+1 ..= i+k-1` were already sent and commit
3575 /// anyway. D-011 promises "earlier chunks committed", and paying for that
3576 /// with a weaker recovery story in exchange for nothing measurable is the
3577 /// wrong trade.
3578 ///
3579 /// Sending stops at the first error, so what commits is exactly the prefix
3580 /// before the failure.
3581 /// # The size is now measured, not assumed (0.12.0, W3)
3582 ///
3583 /// Until 0.11.0 the caller pre-split into `chunks(chunk_rows::WHATEVER)` and
3584 /// this loop sent what it was given. That made the constant *the* size, and
3585 /// D-143 is the record of a constant fitted at one population being wrong at
3586 /// another: all four D-088 shapes agreed the largest in-budget edge chunk was
3587 /// **20** against a shipped 90, and 20 would itself have been wrong at 80,000
3588 /// edges, because per-row cost on that path grows with `links_current`.
3589 ///
3590 /// No row count can bound a duration on such a path, so the loop stopped
3591 /// trying to pick one ahead of time. `ceiling` — still the path's
3592 /// [`chunk_rows`] constant, with its derivation intact — is now the largest
3593 /// size this will ever ask for, and each chunk's measured hold chooses the
3594 /// next through `next_chunk_size`.
3595 ///
3596 /// **Feedback, not preemption.** The chunk in flight always commits in full;
3597 /// the SQLite write lock is not preemptible, so nothing here can shorten a
3598 /// transaction already running. A batch of one chunk gets no protection at
3599 /// all, and convergence costs one or two chunks — which is the price of the
3600 /// bound being a duration rather than a promise.
3601 ///
3602 /// The last chunk's outcome is discarded, there being no next chunk to size.
3603 /// The chunk loop behind all four bulk paths.
3604 ///
3605 /// **Every exit carries `written`** (0.13.8, W7.6, D-181). It used to
3606 /// carry it only out of the success arm: the three error paths were `?` on
3607 /// a [`DbError`], which discards the local, so a caller whose 20,000-row
3608 /// import failed in the last chunk learned that it failed and not that
3609 /// 19,000 rows were already in the database. The count was never expensive
3610 /// to keep — it is right there, and the loop needs it anyway to size the
3611 /// next chunk.
3612 async fn low_chunked<T>(
3613 &self,
3614 items: Vec<T>,
3615 ceiling: usize,
3616 control: BulkControl,
3617 make: impl Fn(Vec<T>, oneshot::Sender<Result<ChunkOutcome>>) -> LowPriCommand,
3618 ) -> BulkResult<usize> {
3619 let total = items.len();
3620 let mut items = items.into_iter();
3621 let mut size = ceiling.max(1);
3622 let mut written = 0usize;
3623 loop {
3624 let chunk: Vec<T> = items.by_ref().take(size).collect();
3625 if chunk.is_empty() {
3626 // Emptiness is checked before cancellation on purpose: a token
3627 // raised after the last chunk committed is asking to stop work
3628 // that is already done, and reporting that as a failure would
3629 // make a race between the caller's two threads decide whether a
3630 // complete import counts as one.
3631 return Ok(written);
3632 }
3633 // Between chunks, never inside one. Nothing is rolled back and no
3634 // transaction is interrupted -- the loop simply stops sending, and
3635 // the prefix that committed is the same kind of prefix a failure
3636 // would have left.
3637 if control.is_cancelled() {
3638 return Err(BulkInterrupted {
3639 written,
3640 cause: DbError::BulkCancelled,
3641 });
3642 }
3643 let stop = |cause: DbError| BulkInterrupted { written, cause };
3644 let (tx, rx) = oneshot::channel();
3645 self.lowpri_tx
3646 .send(make(chunk, tx))
3647 .await
3648 .map_err(|_| stop(DbError::WriterUnavailable))?;
3649 let outcome = match rx.await {
3650 Err(_) => return Err(stop(DbError::WriterDroppedResponder)),
3651 Ok(Err(e)) => return Err(stop(e)),
3652 Ok(Ok(outcome)) => outcome,
3653 };
3654 written += outcome.rows;
3655 control.report(BulkProgress {
3656 written,
3657 total,
3658 rows: outcome.rows,
3659 held: outcome.held,
3660 });
3661 size = next_chunk_size(size, outcome.held, CHUNK_BUDGET, CHUNK_FLOOR, ceiling);
3662 }
3663 }
3664
3665 async fn low<T>(
3666 &self,
3667 make: impl FnOnce(oneshot::Sender<Result<T>>) -> LowPriCommand,
3668 ) -> Result<T> {
3669 let (tx, rx) = oneshot::channel();
3670 self.lowpri_tx
3671 .send(make(tx))
3672 .await
3673 .map_err(|_| DbError::WriterUnavailable)?;
3674 rx.await.map_err(|_| DbError::WriterDroppedResponder)?
3675 }
3676
3677 /// Clean shutdown: stop the Write Actor, then write the final snapshot (§5.1.7).
3678 ///
3679 /// Order matters. The snapshot is taken *after* the actor has stopped and
3680 /// been joined, so no write can land between the fold and the file — the
3681 /// anchor it records is the last thing that happened, not the last thing
3682 /// that happened to be visible.
3683 ///
3684 /// A failed snapshot is reported rather than swallowed. It is not a
3685 /// durability loss — the ledger is in the WAL and the log replays without
3686 /// it — but it means the next open starts from an older anchor, and a caller
3687 /// that never hears about it cannot know why startup got slower.
3688 ///
3689 /// **The cadence stops first (§5.5, D-053).** Both it and `write_final` end
3690 /// by running retention over the snapshot directory, and retention deletes
3691 /// files. Letting them overlap would mean one pass enumerating the directory
3692 /// while the other removes from it — not a correctness problem for the
3693 /// ledger, which is why the ordering is stated rather than locked, but a
3694 /// source of spurious warnings and of a final anchor that could be deleted
3695 /// by a cleanup that started before it existed. Stopping the cadence, then
3696 /// the actor, then taking the snapshot leaves exactly one writer at each
3697 /// step.
3698 pub async fn close(mut self) -> Result<()> {
3699 if let Some(stop) = self.cadence_stop.take() {
3700 let _ = stop.send(true);
3701 }
3702 if let Some(handle) = self.cadence.take() {
3703 let _ = handle.await;
3704 }
3705
3706 // Top up the planner's statistics while the actor is still alive to do
3707 // it (0.12.4, D-149). `PRAGMA optimize` re-analyses only what SQLite
3708 // believes has gone stale, so on a database that did nothing this costs
3709 // nothing, and on one that was just bulk-loaded it is the difference
3710 // between the next process planning on measurements and planning on
3711 // built-in guesses.
3712 //
3713 // **Deliberately not fatal.** A failure here costs plan quality on the
3714 // next open and nothing else — no ledger state depends on it — and
3715 // `close()` is where a caller learns whether their *writes* survived.
3716 // Turning a stale-statistics problem into a failed close would bury that
3717 // answer under a much less important one.
3718 if let Err(e) = self.optimize().await {
3719 tracing::warn!(
3720 "PRAGMA optimize failed during close(): {e}. Statistics may be \
3721 stale for the next process; call analyze() to rebuild them. \
3722 Nothing else is affected."
3723 );
3724 }
3725
3726 let (tx, rx) = oneshot::channel();
3727 let _ = self
3728 .highpri_tx
3729 .send(HighPriCommand::Shutdown { responder: tx })
3730 .await;
3731 let _ = rx.await;
3732
3733 // **The writer's exit status is propagated, not discarded (Wave 4.2).**
3734 // It used to be `let _ = handle.await`, so an actor that had died closed
3735 // "successfully" and the caller's last chance to learn that the write
3736 // path was gone was spent silently.
3737 //
3738 // Through 0.13.3 this awaited a `JoinHandle<Result<()>>` and did
3739 // `Ok(res) => res?`, which looked like two failure paths and was one:
3740 // the actor's `Result` could not be `Err` (W7.3, D-177). What remains is
3741 // the branch that can fire — the actor panicked or was aborted — mapped
3742 // by `writer_exit`, which is tested against a real `JoinError`.
3743 //
3744 // Ordered before the final snapshot on purpose: a snapshot written after
3745 // a dead writer records a state the caller has no reason to trust, and
3746 // returning the error while also having written that file is worse than
3747 // not writing it.
3748 if let Some(handle) = self.writer.take() {
3749 writer_exit(handle.await)?;
3750 }
3751
3752 let ts = self.clock.now();
3753 let archive = crate::temporal::archive::archive_present(&self.archive_path)
3754 .then_some(self.archive_path.as_path());
3755 snapshot::write_final(&self.read_conn, &self.snapshots_dir, &ts, archive).await?;
3756
3757 // Marks the handle closed so `Drop` knows not to complain.
3758 self.closed = true;
3759 Ok(())
3760 }
3761}
3762
3763/// Notes a missed `close()` at `warn!`, and deliberately does **not** assert.
3764///
3765/// **§7.3 offered option B — document `close()` as mandatory and `debug_assert`
3766/// in `Drop` — and Wave 4.2 implemented it, measured the consequence, and
3767/// reduced it to a warning.** The assert fired on roughly thirty tests on its
3768/// first run. That is the signal it was built to produce, and the right reading
3769/// of it was not "thirty tests are wrong".
3770///
3771/// What dropping actually costs is one final snapshot. Nothing else: every
3772/// public write method awaits its responder, so by the time a caller *can* drop
3773/// the handle, every write it issued has already committed; and the cadence stops
3774/// on its own, because `cadence_stop` is a `watch::Sender` whose drop signals the
3775/// task. A snapshot is derivative state under Doctrine VI — disposable,
3776/// reconstructible, and never the only copy of anything. Losing one makes the
3777/// next `reconstruct` fold from an older anchor, which is **slower, not wrong**.
3778///
3779/// A `debug_assert` aborts a test run. Spending that on a performance loss, in a
3780/// project whose own notes say a suite that fails for reasons unrelated to the
3781/// code under test trains people to ignore red, is the wrong trade — and paying
3782/// it in thirty places would have made `close()` look mandatory by ceremony
3783/// rather than by consequence. `close()` remains the right thing to call, and
3784/// the two reasons to call it are now stated where they can be acted on: the
3785/// snapshot, and the writer's `Result`, which only `close()` can return.
3786///
3787/// Option A ("abort the actor and log") stays rejected, for the reason it was
3788/// rejected twice before: `Drop` cannot await, so it cannot drain, and cleanup
3789/// that cannot clean up is worse than none — it looks like cleanup.
3790impl Drop for Database {
3791 fn drop(&mut self) {
3792 if !self.closed {
3793 tracing::warn!(
3794 "Database dropped without close(): the final snapshot was not written, \
3795 so the next reconstruct folds from an older anchor, and the write \
3796 actor's exit status was not checked. Prefer close().await."
3797 );
3798 }
3799 }
3800}
3801
3802/// A failure before the first chunk was sent, which committed nothing.
3803///
3804/// Normalisation runs over the whole batch up front, so its errors are the one
3805/// class the chunk loop never sees — and they are still [`BulkInterrupted`],
3806/// because a caller matching on one error type should not have to match on two
3807/// to find out that nothing landed (0.13.8, W7.6).
3808fn before_any_chunk(cause: DbError) -> BulkInterrupted {
3809 BulkInterrupted { written: 0, cause }
3810}
3811
3812fn normalize_all(edges: Vec<EdgeAssertion>) -> Result<Vec<EdgeAssertion>> {
3813 edges.into_iter().map(EdgeAssertion::normalized).collect()
3814}
3815
3816/// One `FULL` checkpoint for the numbers, then a `TRUNCATE` for the file.
3817///
3818/// # Why `TRUNCATE` and not a mode parameter
3819///
3820/// The four SQLite modes are not four things a caller of *this* crate wants.
3821/// `PASSIVE` is what the automatic checkpointer already runs on its own, so an
3822/// explicit `PASSIVE` asks for something that was going to happen anyway;
3823/// `RESTART` and `FULL` differ from `TRUNCATE` only in whether the WAL file is
3824/// left at its high-water size. The reason W5.2 exists is
3825/// [`Tuning::wal_autocheckpoint`] — a bulk importer turns the automatic
3826/// checkpointer off and calls this once at the end — and what that caller wants
3827/// is the WAL *gone*, not smaller than it was. So the mode is fixed and decided
3828/// here rather than pushed to the caller as a choice they would have to read
3829/// SQLite's documentation to make. If a mode ever needs selecting, that is an
3830/// additive method, not a change to this one.
3831///
3832/// # Why it is two pragmas, which is not the obvious implementation
3833///
3834/// **A successful `TRUNCATE` reports `busy=0, log=0, checkpointed=0`** — the
3835/// counts describe the WAL *after* the operation, and after a truncation there
3836/// is no WAL to describe. Measured, not inferred: on a 387-frame WAL, `PASSIVE`
3837/// returns `0, 387, 387` and `TRUNCATE` on the same file returns `0, 0, 0`. So
3838/// the single-pragma implementation returns a [`CheckpointReport`] whose two
3839/// counts are structurally zero on success, which makes the whole struct a
3840/// less useful `bool`.
3841///
3842/// `FULL` copies every frame back and reports what it moved; the `TRUNCATE`
3843/// that follows finds nothing left to copy and resets the file. The second pass
3844/// is close to free for exactly that reason — it is a file operation, not a
3845/// second copy. `busy` is the **union**: a checkpoint that was blocked in
3846/// either phase did not fully happen, and a caller about to copy the database
3847/// file elsewhere needs the pessimistic answer.
3848///
3849/// # They return rows, so they go through `query()`
3850///
3851/// The same libsql constraint the pragmas in `configure` document: `execute()`
3852/// rejects any statement that yields rows, and these yield the row that is the
3853/// entire point.
3854async fn run_checkpoint(conn: &libsql::Connection) -> Result<CheckpointReport> {
3855 // The columns are `busy, log, checkpointed`. SQLite reports -1 for the two
3856 // counts when the checkpoint could not run; clamped to 0 rather than
3857 // surfaced as a signed count, because `busy` already carries "this did not
3858 // happen" and a negative frame count is not a quantity anyone can use.
3859 //
3860 // A database not in WAL mode returns no row at all. `configure` puts every
3861 // connection this crate opens into WAL, so that is unreachable here — but a
3862 // zeroed report is a better failure than a panic if it stops being.
3863 async fn one(conn: &libsql::Connection, sql: &str) -> Result<(bool, u64, u64)> {
3864 let mut rows = conn.query(sql, ()).await?;
3865 let Some(row) = rows.next().await? else {
3866 return Ok((false, 0, 0));
3867 };
3868 let field = |i: i32| -> u64 { row.get::<i64>(i).unwrap_or(0).max(0) as u64 };
3869 Ok((row.get::<i64>(0).unwrap_or(0) != 0, field(1), field(2)))
3870 }
3871
3872 let (full_busy, _, moved) = one(conn, "PRAGMA wal_checkpoint(FULL)").await?;
3873 let (trunc_busy, log_frames, _) = one(conn, "PRAGMA wal_checkpoint(TRUNCATE)").await?;
3874
3875 Ok(CheckpointReport {
3876 busy: full_busy || trunc_busy,
3877 log_frames,
3878 checkpointed_frames: moved,
3879 })
3880}
3881
3882/// Pragmas that mean something on **any** connection, including one opened
3883/// `SQLITE_OPEN_READ_ONLY` (0.12.16, W5.5, D-159).
3884///
3885/// Both of these are per-connection state that a reader is subject to just as a
3886/// writer is. `busy_timeout` is the one that made this a finding:
3887/// [`Database::diagnostic_conn`] ran with SQLite's default of **0** — return
3888/// `SQLITE_BUSY` immediately — while every other connection in the process
3889/// waited 5 s, so the one surface whose job is to answer questions when the
3890/// typed path is already suspect was also the one most likely to fail with
3891/// "database is locked" under exactly the contention that prompted the
3892/// question.
3893/// Empty the slot unless what is in it is fit to hand to the next caller
3894/// (0.15.15, W15.5, [D-257]).
3895///
3896/// Shared by [`Database::diagnostic_conn`], which runs it on the way in, and
3897/// [`Database::scrub_diagnostic_conn`], which the Python binding runs on the way
3898/// out. One implementation because the two must agree: a connection the entry
3899/// path would have discarded is one the exit path must not leave sitting there.
3900async fn scrub(slot: &mut Option<libsql::Connection>) {
3901 let Some(conn) = slot.as_ref() else {
3902 return;
3903 };
3904 // A leaked `BEGIN` is the one that leaves this surface: it pins a WAL read
3905 // snapshot, so later diagnostic reads answer from it and `checkpoint()`
3906 // cannot truncate past it. Rolled back rather than reported, because the
3907 // caller who would read the report is the one who did not do it.
3908 let recovered = conn.is_autocommit() || conn.execute("ROLLBACK", ()).await.is_ok();
3909 // A connection that will not roll back is not one to hand on, and neither
3910 // is one whose own state cannot be read: either way the answer is a fresh
3911 // connection, which costs 56.5 us on the call that dirtied it and nothing
3912 // on any other.
3913 if !recovered || diagnostic_is_dirty(conn).await.unwrap_or(true) {
3914 *slot = None;
3915 }
3916}
3917
3918/// Is this cached diagnostic connection carrying state from an earlier caller
3919/// (0.15.15, W15.5, [D-257])?
3920///
3921/// Two questions — are there temp objects, is anything attached beyond `main`
3922/// and `temp` — asked as pragmas rather than as a query over
3923/// `temp.sqlite_master` and `pragma_database_list`. Same answers, and
3924/// `examples/diagnostic_hygiene_probe.rs` measures the pragma form at
3925/// **2.4 µs** against the query form's **7.8 µs**, on a call whose whole warm
3926/// cost is the `stat` in front of it. Detection is measured rather than
3927/// assumed: `temp.schema_version` goes 0 → 1 on `CREATE TEMP TABLE`, and
3928/// `database_list` 2 → 3 on `ATTACH`.
3929///
3930/// **What it cannot see is a `PRAGMA`**, and neither can any other cheap check:
3931/// SQLite does not enumerate connection-scoped pragma state. The crate restates
3932/// the two it sets instead — see the call to `configure_common` in
3933/// [`Database::diagnostic_conn`].
3934///
3935/// An `Err` here is not propagated by the caller: a connection whose own state
3936/// cannot be read is replaced rather than reported on.
3937///
3938/// [D-257]: ../../docs/architecture/s13-decision-register.md#d-257
3939async fn diagnostic_is_dirty(conn: &libsql::Connection) -> Result<bool> {
3940 let mut rows = conn.query("PRAGMA temp.schema_version", ()).await?;
3941 let temp_schema: i64 = match rows.next().await? {
3942 Some(row) => row.get(0)?,
3943 // No row at all is not a clean connection, it is an answer this
3944 // function did not understand.
3945 None => return Ok(true),
3946 };
3947 if temp_schema != 0 {
3948 return Ok(true);
3949 }
3950 let mut rows = conn.query("PRAGMA database_list", ()).await?;
3951 let mut databases = 0_usize;
3952 while rows.next().await?.is_some() {
3953 databases += 1;
3954 }
3955 // `main` and `temp`, always both, on a connection nobody has attached to.
3956 Ok(databases > 2)
3957}
3958
3959async fn configure_common(conn: &libsql::Connection, cache_size: Option<i32>) -> Result<()> {
3960 // NOTE: `busy_timeout` returns its resulting value as a row, and libsql's
3961 // `execute()` rejects any statement that yields rows ("Execute returned
3962 // rows"). It must be issued through `query()`.
3963 let _ = conn.query("PRAGMA busy_timeout = 5000", ()).await?;
3964 // Per-connection, and split writer from reader since 0.12.15 (W5.4,
3965 // D-158). `None` runs no pragma at all rather than restating SQLite's
3966 // default, so the default remains SQLite's to change.
3967 if let Some(pages) = cache_size {
3968 conn.execute(&format!("PRAGMA cache_size = {pages}"), ())
3969 .await?;
3970 }
3971 Ok(())
3972}
3973
3974/// Pragmas that only mean anything where writes can happen (0.12.16, W5.5).
3975///
3976/// Not run on [`Database::diagnostic_conn`], and the reason is not tidiness:
3977/// `journal_mode = WAL` is a change to the *database file*, which a connection
3978/// opened `SQLITE_OPEN_READ_ONLY` cannot make. The rest —
3979/// `synchronous`, `foreign_keys`, `recursive_triggers`, and the `ANALYZE`
3980/// bound — govern how writes behave, and a connection that cannot write is not
3981/// governed by them.
3982///
3983/// The write connection and the two internal readers all still get these. The
3984/// internal readers are opened from the same read-write `libsql::Database`, so
3985/// the pragmas apply; leaving them out would be a behaviour change made for
3986/// symmetry, which is not a reason.
3987async fn configure_writable(conn: &libsql::Connection) -> Result<()> {
3988 // Returns its resulting value as a row — see the note in `configure_common`.
3989 let _ = conn.query("PRAGMA journal_mode = WAL", ()).await?;
3990 conn.execute("PRAGMA synchronous = NORMAL", ()).await?;
3991 conn.execute("PRAGMA foreign_keys = ON", ()).await?;
3992 conn.execute("PRAGMA recursive_triggers = OFF", ()).await?;
3993 // Bounds every `ANALYZE` this connection will ever run, explicit or
3994 // triggered by `PRAGMA optimize` (D-149). Set here rather than around the
3995 // call sites so the scheduled path is bounded too — that is the half that
3996 // runs with nobody watching. Returns the previous limit as a row, so it goes
3997 // through `query()` for the reason the note above gives.
3998 let _ = conn.query(crate::schema::ddl::ANALYSIS_LIMIT, ()).await?;
3999 Ok(())
4000}
4001
4002/// Full pragma configuration, for a connection that can write.
4003async fn configure(
4004 conn: libsql::Connection,
4005 cache_size: Option<i32>,
4006) -> Result<libsql::Connection> {
4007 configure_writable(&conn).await?;
4008 configure_common(&conn, cache_size).await?;
4009 Ok(conn)
4010}
4011
4012/// Helper to derive the snapshot directory by convention: foo.db -> foo_snapshots/
4013fn derive_snapshots_dir(path: &Path) -> PathBuf {
4014 let mut dir = path.to_path_buf();
4015 let stem = path
4016 .file_stem()
4017 .and_then(|s| s.to_str())
4018 .unwrap_or("macrame");
4019 dir.set_file_name(format!("{stem}_snapshots"));
4020 dir
4021}
4022
4023/// Helper to derive archive database path by convention: foo.db -> foo_archive.db
4024fn derive_archive_path(path: &Path) -> PathBuf {
4025 let mut archive = path.to_path_buf();
4026 if let Some(stem) = path.file_stem().and_then(|s| s.to_str()) {
4027 let ext = path.extension().and_then(|e| e.to_str()).unwrap_or("db");
4028 archive.set_file_name(format!("{stem}_archive.{ext}"));
4029 } else {
4030 archive.set_extension("archive.db");
4031 }
4032 archive
4033}
4034
4035/// Dedicated Write Actor event loop prioritizing high-priority UI requests over low-priority background work.
4036///
4037/// # The turn is the unit, not the statement (T1.4)
4038///
4039/// One iteration of this loop is one *hold*: the actor is single-threaded and
4040/// the SQLite write lock is not preemptible, so from the moment a command starts
4041/// executing until it returns, nothing else writes. That is the quantity
4042/// [`CHUNK_BUDGET`] bounds, and so it is the quantity
4043/// [`crate::metrics::ActorMetrics`] measures — deliberately around the whole
4044/// `execute` call rather than inside it. Timing the SQL alone would have
4045/// reported a bound that held while callers waited.
4046///
4047/// Queue depth is sampled *before* the `select!`, so it is the backlog the turn
4048/// found on arrival rather than the one it left behind.
4049///
4050/// # `biased` has no floor, and since 0.12.10 that is measured (W4.4, D-153)
4051///
4052/// `biased` makes the arms poll in declaration order, so high-priority work is
4053/// taken whenever any is ready. Nothing bounds how long that can continue:
4054/// sustained interactive traffic can hold the low tier off indefinitely, and
4055/// through 0.12.9 nothing in the crate could say whether it ever did.
4056/// `record_priority_choice` counts the turns where the choice went against
4057/// queued low-priority work, and the longest unbroken run of them, which is the
4058/// half that distinguishes "prioritised" from "starved".
4059///
4060/// **No forced yield is added here.** Whether one is needed is the question the
4061/// counter answers, and adding a policy now would be fixing a bound nobody has
4062/// observed being hit — the same mistake D-124 was retracted for.
4063///
4064/// # It returns nothing, and used to return a `Result` it could not fail
4065/// (0.13.4, W7.3, §3.5, [D-177])
4066///
4067/// The two exits are `LoopCtl::Break` from [`HighPriCommand::Shutdown`] and the
4068/// `else` arm when both channels are closed. Neither can fail, and neither
4069/// could before: every command's error goes back on that command's own
4070/// responder, where the caller who issued it can act on it. There was no third
4071/// thing for an actor-level `Err` to carry, and none was ever constructed.
4072///
4073/// A `Result` that is structurally always `Ok` is not free. It reads as a
4074/// failure path under review, so `close()`'s `res?` looked like it was doing
4075/// something, and the branch that actually fires — a **panicked** actor,
4076/// reported as a `JoinError` — sat beside it untested. That is the swap this
4077/// change makes: the unfireable branch is gone and the real one is pinned, in
4078/// [`writer_exit`].
4079async fn run_writer_actor(
4080 conn: libsql::Connection,
4081 clock: Arc<dyn Clock>,
4082 mut highpri_rx: mpsc::Receiver<HighPriCommand>,
4083 mut lowpri_rx: mpsc::Receiver<LowPriCommand>,
4084 shared: Arc<ActorShared>,
4085) {
4086 // Owned by the loop and lent to each command, which is the whole of A-3:
4087 // the actor had a connection and no memory, so every turn re-established
4088 // what the turn before it had just established (0.15.6, W14.3, D-248).
4089 let mut state = ActorState::new();
4090 loop {
4091 // Read once and reused by both the depth sample and the starvation
4092 // counter, so the two cannot disagree about what was queued when this
4093 // turn went looking (W4.4, D-153).
4094 let low_queued = lowpri_rx.len();
4095 shared.metrics.record_turn(highpri_rx.len(), low_queued);
4096
4097 let ctl = tokio::select! {
4098 biased;
4099 Some(cmd) = highpri_rx.recv() => {
4100 shared.metrics.record_priority_choice(true, low_queued);
4101 let turn = Turn::start(cmd.kind(), &shared);
4102 cmd.execute(&conn, &*clock, &turn, &mut state).await
4103 }
4104 Some(cmd) = lowpri_rx.recv() => {
4105 shared.metrics.record_priority_choice(false, low_queued);
4106 let turn = Turn::start(cmd.kind(), &shared);
4107 cmd.execute(&conn, &*clock, &turn, &mut state).await
4108 }
4109 else => LoopCtl::Break,
4110 };
4111 if matches!(ctl, LoopCtl::Break) {
4112 break;
4113 }
4114 }
4115}
4116
4117/// Turn the write actor's join status into the error `close()` reports.
4118///
4119/// One line of mapping, given a name so it can be tested against a real
4120/// [`tokio::task::JoinError`]. Before 0.13.4 this was inline beside a `res?` on
4121/// an actor `Result` that could only ever be `Ok`, and the arrangement had the
4122/// coverage exactly backwards: the branch that cannot fire was plumbed through
4123/// two signatures, and the branch that does fire — the actor panicked, and the
4124/// caller's writes are going nowhere — had no test at all (W7.3, D-177).
4125///
4126/// Cancellation is folded in with panics deliberately. `JoinError` distinguishes
4127/// them, and nothing in the crate ever aborts this task, so a cancelled writer
4128/// means something outside the crate reached in and stopped it. That is not a
4129/// gentler condition than a panic and must not read as one.
4130fn writer_exit(joined: std::result::Result<(), tokio::task::JoinError>) -> Result<()> {
4131 joined.map_err(|e| DbError::WriterStopped(format!("the write actor did not exit cleanly: {e}")))
4132}
4133
4134/// One command's hold: the timer, its label, and the counters it reports to.
4135///
4136/// # The hold is recorded *before* the caller is answered, and it has to be
4137///
4138/// The obvious placement — time the whole `execute` call from the loop — is
4139/// wrong in a way that only shows up under test. Every arm of `execute` ends by
4140/// sending on a `oneshot`, which wakes the waiting caller; the actor then
4141/// returns to the loop and records. Those are two tasks, so a caller that awaits
4142/// its own write and immediately reads [`Database::metrics`] can be scheduled
4143/// first and see a turn count that does not include the write it just did.
4144///
4145/// Not a correctness bug in the ledger, and it would never have been noticed in
4146/// production — a dashboard sampling every few seconds cannot see the window.
4147/// It makes every test and diagnostic of the counters flaky, which is worse: the
4148/// instrumentation would have been *believed* while being wrong exactly when
4149/// someone tried to check it. `examples/bulk_atomic_diag.rs` was the thing that
4150/// caught it, reporting a 20,000-row batch as a 0 ms hold.
4151///
4152/// So `answer` records and then sends, in that order, and the ordering is the
4153/// method's whole reason to exist. What it costs is that the `oneshot::send`
4154/// itself falls outside the measurement, which is a few nanoseconds against a
4155/// turn measured in microseconds at best.
4156struct Turn<'a> {
4157 kind: crate::metrics::CommandKind,
4158 timer: crate::metrics::HoldTimer,
4159 shared: &'a ActorShared,
4160}
4161
4162/// State the actor owns and a `Turn` needs to reach.
4163///
4164/// `archive_epoch` is here rather than in [`crate::metrics::ActorMetrics`]
4165/// because it is **not** a metric: T1.2's shadow rebuild reads it to decide
4166/// whether its work is still valid, so it has to be present in every build, not
4167/// only under the `metrics` feature. Counting archives happens to be what both
4168/// want; only one of them is allowed to be compiled out.
4169///
4170/// `turns` is here for the same reason and serves the snapshot cadence — see
4171/// its own note.
4172#[derive(Default)]
4173struct ActorShared {
4174 metrics: crate::metrics::ActorMetrics,
4175 archive_epoch: std::sync::atomic::AtomicU64,
4176 /// Commands this actor has answered `Ok` to (0.15.19, review C-19).
4177 ///
4178 /// # What it is for, and why it is not a `seq_id`
4179 ///
4180 /// `snapshot::run_cadence` used to run `SELECT MAX(seq_id), MAX(recorded_at)`
4181 /// on **every tick**, five seconds apart by default, whether or not
4182 /// anything had been written. Two aggregates on an idle database, for a
4183 /// fact the actor already had: *nothing has happened*.
4184 ///
4185 /// The review asked for a `watch<u64>` of the last committed `seq_id`. This
4186 /// is the same idea one step cheaper, and the difference matters. The actor
4187 /// does not currently know the `seq_id` its writes produced — the log rows
4188 /// are written by triggers — so publishing one would mean adding a query to
4189 /// **every write** in order to remove a query from an idle timer, which is
4190 /// the wrong direction. A turn count needs no query at all: one relaxed
4191 /// `fetch_add` on a path already doing a database round trip.
4192 ///
4193 /// # Why this cannot change when a snapshot is written
4194 ///
4195 /// The cadence skips its tick when the count has not moved since the last
4196 /// one. That is sound because the implication runs the right way: if
4197 /// `MAX(seq_id)` grew, some command committed, so some turn answered `Ok`,
4198 /// so the count moved. The converse is not claimed and does not need to be
4199 /// — a turn that answered `Ok` without writing a log row makes the cadence
4200 /// do exactly the query it used to do every time. It over-counts, never
4201 /// under-counts, and the tick it protects had nothing to do anyway.
4202 ///
4203 /// `Relaxed` because nothing is ordered against it. The cadence reads a
4204 /// number to compare with a number it read before; a value one tick stale
4205 /// costs one deferred tick and no correctness, and the same is true of the
4206 /// `archive_epoch` beside it.
4207 turns: std::sync::atomic::AtomicU64,
4208}
4209
4210impl crate::temporal::snapshot::CommittedTurns for ActorShared {
4211 fn committed_turns(&self) -> u64 {
4212 self.turns.load(std::sync::atomic::Ordering::Relaxed)
4213 }
4214}
4215
4216impl<'a> Turn<'a> {
4217 fn start(kind: crate::metrics::CommandKind, shared: &'a ActorShared) -> Self {
4218 Self {
4219 kind,
4220 timer: crate::metrics::HoldTimer::start(),
4221 shared,
4222 }
4223 }
4224
4225 fn epoch(&self) -> u64 {
4226 self.shared
4227 .archive_epoch
4228 .load(std::sync::atomic::Ordering::Relaxed)
4229 }
4230
4231 /// Record that an archive session committed.
4232 ///
4233 /// Bumped on **success only**: a failed archive rolls back, so it deletes
4234 /// nothing and invalidates no shadow build.
4235 fn archive_committed(&self) {
4236 self.shared
4237 .archive_epoch
4238 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
4239 }
4240
4241 /// Close the hold and hand the result back. Never the other way round.
4242 ///
4243 /// The `let _ =` on the send is deliberate and predates this: a caller that
4244 /// dropped its receiver — `tokio::time::timeout` around a write, which
4245 /// [`Database`]'s write surface explicitly documents — is not an actor
4246 /// error, and the command committed regardless.
4247 fn answer<T>(&self, responder: oneshot::Sender<Result<T>>, res: Result<T>) {
4248 self.shared
4249 .metrics
4250 .record_hold(self.kind, self.timer.elapsed());
4251 if res.is_ok() {
4252 self.turn_committed();
4253 }
4254 let _ = responder.send(res);
4255 }
4256
4257 /// [`answer`](Self::answer) for a chunk: the same reading, handed back to the
4258 /// caller as well as recorded (0.12.0, W1).
4259 ///
4260 /// One `elapsed()` serves both, so the duration the chunk loop sizes against
4261 /// is *the same number* the histogram shows — a controller and a dashboard
4262 /// disagreeing about what a chunk cost would be a bad way to spend a
4263 /// debugging session.
4264 ///
4265 /// The record-then-send ordering documented on [`Turn`] is preserved, and
4266 /// matters here for the same reason: the send wakes the caller, which may be
4267 /// scheduled before this method returns.
4268 fn answer_chunk(&self, responder: oneshot::Sender<Result<ChunkOutcome>>, res: Result<usize>) {
4269 let held = self.timer.elapsed();
4270 self.shared.metrics.record_hold(self.kind, held);
4271 if res.is_ok() {
4272 self.turn_committed();
4273 }
4274 let _ = responder.send(res.map(|rows| ChunkOutcome { rows, held }));
4275 }
4276
4277 /// Record that a turn answered `Ok`, for [`ActorShared::turns`].
4278 ///
4279 /// Called from both answer paths rather than from the loop, because those
4280 /// are the two places that know the result. A command that returns an error
4281 /// does not bump it: a failed write rolls back and the log is where it was.
4282 fn turn_committed(&self) {
4283 self.shared
4284 .turns
4285 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
4286 }
4287}
4288
4289const INSERT_LINK: &str = "INSERT INTO links \
4290 (source_id, target_id, edge_type, valid_from, valid_to, weight, properties, \
4291 recorded_at, branch_id) \
4292 VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)";
4293
4294/// The parameter row for [`INSERT_LINK`], in one place since 0.14.8.
4295///
4296/// The single-edge path and the chunk path spelled these out separately, which
4297/// was survivable at eight and is not at nine: `branch_id` is the one parameter
4298/// whose omission is *silent* — the column defaults to `'main'`, so a path that
4299/// forgot it would write to the trunk and pass every test that did not fork.
4300/// [`concept_params`] has existed for this reason since D-056.
4301fn edge_params<'a>(edge: &'a EdgeAssertion, stamp: &'a str) -> [libsql::Value; 9] {
4302 [
4303 edge.source.as_str().into(),
4304 edge.target.as_str().into(),
4305 edge.edge_type.as_str().into(),
4306 edge.valid_from.as_str().into(),
4307 edge.valid_to.as_str().into(),
4308 edge.weight.into(),
4309 edge.properties.as_str().into(),
4310 stamp.into(),
4311 edge.branch_name().into(),
4312 ]
4313}
4314
4315/// Shared by the single-concept write and the chunked one, so the two paths
4316/// cannot drift into upserting different column sets — and so the chunk has a
4317/// statement text it can prepare once (D-056).
4318const UPSERT_CONCEPT: &str = "INSERT INTO concepts \
4319 (id, title, content, embedding_model, valid_from, valid_to, recorded_at, retired, \
4320 branch_id) \
4321 VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9) \
4322 ON CONFLICT(id) DO UPDATE SET \
4323 title = excluded.title, \
4324 content = excluded.content, \
4325 embedding_model = excluded.embedding_model, \
4326 valid_from = excluded.valid_from, \
4327 valid_to = excluded.valid_to, \
4328 recorded_at = excluded.recorded_at, \
4329 retired = excluded.retired";
4330// `branch_id` is deliberately **not** in that `DO UPDATE` list. The column is
4331// provenance and minting happened once (D-214), and
4332// `trg_concepts_branch_immutable` would abort an update that moved it — so
4333// listing it would turn every re-upsert of an inherited concept into a guard
4334// abort instead of the no-op it is. The insert arm carries it; the update arm
4335// leaves the row where it was minted.
4336
4337/// The parameter row for [`UPSERT_CONCEPT`], in one place for the same reason.
4338fn concept_params<'a>(concept: &'a ConceptUpsert, stamp: &'a str) -> [libsql::Value; 9] {
4339 [
4340 concept.id.as_str().into(),
4341 concept.title.as_str().into(),
4342 concept.content.as_str().into(),
4343 concept
4344 .embedding_model
4345 .as_deref()
4346 .map_or(libsql::Value::Null, Into::into),
4347 concept.valid_from.as_str().into(),
4348 concept.valid_to.as_str().into(),
4349 stamp.into(),
4350 (concept.retired as i64).into(),
4351 concept.branch_name().into(),
4352 ]
4353}
4354
4355/// Check every lineage a write names, and decide which shape its guard takes.
4356///
4357/// **One function, two answers, one query per distinct lineage** — and it is
4358/// [`Lineages::shape`](crate::graph::lineage::Lineages::shape), the same function
4359/// the read path calls, for the same reason it calls it. A write naming a
4360/// branch that is not in `branches` has asked about something that does not
4361/// exist, and answering it by writing to the trunk is [D-069]'s failure in its
4362/// most expensive form: not a right-looking answer to a question that was not
4363/// asked, but a *durable* one.
4364///
4365/// Relying on the foreign key instead would refuse the write — `branch_id`
4366/// `REFERENCES branches(branch_id)` and the key is enforced — but it would
4367/// refuse it as an unqualified "FOREIGN KEY constraint failed" from inside a
4368/// rolled-back transaction, naming neither the column nor the branch. The same
4369/// argument [`classify`](crate::error::classify) makes for annotations and
4370/// edges, one table further along.
4371///
4372/// # Why a trunk write pays for it too
4373///
4374/// `None` resolves to `'main'` here rather than skipping the check, and that is
4375/// not tidiness. Once a second lineage can write, the *trunk's* overlap guard
4376/// is wrong in the other direction — it would be refused for overlapping a
4377/// branch's belief it cannot see — so the shape decision is one every write
4378/// needs, not one that branched writes need. On a database that has never
4379/// forked the answer is [`LineageShape::Trunk`] and the guard is the statement
4380/// it has always been.
4381///
4382/// # This was a query per name until 0.15.6
4383///
4384/// It ran [`crate::graph::lineage::Lineages::shape`] once per name and kept the
4385/// last answer — one round trip per write, for a table only this task writes.
4386/// [`ActorState`] holds `branches` instead, and [`Lineages::shape_of`] carries
4387/// what is left of this function's reasoning, including the part about the last
4388/// answer that stopped being true at 0.15.2.
4389///
4390/// [D-069]: ../../docs/architecture/s13-decision-register.md
4391async fn check_lineages(
4392 state: &mut ActorState,
4393 conn: &libsql::Connection,
4394 names: &[&str],
4395) -> Result<LineageShape> {
4396 state.shape_of(conn, names).await
4397}
4398
4399/// The shape and the rows, for a caller that also has to resolve an ancestry.
4400///
4401/// [`check_lineages`] with the table it read handed back rather than dropped
4402/// (0.15.17). Both callers need it: the guard compiles a statement per lineage,
4403/// and the retirement binds one.
4404async fn check_lineages_with<'a>(
4405 state: &'a mut ActorState,
4406 conn: &libsql::Connection,
4407 names: &[&str],
4408) -> Result<(LineageShape, &'a Lineages)> {
4409 let lineages = state.lineages(conn).await?;
4410 Ok((lineages.shape_of(names)?, lineages))
4411}
4412
4413/// The distinct lineages a batch names, in first-seen order.
4414///
4415/// A `Vec` and a linear scan rather than a set: batches name one lineage in
4416/// every case this crate has, the bound is the number of *branches* and not the
4417/// number of rows, and a `BTreeSet` would allocate per batch to deduplicate a
4418/// list of length one.
4419fn distinct_branches(edges: &[EdgeAssertion]) -> Vec<&str> {
4420 let mut out: Vec<&str> = Vec::with_capacity(1);
4421 for edge in edges {
4422 let name = edge.branch_name();
4423 if !out.contains(&name) {
4424 out.push(name);
4425 }
4426 }
4427 if out.is_empty() {
4428 out.push(crate::schema::ddl::MAIN_BRANCH);
4429 }
4430 out
4431}
4432
4433/// The overlap guard's prepared statement, and which question it asks.
4434///
4435/// The two statements take different parameter counts and mean different things
4436/// by the rows they return, so pairing them with the shape here is what stops
4437/// [`check_prepared`] from having to be told twice.
4438struct OverlapGuard {
4439 stmt: libsql::Statement,
4440 shape: LineageShape,
4441 /// The lineage this statement was compiled for. See [`Self::prepare`].
4442 branch: String,
4443 /// That lineage's ancestry, bound after the statement's own parameters.
4444 /// Empty under both trunk shapes, which emit no `lineage` relation.
4445 ancestry: Vec<Ancestor>,
4446}
4447
4448impl OverlapGuard {
4449 /// Prepare once per turn or per chunk, never per row (D-056, §8.8).
4450 ///
4451 /// # One statement per *lineage* since 0.15.17 ([D-259])
4452 ///
4453 /// The statement used to be a function of the shape alone: the recursive
4454 /// `lineage` CTE derived the ancestry from the branch bound at `?5`, so one
4455 /// compiled form answered for every lineage that shared a shape. A bound
4456 /// ancestry is not derived from anything — it *is* the answer for one
4457 /// reader — so the guard now carries the lineage it was compiled for and
4458 /// the values that lineage binds.
4459 ///
4460 /// The cost is bounded by [`distinct_branches`], which is a `Vec` because
4461 /// every batch this crate has seen names one lineage. A chunk that names
4462 /// two prepares two, which is the price of the resolution being correct for
4463 /// both; the shape it would otherwise share is `Resolved`, since the two
4464 /// trunk shapes each describe a database with exactly one lineage to name.
4465 ///
4466 /// [D-259]: ../../docs/architecture/s13-decision-register.md#d-259
4467 async fn prepare(
4468 conn: &libsql::Connection,
4469 shape: LineageShape,
4470 lineages: &Lineages,
4471 branch: &str,
4472 ) -> Result<Self> {
4473 // Three shapes, three statements, one spelling (0.15.8, W13.3,
4474 // D-250). Until that release the trunk had a hand-written constant and
4475 // the other two shared the resolved form, which was exact for a root
4476 // only because a root's ancestry is itself — so `Lineages::shape_of`
4477 // could return either of them and nothing observable changed. It
4478 // cannot now: the root gets a two-predicate lookup on the projection
4479 // and a branch gets the four-CTE resolution, and D-248's C-24 repair
4480 // is what decides which.
4481 let ancestry = match shape {
4482 LineageShape::Resolved => lineages.ancestry(branch),
4483 _ => Vec::new(),
4484 };
4485 let sql = crate::graph::lineage::overlap_candidates_resolved(shape, &ancestry);
4486 Ok(Self {
4487 stmt: conn.prepare(&sql).await?,
4488 shape,
4489 branch: branch.to_string(),
4490 ancestry,
4491 })
4492 }
4493
4494 /// Whether this guard answers for `branch` under `shape`.
4495 fn answers_for(&self, shape: LineageShape, branch: &str) -> bool {
4496 self.shape == shape && self.branch == branch
4497 }
4498}
4499
4500impl HighPriCommand {
4501 /// The metrics label for this variant (T1.4).
4502 ///
4503 /// Exhaustive for the same reason `execute` is: a new variant that silently
4504 /// borrowed another's label would attribute its holds to the wrong command,
4505 /// and the one question the counters exist to answer is *which* command
4506 /// broke the budget.
4507 fn kind(&self) -> crate::metrics::CommandKind {
4508 use crate::metrics::CommandKind as K;
4509 match self {
4510 HighPriCommand::AssertEdge { .. } => K::AssertEdge,
4511 HighPriCommand::RetireEdge { .. } => K::RetireEdge,
4512 HighPriCommand::UpsertConcept { .. } => K::UpsertConcept,
4513 HighPriCommand::WriteBulkAtomic { .. } => K::WriteBulkAtomic,
4514 HighPriCommand::RebuildCurrent { .. } => K::RebuildCurrent,
4515 HighPriCommand::RegisterModel { .. } => K::RegisterModel,
4516 HighPriCommand::Fork { .. } => K::Fork,
4517 HighPriCommand::Checkpoint { .. } => K::Checkpoint,
4518 HighPriCommand::Shutdown { .. } => K::Shutdown,
4519 }
4520 }
4521
4522 /// Run one command and answer its caller.
4523 ///
4524 /// Deliberately exhaustive — there is no `_` arm. The 0.4.5–0.5.4 actor
4525 /// matched `Shutdown` and `AssertEdge` and sent everything else to
4526 /// `_ => LoopCtl::Continue`, which **dropped the responder**: the caller's
4527 /// `rx.await` resolved to a `RecvError` that no code mapped, so four of six
4528 /// commands were indistinguishable from a hung database. An exhaustive match
4529 /// makes that failure a compile error instead of a runtime silence, which is
4530 /// why adding a variant should break this function.
4531 async fn execute(
4532 self,
4533 conn: &libsql::Connection,
4534 clock: &dyn Clock,
4535 turn: &Turn<'_>,
4536 state: &mut ActorState,
4537 ) -> LoopCtl {
4538 match self {
4539 HighPriCommand::Shutdown { responder } => {
4540 turn.answer(responder, Ok(()));
4541 return LoopCtl::Break;
4542 }
4543 HighPriCommand::Checkpoint { responder } => {
4544 let res = run_checkpoint(conn).await;
4545 turn.answer(responder, res);
4546 }
4547 HighPriCommand::AssertEdge { edge, responder } => {
4548 let stamp = clock.now();
4549 // Before the guard, because a write naming an unregistered
4550 // lineage should be refused by name rather than by whatever the
4551 // guard happens to find when it looks in the wrong place.
4552 let shape = match check_lineages(state, conn, &[edge.branch_name()]).await {
4553 Ok(shape) => shape,
4554 Err(e) => {
4555 turn.answer(responder, Err(e));
4556 return LoopCtl::Continue;
4557 }
4558 };
4559 if let Err(e) = reject_overlapping_interval(state, conn, &edge, shape).await {
4560 turn.answer(responder, Err(e));
4561 return LoopCtl::Continue;
4562 }
4563 // The statement is held across turns, so it is reset before it
4564 // is bound rather than after it was stepped — `check_prepared`
4565 // makes the same argument at more length.
4566 let res = match state.insert_link(conn).await {
4567 Err(e) => Err(e),
4568 Ok(stmt) => {
4569 stmt.reset();
4570 match stmt.execute(edge_params(&edge, &stamp)).await {
4571 Ok(_) => Ok(()),
4572 Err(e) => Err(classify(
4573 conn,
4574 e,
4575 WriteOp::Edge {
4576 source_id: &edge.source,
4577 target_id: &edge.target,
4578 edge_type: &edge.edge_type,
4579 },
4580 )
4581 .await),
4582 }
4583 }
4584 };
4585 turn.answer(responder, res);
4586 }
4587 HighPriCommand::RetireEdge {
4588 source,
4589 target,
4590 edge_type,
4591 valid_from,
4592 valid_to,
4593 branch,
4594 responder,
4595 } => {
4596 let stamp = clock.now();
4597 let name = branch
4598 .as_ref()
4599 .map_or(crate::schema::ddl::MAIN_BRANCH, |b| b.as_str());
4600 let resolved = check_lineages_with(state, conn, &[name])
4601 .await
4602 .map(|(shape, l)| (shape, l.ancestry(name)));
4603 let res = match resolved {
4604 Ok((shape, ancestry)) => {
4605 retire_edge(
4606 conn,
4607 &source,
4608 &target,
4609 &edge_type,
4610 &valid_from,
4611 &valid_to,
4612 &stamp,
4613 name,
4614 shape,
4615 &ancestry,
4616 )
4617 .await
4618 }
4619 Err(e) => Err(e),
4620 };
4621 turn.answer(responder, res);
4622 }
4623 HighPriCommand::UpsertConcept { concept, responder } => {
4624 let stamp = clock.now();
4625 let res = match check_lineages(state, conn, &[concept.branch_name()]).await {
4626 Ok(_) => upsert_concept(conn, &concept, &stamp).await,
4627 Err(e) => Err(e),
4628 };
4629 turn.answer(responder, res);
4630 }
4631 HighPriCommand::WriteBulkAtomic { edges, responder } => {
4632 // One stamp for the whole batch (D-014): the rows were asserted
4633 // by one act, and giving them different transaction times would
4634 // invent an ordering the caller never expressed.
4635 let stamp = clock.now();
4636 let res = write_edges_atomic(state, conn, &edges, &stamp).await;
4637 turn.answer(responder, res);
4638 }
4639 HighPriCommand::RebuildCurrent { responder } => {
4640 turn.answer(responder, rebuild_current(conn).await);
4641 }
4642 HighPriCommand::RegisterModel {
4643 model,
4644 dim,
4645 responder,
4646 } => {
4647 turn.answer(
4648 responder,
4649 crate::vector::register_model(conn, &model, dim).await,
4650 );
4651 }
4652 HighPriCommand::Fork {
4653 name,
4654 parent,
4655 responder,
4656 } => {
4657 // The same clock as every other write, and the same instant in
4658 // both columns: `forked_at` is a transaction-time point in the
4659 // parent's history, and the point this release can fork from is
4660 // now. See `branch::Branch::created_at` for why they are two
4661 // columns anyway.
4662 let stamp = clock.now();
4663 let res = crate::branch::fork(conn, &name, &parent, &stamp).await;
4664 // `branches` has a row it did not have. Unconditional rather
4665 // than `if res.is_ok()`: a fork that failed leaves the table
4666 // as it was, so forgetting costs one query and asserting that
4667 // it failed cleanly costs an argument (0.15.6, D-248).
4668 state.forget_lineages();
4669 turn.answer(responder, res);
4670 }
4671 }
4672 LoopCtl::Continue
4673 }
4674}
4675
4676impl LowPriCommand {
4677 /// The metrics label for this variant (T1.4). See [`HighPriCommand::kind`].
4678 fn kind(&self) -> crate::metrics::CommandKind {
4679 use crate::metrics::CommandKind as K;
4680 match self {
4681 LowPriCommand::WriteConceptsChunk { .. } => K::WriteConceptsChunk,
4682 LowPriCommand::WriteAnalyticsChunk { .. } => K::WriteAnalyticsChunk,
4683 LowPriCommand::UpsertEmbeddingChunk { .. } => K::UpsertEmbeddingChunk,
4684 LowPriCommand::BulkImportChunk { .. } => K::BulkImportChunk,
4685 LowPriCommand::Archive { .. } => K::Archive,
4686 // Its own counter since 0.12.9 (W4.3, D-152). It reported as
4687 // `K::Archive` from 0.9.0 to 0.12.8 — the budget really is shared,
4688 // but attribution is not budget, and an operator reading a long
4689 // `archive` hold could not tell whether anything had been archived.
4690 // What kept it folded was that a `CommandKind` variant was a
4691 // breaking addition; `#[non_exhaustive]` (W4.2) removed that.
4692 LowPriCommand::Rehydrate { .. } => K::Rehydrate,
4693 // Its own counter from the day it shipped, which is the whole point
4694 // of the paragraph above: `Rehydrate` spent four releases folded
4695 // into `Archive` for a reason that was never good, and the cost of
4696 // unfolding it was a rung's worth of care about declaration order.
4697 LowPriCommand::ArchiveBranch { .. } => K::ArchiveBranch,
4698 LowPriCommand::RebuildFts { .. } => K::RebuildFts,
4699 // Two kinds out of one variant since 0.13.24 (W10.5, D-197). The
4700 // command carries the flag; the counter has to carry it too, or the
4701 // budget exemption for either half is decided about both (D-168).
4702 LowPriCommand::Analyze { incremental, .. } => {
4703 if *incremental {
4704 K::Optimize
4705 } else {
4706 K::Analyze
4707 }
4708 }
4709 // Two kinds out of one variant since 0.14.16 (W12.16, D-233),
4710 // and for D-197's reason one line up: the command carries the step,
4711 // so the counter has to carry it too, or the budget exemption for
4712 // either half is decided about both. Here that is not hypothetical
4713 // — the halves want opposite answers. The swap is over budget by
4714 // construction and the fill chunks are meant to fit, so a merged
4715 // kind's `over_budget` read `N(rebuilds) + regressions` and could
4716 // not be decomposed.
4717 LowPriCommand::ShadowRebuild { step, .. } => match step {
4718 crate::integrity::ShadowStep::Swap { .. } => K::ShadowSwap,
4719 crate::integrity::ShadowStep::Begin | crate::integrity::ShadowStep::Fill { .. } => {
4720 K::ShadowRebuild
4721 }
4722 },
4723 }
4724 }
4725
4726 /// Run one background command and answer its caller.
4727 ///
4728 /// Also exhaustive. The pre-0.5.4 version was a single `LoopCtl::Continue`
4729 /// for *every* variant — every background write silently discarded, its
4730 /// caller waiting forever.
4731 async fn execute(
4732 self,
4733 conn: &libsql::Connection,
4734 clock: &dyn Clock,
4735 turn: &Turn<'_>,
4736 state: &mut ActorState,
4737 ) -> LoopCtl {
4738 match self {
4739 LowPriCommand::BulkImportChunk { chunk, responder } => {
4740 // A stamp per chunk, not per batch: the chunks commit
4741 // separately, so a shared stamp would claim a simultaneity the
4742 // storage does not have.
4743 let stamp = clock.now();
4744 turn.answer_chunk(
4745 responder,
4746 write_edges_atomic(state, conn, &chunk, &stamp).await,
4747 );
4748 }
4749 LowPriCommand::WriteConceptsChunk { chunk, responder } => {
4750 let stamp = clock.now();
4751 turn.answer_chunk(
4752 responder,
4753 write_concepts_atomic(state, conn, &chunk, &stamp).await,
4754 );
4755 }
4756 LowPriCommand::WriteAnalyticsChunk { chunk, responder } => {
4757 let stamp = clock.now();
4758 turn.answer_chunk(
4759 responder,
4760 write_annotations_atomic(conn, &chunk, &stamp).await,
4761 );
4762 }
4763 LowPriCommand::UpsertEmbeddingChunk {
4764 model,
4765 chunk,
4766 responder,
4767 } => {
4768 // No clock reading: an embedding carries no timestamp on either
4769 // axis. It is a derived artifact of a model applied to content
4770 // (Doctrine VII), and the ledger already records when the
4771 // content changed.
4772 turn.answer_chunk(
4773 responder,
4774 crate::vector::search::upsert_embedding_chunk(conn, &model, &chunk).await,
4775 );
4776 }
4777 LowPriCommand::Archive {
4778 cutoff,
4779 archive_path,
4780 responder,
4781 } => {
4782 // The archive *time*, not the cutoff. `archive_horizon` records
4783 // both and they are different facts — see `archive()` (Wave 4.5).
4784 let archived_at = clock.now();
4785 let res = archive(conn, &cutoff, &archived_at, &archive_path).await;
4786 // The session attached and detached a second database. The
4787 // statements are dropped rather than trusted to recompile —
4788 // see [`ActorState::forget_statements`].
4789 state.forget_statements();
4790 // Before the answer, so a shadow rebuild that reads the epoch on
4791 // its next turn cannot miss an archive that has already deleted
4792 // rows out from under it (T1.2).
4793 if res.is_ok() {
4794 turn.archive_committed();
4795 }
4796 turn.answer(responder, res);
4797 }
4798 LowPriCommand::ArchiveBranch {
4799 branch,
4800 archive_path,
4801 responder,
4802 } => {
4803 // The wall clock, recorded in `cold.branches.archived_at`: when
4804 // the ledger stopped knowing about the lineage. Not a ledger
4805 // fact and not on either of Doctrine II's timelines — nothing
4806 // was asserted or retired here.
4807 let archived_at = clock.now();
4808 let res = crate::temporal::archive::archive_branch(
4809 conn,
4810 &branch,
4811 &archived_at,
4812 &archive_path,
4813 )
4814 .await;
4815 // Both: this session attaches a second database *and* deletes
4816 // the lineage's row from `branches`.
4817 state.forget_everything();
4818 // `Archive`'s reason exactly: a shadow rebuild reading the epoch
4819 // on its next turn must not miss a session that has already
4820 // deleted rows out from under it (T1.2).
4821 if res.is_ok() {
4822 turn.archive_committed();
4823 }
4824 turn.answer(responder, res);
4825 }
4826 LowPriCommand::Rehydrate {
4827 ids,
4828 archive_path,
4829 responder,
4830 } => {
4831 let refs: Vec<&str> = ids.iter().map(String::as_str).collect();
4832 let res = rehydrate(conn, &refs, &archive_path).await;
4833 // Attaches, like the two archive sessions. It restores concepts
4834 // and log rows, never a lineage — `cold.branches` is read by
4835 // `archive_hint` and not written back — so the lineages stand.
4836 state.forget_statements();
4837 // Same reason as `Archive`: rehydration moves rows into `links`'
4838 // parent table, so a shadow rebuild in flight must see the epoch
4839 // move before the caller is answered (T1.2).
4840 if res.is_ok() {
4841 turn.archive_committed();
4842 }
4843 turn.answer(responder, res);
4844 }
4845 LowPriCommand::ShadowRebuild { step, responder } => {
4846 use crate::integrity::{shadow, ShadowOutcome, ShadowStep};
4847 let res = match step {
4848 ShadowStep::Begin => {
4849 shadow::begin(conn)
4850 .await
4851 .map(|build_start| ShadowOutcome::Started {
4852 build_start,
4853 epoch: turn.epoch(),
4854 })
4855 }
4856 ShadowStep::Fill { after } => shadow::fill_chunk(conn, after.as_deref())
4857 .await
4858 .map(|last| ShadowOutcome::Filled { last }),
4859 ShadowStep::Swap { build_start, epoch } => {
4860 shadow::swap(conn, &build_start, epoch, turn.epoch())
4861 .await
4862 .map(|rows| ShadowOutcome::Swapped { rows })
4863 }
4864 };
4865 turn.answer(responder, res);
4866 }
4867 LowPriCommand::RebuildFts { responder } => {
4868 let res = conn
4869 .execute(crate::schema::ddl::REBUILD_CONCEPTS_FTS, ())
4870 .await
4871 .map(|_| ())
4872 .map_err(Into::into);
4873 turn.answer(responder, res);
4874 }
4875 LowPriCommand::Analyze {
4876 incremental,
4877 responder,
4878 } => {
4879 // Both go through `query()`, not `execute()`. `PRAGMA optimize`
4880 // yields rows, and libsql's `execute()` rejects any statement
4881 // that does ("Execute returned rows") — the same trap
4882 // `configure` documents. `ANALYZE` does not yield rows, but is
4883 // issued the same way so the two arms cannot drift into needing
4884 // different call shapes for no visible reason.
4885 let sql = if incremental {
4886 crate::schema::ddl::OPTIMIZE
4887 } else {
4888 crate::schema::ddl::ANALYZE
4889 };
4890 let res = conn.query(sql, ()).await.map(|_| ()).map_err(Into::into);
4891 turn.answer(responder, res);
4892 }
4893 }
4894 LoopCtl::Continue
4895 }
4896}
4897
4898/// Close an open interval by asserting its successor (Doctrine III).
4899///
4900/// Never an `UPDATE`. The replacement row copies weight and properties from
4901/// current belief and differs only in `valid_to` and `recorded_at`, so the
4902/// original assertion survives intact and `reconstruct` at an earlier instant
4903/// still sees the interval open — which is the entire point of a bitemporal
4904/// ledger.
4905// The first of these in the crate proper (0.14.8). All nine are the edge key,
4906// two stamps and the lineage — a struct to carry them would exist for one call
4907// site and would put a name between the caller and parameters it already spells
4908// out positionally at the only place it calls this.
4909#[allow(clippy::too_many_arguments)]
4910async fn retire_edge(
4911 conn: &libsql::Connection,
4912 source: &str,
4913 target: &str,
4914 edge_type: &str,
4915 valid_from: &str,
4916 valid_to: &str,
4917 stamp: &str,
4918 branch: &str,
4919 shape: LineageShape,
4920 ancestry: &[Ancestor],
4921) -> Result<()> {
4922 // Shadow retirement: the row being closed may belong to an ancestor, and
4923 // the row written carries *this* lineage's id. See
4924 // `lineage::retire_from_resolved`.
4925 //
4926 // One statement for all three shapes since 0.15.8 (W13.3, D-250). The
4927 // trunk had its own until then, kept apart on [`LineageShape`]'s ground —
4928 // the resolved form was opaque to the planner and cost 3.0x where there
4929 // was nothing to resolve (D-220). That ground is gone rather than
4930 // overruled: a keyed `Trunk` resolution lowers to no CTEs at all, so the
4931 // statement the lowering emits *is* the one this arm used to hold, with
4932 // the lineage stamped rather than defaulted.
4933 // The ancestry follows the seven, at `RETIRE_ANCESTRY_SLOT`.
4934 let mut params: Vec<libsql::Value> = vec![
4935 source.into(),
4936 target.into(),
4937 edge_type.into(),
4938 valid_from.into(),
4939 branch.into(),
4940 valid_to.into(),
4941 stamp.into(),
4942 ];
4943 params.extend(crate::graph::lineage::ancestry_params(ancestry));
4944 let affected = conn
4945 .execute(
4946 &crate::graph::lineage::retire_from_resolved(shape, ancestry),
4947 params,
4948 )
4949 .await
4950 .map_err(DbError::Engine)?;
4951
4952 if affected == 0 {
4953 return Err(DbError::NotFound(format!(
4954 "{source} -> {target} ({edge_type}) at {valid_from}"
4955 )));
4956 }
4957 Ok(())
4958}
4959
4960async fn upsert_concept(
4961 conn: &libsql::Connection,
4962 concept: &ConceptUpsert,
4963 stamp: &str,
4964) -> Result<()> {
4965 let res = conn
4966 .execute(UPSERT_CONCEPT, concept_params(concept, stamp))
4967 .await;
4968
4969 match res {
4970 Ok(_) => Ok(()),
4971 Err(e) => Err(classify(
4972 conn,
4973 e,
4974 WriteOp::Concept {
4975 id: &concept.id,
4976 recorded_at: stamp,
4977 branch: concept.branch_name(),
4978 },
4979 )
4980 .await),
4981 }
4982}
4983
4984/// Whether this pair is the storage layer's case rather than this guard's.
4985///
4986/// Two **open** intervals overlap — they share every instant from the later
4987/// start onwards — so a naive overlap check reports them, and reporting them
4988/// here would leave `DbError::SingleOpenViolation` constructible by nothing.
4989/// That variant is the more specific error, it is enforced by
4990/// `trg_links_single_open` rather than by this function, and its field names
4991/// were ratified in §1.2. Shadowing it with a general one would be defect Q's
4992/// shape reintroduced by a fix: a typed error that no code path can produce.
4993///
4994/// So the two guards partition the space rather than overlapping it. Both open
4995/// belongs to the trigger. Everything else — open against closed, closed against
4996/// closed — is unguarded at the storage layer and belongs here. That the split
4997/// is exactly the trigger's `WHEN` clause is not a coincidence; it is the
4998/// definition of what was missing.
4999fn defer_to_single_open(proposed: &Interval, existing: &Interval) -> bool {
5000 proposed.is_open() && existing.is_open()
5001}
5002
5003/// Refuse an assertion whose valid-time interval overlaps one already recorded
5004/// for the same `(source, target, edge_type)` — **defect AA, D-060**.
5005///
5006/// `trg_links_single_open` fires only `WHEN NEW.valid_to = '9999-…'`, so it
5007/// guards the open sentinel and nothing else. Two *closed* intervals that
5008/// overlap were accepted without complaint, and `query_as_of_edges` at an
5009/// instant inside both returned one relationship as two edges.
5010///
5011/// **This runs in the write actor, which is what makes it sound.** The obvious
5012/// place is `EdgeAssertion::normalized`, and it cannot go there — `normalized`
5013/// is a pure function with no connection, and doing the read at the API boundary
5014/// instead would leave a check-then-write race between the read and the actor's
5015/// insert. Inside the actor there is one writer by construction (D-014), and for
5016/// the batch paths this runs inside the same transaction as the insert, so the
5017/// window does not exist rather than being small.
5018///
5019/// **What it does not cover, and §4.2 now says so:** raw SQL against the same
5020/// file. The storage layer permits what this API refuses, which is the honest
5021/// cost of not putting the check in a trigger. The alternative was a second
5022/// index probe inside `trg_links_single_open` on every insert — on the path
5023/// D-059 has just finished making fast — for a guarantee that only holds against
5024/// callers who were going through the actor anyway.
5025///
5026/// `valid_from <> ?4` excludes the row being re-asserted. Re-assertion at the
5027/// same `valid_from` is Doctrine III's ordinary case — a new belief about the
5028/// same interval — and is settled by the primary key and the single-open
5029/// trigger, not here.
5030/// The single-assertion path holds one statement across turns (0.15.6, D-248);
5031/// the batch path prepares one inside its own transaction and calls
5032/// [`check_prepared`] per row.
5033async fn reject_overlapping_interval(
5034 state: &mut ActorState,
5035 conn: &libsql::Connection,
5036 edge: &EdgeAssertion,
5037 shape: LineageShape,
5038) -> Result<()> {
5039 let branch = edge.branch_name().to_string();
5040 check_prepared(state.guard(conn, shape, &branch).await?, edge).await
5041}
5042
5043/// The guard's body, against a statement the caller has already prepared.
5044///
5045/// **Split out because preparing per row was worth 10.4 ms on a 90-edge chunk**
5046/// (§8.8) — the same defect D-056 and D-057 diagnosed and fixed for
5047/// `INSERT_LINK`, reintroduced by the Wave 2 guard that was written beside it.
5048/// Measured with and without the guard, on a 2,000-edge hub: 8.65 ms → 19.25 ms,
5049/// and *identical* with and without `idx_lc_open_interval`, which is what
5050/// identified preparation rather than a scan as the cost. A guard that reads an
5051/// index correctly and prepares its statement 90 times is indistinguishable, at
5052/// the call site, from one that scans.
5053///
5054/// `reset()` between rows is not optional: libsql binds and steps without
5055/// resetting, so a reused statement must be returned to its initial state.
5056///
5057/// # And once more on the way out (0.15.6, W14.3)
5058///
5059/// The reset used to be enough at the top, because the guard was compiled per
5060/// call or per chunk and dropped where it was made — the drop finalized it, and
5061/// SQLite's objection to a live statement never came up. [`ActorState`] holds
5062/// this one across turns, and the loop below can leave a cursor open on it: the
5063/// overlap arm returns from inside the `while`. A statement left mid-scan is
5064/// what makes SQLite refuse to end a transaction, so the next `Archive` — not
5065/// the next assertion — would be the thing that failed, a command and a
5066/// diagnosis apart from the code that caused it. So the scan is a function of
5067/// its own, and the statement is reset on both ways out of it.
5068async fn check_prepared(guard: &OverlapGuard, edge: &EdgeAssertion) -> Result<()> {
5069 let proposed = Interval::new(edge.valid_from.clone(), edge.valid_to.clone());
5070
5071 guard.stmt.reset();
5072 // The resolved form takes a fifth parameter, the writing lineage, and
5073 // returns what that lineage can see; the trunk form takes four and returns
5074 // the table. Binding five to the trunk statement would be an error from
5075 // libsql rather than a wrong answer, which is the failure mode to prefer.
5076 let mut rows = match guard.shape {
5077 LineageShape::Trunk => {
5078 guard
5079 .stmt
5080 .query(libsql::params![
5081 edge.source.as_str(),
5082 edge.target.as_str(),
5083 edge.edge_type.as_str(),
5084 edge.valid_from.as_str()
5085 ])
5086 .await?
5087 }
5088 LineageShape::Resolved | LineageShape::TrunkOnForked => {
5089 // The ancestry follows the five, at `GUARD_ANCESTRY_SLOT`, and is
5090 // empty for `TrunkOnForked` — a root emits no `lineage` relation.
5091 let mut params: Vec<libsql::Value> = vec![
5092 edge.source.as_str().into(),
5093 edge.target.as_str().into(),
5094 edge.edge_type.as_str().into(),
5095 edge.valid_from.as_str().into(),
5096 edge.branch_name().into(),
5097 ];
5098 params.extend(crate::graph::lineage::ancestry_params(&guard.ancestry));
5099 guard.stmt.query(params).await?
5100 }
5101 };
5102
5103 let verdict = scan_candidates(&mut rows, &proposed, edge).await;
5104 drop(rows);
5105 guard.stmt.reset();
5106 verdict
5107}
5108
5109/// The guard's loop, over candidates the statement has already produced.
5110///
5111/// Split from [`check_prepared`] so that the statement is reset on both exits
5112/// from it, including the one that returns an overlap.
5113async fn scan_candidates(
5114 rows: &mut libsql::Rows,
5115 proposed: &Interval,
5116 edge: &EdgeAssertion,
5117) -> Result<()> {
5118 while let Some(row) = rows.next().await? {
5119 let existing = Interval::new(row.get::<String>(0)?, row.get::<String>(1)?);
5120 if defer_to_single_open(proposed, &existing) {
5121 continue;
5122 }
5123 if proposed.overlaps(&existing) {
5124 return Err(DbError::OverlappingInterval {
5125 overlap: Box::new(crate::error::Overlap {
5126 source_id: edge.source.clone(),
5127 target_id: edge.target.clone(),
5128 edge_type: edge.edge_type.clone(),
5129 valid_from: edge.valid_from.clone(),
5130 valid_to: edge.valid_to.clone(),
5131 existing_from: existing.valid_from,
5132 existing_to: existing.valid_to,
5133 // This guard reads committed rows, so the interval it names
5134 // is one the caller can go and look at (D-180).
5135 within_batch: false,
5136 }),
5137 });
5138 }
5139 }
5140
5141 Ok(())
5142}
5143
5144/// The same guard applied *within* a batch, before any of it is written.
5145///
5146/// The database check cannot see rows that are not in the database yet, so a
5147/// batch carrying two overlapping intervals for one relationship would pass
5148/// every per-row check and commit the overlap in one transaction.
5149///
5150/// # Sorted and swept rather than compared pairwise (0.13.6, W7.5, D-179)
5151///
5152/// This used to compare every pair. At [`chunk_rows::EDGES`] = 90 that is
5153/// nothing, and the chunked paths are the only ones where 90 is the bound —
5154/// [`Database::write_bulk_atomic`] is exempt from [`CHUNK_BUDGET`] by contract,
5155/// so its batch is whatever the caller passed, and the quadratic term is what
5156/// made 20,000 corrections to one relationship's history cost seconds rather
5157/// than milliseconds. Sorting by `(source, target, edge_type, valid_from)` and
5158/// sweeping costs `n log n` and changes nothing a caller can observe except the
5159/// wait.
5160///
5161/// **Adjacent pairs are not sufficient, and that is the whole difficulty.** For
5162/// plain intervals they would be: sort by start, and if any two overlap then
5163/// some neighbouring two overlap. That proof needs every pair to be *eligible*,
5164/// and here two are not — identical `valid_from` is re-assertion rather than
5165/// overlap, and two open intervals belong to `trg_links_single_open`. Skip an
5166/// adjacent pair for either reason and a real overlap can hide behind it:
5167/// `[5,20)`, `[5,6)`, `[7,8)` has the first pair skipped for equal `valid_from`
5168/// and the second not overlapping, while `[5,20)` and `[7,8)` overlap plainly.
5169/// So the sweep carries the widest `valid_to` reached so far instead of looking
5170/// only backwards one step, and carries a second one restricted to closed
5171/// intervals — because an open predecessor is excluded for an open candidate
5172/// and eligible for a closed one, which are different questions with different
5173/// answers.
5174///
5175/// Equal `valid_from` is handled by advancing in runs: everything with the same
5176/// start is checked against the maxima, and only then folded into them, so the
5177/// members of a run never see each other.
5178///
5179/// The report names the *earlier* interval as the existing one, which is the
5180/// pairwise version's input order only by accident. Within a batch neither is
5181/// older in transaction time — they arrive under one stamp — so valid-time order
5182/// is the only ordering that means anything, and it is the one a reader will
5183/// assume the words carry.
5184fn reject_overlaps_within(edges: &[EdgeAssertion]) -> Result<()> {
5185 // Indices, not the edges. The batch is borrowed and its order is the order
5186 // the rows are written in; sorting it would either clone it or reorder the
5187 // caller's data, which is `estimated_bulk_hold`'s reason for grouping too.
5188 let mut order: Vec<u32> = (0..edges.len() as u32).collect();
5189 order.sort_unstable_by(|&i, &j| {
5190 let a = &edges[i as usize];
5191 let b = &edges[j as usize];
5192 (
5193 &a.source,
5194 &a.target,
5195 &a.edge_type,
5196 a.branch_name(),
5197 &a.valid_from,
5198 )
5199 .cmp(&(
5200 &b.source,
5201 &b.target,
5202 &b.edge_type,
5203 b.branch_name(),
5204 &b.valid_from,
5205 ))
5206 });
5207
5208 fn key(e: &EdgeAssertion) -> (&str, &str, &str, &str) {
5209 (
5210 e.source.as_str(),
5211 e.target.as_str(),
5212 e.edge_type.as_str(),
5213 e.branch_name(),
5214 )
5215 }
5216 let at = |k: usize| &edges[order[k] as usize];
5217
5218 let mut group = 0;
5219 while group < order.len() {
5220 let mut group_end = group + 1;
5221 while group_end < order.len() && key(at(group_end)) == key(at(group)) {
5222 group_end += 1;
5223 }
5224
5225 // The furthest `valid_to` reached by anything already swept in this key
5226 // group, and the edge it came from so the error can name it. The second
5227 // one ignores open intervals: an open candidate may not be compared
5228 // against an open predecessor, and the sentinel would otherwise win the
5229 // maximum every time and make every such pair look like an overlap.
5230 let mut widest: Option<&EdgeAssertion> = None;
5231 let mut widest_closed: Option<&EdgeAssertion> = None;
5232
5233 let mut run = group;
5234 while run < group_end {
5235 let mut run_end = run + 1;
5236 while run_end < group_end && at(run_end).valid_from == at(run).valid_from {
5237 run_end += 1;
5238 }
5239
5240 for k in run..run_end {
5241 let e = at(k);
5242 let existing = if e.valid_to == timestamp::OPEN_SENTINEL {
5243 widest_closed
5244 } else {
5245 widest
5246 };
5247 let Some(p) = existing else { continue };
5248 // `Interval::overlaps` is `max(from) < min(to)`, and the sort
5249 // has already settled the max: `p.valid_from <= e.valid_from`.
5250 // What is left is the same predicate with the maximum resolved,
5251 // and it is written out rather than allocating two `Interval`s
5252 // per row to ask the same question.
5253 if e.valid_from < p.valid_to && e.valid_from < e.valid_to {
5254 return Err(DbError::OverlappingInterval {
5255 overlap: Box::new(crate::error::Overlap {
5256 source_id: e.source.clone(),
5257 target_id: e.target.clone(),
5258 edge_type: e.edge_type.clone(),
5259 valid_from: e.valid_from.clone(),
5260 valid_to: e.valid_to.clone(),
5261 existing_from: p.valid_from.clone(),
5262 existing_to: p.valid_to.clone(),
5263 // Nothing here is in the database, and the batch is
5264 // refused whole, so nothing here ever will be. The
5265 // message has to say so (D-180).
5266 within_batch: true,
5267 }),
5268 });
5269 }
5270 }
5271
5272 for k in run..run_end {
5273 let e = at(k);
5274 if widest.is_none_or(|w| e.valid_to > w.valid_to) {
5275 widest = Some(e);
5276 }
5277 if e.valid_to != timestamp::OPEN_SENTINEL
5278 && widest_closed.is_none_or(|w| e.valid_to > w.valid_to)
5279 {
5280 widest_closed = Some(e);
5281 }
5282 }
5283
5284 run = run_end;
5285 }
5286
5287 group = group_end;
5288 }
5289
5290 Ok(())
5291}
5292
5293/// Write every edge or none, under a single stamp.
5294///
5295/// **The statement is prepared once for the whole chunk (§9, D-056).** It used to
5296/// be `tx.execute(INSERT_LINK, …)` per row, which re-prepares on every call — and
5297/// `links` carries two triggers, so each preparation compiles their bodies along
5298/// with the insert.
5299///
5300/// Measured at 500 rows: **≈62 ms → ≈37 ms, a 41% saving.** Preparation was a
5301/// large cost and *not* the dominant one, which the first guess had it as. The
5302/// residual is the triggers themselves: the same 500 rows with
5303/// `trg_links_log_insert` and `trg_links_current_sync` dropped commit in **2.96
5304/// ms**, so trigger amplification is ~92% of what remains. There is no further
5305/// win available here without changing what the ledger records, and Doctrine IV
5306/// is what says it must be recorded. See D-056 for what that implies about §9's
5307/// ≤ 3 ms budget — briefly, 2.96 ms *is* the un-amplified figure, so the budget
5308/// appears to have been set without the amplification its own preamble says is
5309/// included.
5310///
5311/// `reset()` between rows is not optional: libsql's `execute` binds and steps
5312/// without resetting, so a reused statement must be returned to its initial state
5313/// or the second row steps a completed statement.
5314async fn write_edges_atomic(
5315 state: &mut ActorState,
5316 conn: &libsql::Connection,
5317 edges: &[EdgeAssertion],
5318 stamp: &str,
5319) -> Result<usize> {
5320 if edges.is_empty() {
5321 return Ok(0);
5322 }
5323
5324 // Before the transaction opens: a batch that contradicts itself is refused
5325 // without taking the write lock at all (D-060), and a batch naming a
5326 // lineage that does not exist is refused before it can take the lock at all
5327 // (0.14.8).
5328 reject_overlaps_within(edges)?;
5329 let branches = distinct_branches(edges);
5330 let (shape, lineages) = check_lineages_with(state, conn, &branches).await?;
5331 let lineages = lineages.clone();
5332
5333 let tx = conn
5334 .transaction_with_behavior(libsql::TransactionBehavior::Immediate)
5335 .await?;
5336
5337 // Inside the transaction, so the rows this checks against cannot change
5338 // between the check and the insert.
5339 // One preparation for the whole chunk, not one per row — see
5340 // `check_prepared`, and D-056 for the same lesson learned on `INSERT_LINK`.
5341 // One per lineage the chunk names, not one per row — see
5342 // `OverlapGuard::prepare` for why the shape alone stopped being enough, and
5343 // `distinct_branches` for why this is a `Vec` of length one in every batch
5344 // this crate has seen.
5345 let mut guards = Vec::with_capacity(branches.len());
5346 for name in &branches {
5347 guards.push(OverlapGuard::prepare(&tx, shape, &lineages, name).await?);
5348 }
5349 for edge in edges {
5350 let guard = guards
5351 .iter()
5352 .find(|g| g.answers_for(shape, edge.branch_name()))
5353 .expect("a guard per distinct branch, and the row names one of them");
5354 if let Err(e) = check_prepared(guard, edge).await {
5355 // Released before the rollback: a live statement on the connection
5356 // is what makes SQLite refuse to end a transaction.
5357 drop(guards);
5358 let _ = tx.rollback().await;
5359 return Err(e);
5360 }
5361 }
5362 drop(guards);
5363
5364 let stmt = tx.prepare(INSERT_LINK).await?;
5365
5366 for edge in edges {
5367 stmt.reset();
5368 let res = stmt.execute(edge_params(edge, stamp)).await;
5369
5370 if let Err(e) = res {
5371 let typed = classify(
5372 &tx,
5373 e,
5374 WriteOp::Edge {
5375 source_id: &edge.source,
5376 target_id: &edge.target,
5377 edge_type: &edge.edge_type,
5378 },
5379 )
5380 .await;
5381 // Released before the rollback: a live statement on the connection
5382 // is exactly what makes SQLite refuse to end a transaction.
5383 drop(stmt);
5384 let _ = tx.rollback().await;
5385 return Err(typed);
5386 }
5387 }
5388
5389 drop(stmt);
5390 tx.commit().await?;
5391 Ok(edges.len())
5392}
5393
5394/// Write every concept or none, under a single stamp.
5395/// Upsert one chunk of derived annotations in a single transaction (D-041).
5396///
5397/// `stamp` is the actor's clock reading, exactly as for every other chunk — but
5398/// it lands in `computed_at`, not in a `recorded_at`, and the difference is not
5399/// cosmetic. `recorded_at` is the transaction-time axis and is subject to
5400/// Doctrine II and the monotonicity guard; `computed_at` is a note about when a
5401/// derivation last ran, on a table the ledger does not see. Rerunning an
5402/// algorithm therefore replaces the row and advances the note, rather than
5403/// versioning a concept the world did not change.
5404///
5405/// # Failures name the concept (0.13.3, W7.2, D-176)
5406///
5407/// This was the one write path in the crate that returned
5408/// [`DbError::Engine`] raw, and the omission looked harmless: the table
5409/// carries no triggers, so none of [`crate::error::AbortKind`]'s guards can
5410/// fire on it and [`classify`] would have returned the same raw error it was
5411/// given. What that reasoning missed is the foreign key onto `concepts`, which
5412/// the engine enforces itself. Annotating a concept that does not exist is the
5413/// one failure a caller can cause here, and it reported as
5414/// `FOREIGN KEY constraint failed` with no row named — out of a chunk of up to
5415/// [`chunk_rows::ANNOTATIONS`].
5416///
5417/// It now goes through [`classify`] with [`WriteOp::Annotation`] like every
5418/// other write, and a missing concept returns [`DbError::NotFound`] carrying
5419/// its id.
5420async fn write_annotations_atomic(
5421 conn: &libsql::Connection,
5422 annotations: &[Annotation],
5423 stamp: &str,
5424) -> Result<usize> {
5425 if annotations.is_empty() {
5426 return Ok(0);
5427 }
5428
5429 let tx = conn
5430 .transaction_with_behavior(libsql::TransactionBehavior::Immediate)
5431 .await?;
5432
5433 let stmt = tx
5434 .prepare(
5435 "INSERT INTO analytics_annotations (concept_id, label, value, computed_at) \
5436 VALUES (?1, ?2, ?3, ?4) \
5437 ON CONFLICT(concept_id, label) DO UPDATE SET \
5438 value = excluded.value, computed_at = excluded.computed_at",
5439 )
5440 .await?;
5441
5442 for a in annotations {
5443 stmt.reset();
5444 let res = stmt
5445 .execute(libsql::params![
5446 a.concept_id.as_str(),
5447 a.label.as_str(),
5448 a.value.as_str(),
5449 stamp
5450 ])
5451 .await;
5452 if let Err(e) = res {
5453 let typed = classify(
5454 &tx,
5455 e,
5456 WriteOp::Annotation {
5457 concept_id: &a.concept_id,
5458 },
5459 )
5460 .await;
5461 drop(stmt);
5462 let _ = tx.rollback().await;
5463 return Err(typed);
5464 }
5465 }
5466
5467 drop(stmt);
5468 tx.commit().await?;
5469 Ok(annotations.len())
5470}
5471
5472async fn write_concepts_atomic(
5473 state: &mut ActorState,
5474 conn: &libsql::Connection,
5475 concepts: &[ConceptUpsert],
5476 stamp: &str,
5477) -> Result<usize> {
5478 if concepts.is_empty() {
5479 return Ok(0);
5480 }
5481
5482 // Named lineages, before the write lock — `check_lineages`' reason, and the
5483 // shape it also returns is unused here because `concepts` is keyed by
5484 // identity and has no resolution to do (see `ConceptUpsert::branch`).
5485 let mut named: Vec<&str> = Vec::with_capacity(1);
5486 for concept in concepts {
5487 let name = concept.branch_name();
5488 if !named.contains(&name) {
5489 named.push(name);
5490 }
5491 }
5492 check_lineages(state, conn, &named).await?;
5493
5494 let tx = conn
5495 .transaction_with_behavior(libsql::TransactionBehavior::Immediate)
5496 .await?;
5497
5498 // Prepared once, like the edge chunk (D-056). This no longer routes through
5499 // [`upsert_concept`] — that function prepares per call by construction — but
5500 // it shares that function's statement text and parameter row, so the two
5501 // cannot upsert different columns.
5502 let stmt = tx.prepare(UPSERT_CONCEPT).await?;
5503
5504 for concept in concepts {
5505 stmt.reset();
5506 let res = stmt.execute(concept_params(concept, stamp)).await;
5507
5508 if let Err(e) = res {
5509 let typed = classify(
5510 &tx,
5511 e,
5512 WriteOp::Concept {
5513 id: &concept.id,
5514 recorded_at: stamp,
5515 branch: concept.branch_name(),
5516 },
5517 )
5518 .await;
5519 drop(stmt);
5520 let _ = tx.rollback().await;
5521 return Err(typed);
5522 }
5523 }
5524
5525 drop(stmt);
5526 tx.commit().await?;
5527 Ok(concepts.len())
5528}
5529
5530#[cfg(test)]
5531mod lineage_cache {
5532 //! [`Lineages::shape_of`] against every shape combination (0.15.6, W14.3).
5533 //!
5534 //! Unit tests rather than a write through the actor, because the case this
5535 //! function exists for **cannot be observed from outside**: where a batch
5536 //! names lineages of different shapes, both of them currently compile the
5537 //! same overlap statement, so a wrong choice between them is invisible
5538 //! until W13.3 gives the guard a third lowering. A behavioural test would
5539 //! pass on the code this replaces and on the code that replaces it, and
5540 //! would go on passing through the release that made it matter.
5541
5542 use super::*;
5543
5544 /// `(name, is_root)`, kept as the fixture's spelling because that is what
5545 /// these tests are about — the shape, not the ancestry. A root is a row
5546 /// with no parent and no fork point, which is the pairing the `branches`
5547 /// CHECK enforces; a non-root is given both, since a row with one and not
5548 /// the other is not a state the schema permits.
5549 fn lineages(rows: &[(&str, bool)]) -> Lineages {
5550 Lineages {
5551 rows: rows
5552 .iter()
5553 .map(|(id, root)| crate::graph::lineage::BranchRow {
5554 id: (*id).into(),
5555 parent: (!root).then(|| "main".to_string()),
5556 forked_at: (!root).then(|| "2026-01-01T00:00:00.000000Z".to_string()),
5557 })
5558 .collect(),
5559 }
5560 }
5561
5562 fn trunk_only() -> Lineages {
5563 lineages(&[("main", true)])
5564 }
5565
5566 fn forked() -> Lineages {
5567 lineages(&[("main", true), ("alt", false), ("other", false)])
5568 }
5569
5570 /// One lineage is the trunk, whoever asks.
5571 #[test]
5572 fn one_lineage_is_the_trunk() {
5573 assert_eq!(
5574 trunk_only().shape_of(&["main"]).unwrap(),
5575 LineageShape::Trunk
5576 );
5577 }
5578
5579 /// A root on a forked database is `TrunkOnForked`; anything else resolves.
5580 #[test]
5581 fn the_shape_reads_the_name_and_not_only_the_count() {
5582 let l = forked();
5583 assert_eq!(
5584 l.shape_of(&["main"]).unwrap(),
5585 LineageShape::TrunkOnForked,
5586 "a root has no ancestors and D-244 emits that reduction directly"
5587 );
5588 assert_eq!(l.shape_of(&["alt"]).unwrap(), LineageShape::Resolved);
5589 }
5590
5591 /// The case review C-24 is about: names of two different shapes.
5592 ///
5593 /// Pinned in **both orders**, which is the whole content of the finding.
5594 /// The loop this replaces returned whichever came last, so a batch naming
5595 /// `[main, alt]` and one naming `[alt, main]` disagreed about their own
5596 /// shape while describing the same set of lineages.
5597 #[test]
5598 fn a_batch_of_mixed_shapes_resolves_rather_than_taking_the_last_name() {
5599 let l = forked();
5600 assert_eq!(
5601 l.shape_of(&["main", "alt"]).unwrap(),
5602 LineageShape::Resolved,
5603 "the resolved form is exact for a root as well, which is the \
5604 argument OverlapGuard::prepare already makes"
5605 );
5606 assert_eq!(
5607 l.shape_of(&["alt", "main"]).unwrap(),
5608 LineageShape::Resolved,
5609 "and it must not depend on the order the batch happened to name them"
5610 );
5611 }
5612
5613 /// Agreement is kept rather than widened: two forks are still `Resolved`,
5614 /// and two mentions of one root are still `TrunkOnForked`.
5615 #[test]
5616 fn names_that_agree_keep_their_shape() {
5617 let l = forked();
5618 assert_eq!(
5619 l.shape_of(&["alt", "other"]).unwrap(),
5620 LineageShape::Resolved
5621 );
5622 assert_eq!(
5623 l.shape_of(&["main", "main"]).unwrap(),
5624 LineageShape::TrunkOnForked
5625 );
5626 }
5627
5628 /// Existence is checked for **every** name, not only the one that decides.
5629 ///
5630 /// The check is why the loop existed at all, and the shape was the thing it
5631 /// returned on the way past. A version that stopped at the first name would
5632 /// let a batch name a lineage that does not exist and be refused later by
5633 /// the foreign key, from inside a rolled-back transaction, naming neither
5634 /// the column nor the branch — `check_lineages`' opening paragraph.
5635 #[test]
5636 fn an_unknown_name_is_refused_wherever_it_sits() {
5637 let l = forked();
5638 for names in [
5639 ["ghost", "main"].as_slice(),
5640 ["main", "ghost"].as_slice(),
5641 ["main", "ghost", "alt"].as_slice(),
5642 ] {
5643 match l.shape_of(names) {
5644 Err(DbError::UnknownBranch(name)) => assert_eq!(name, "ghost"),
5645 other => panic!("expected UnknownBranch for {names:?}, got {other:?}"),
5646 }
5647 }
5648 }
5649}
5650
5651#[cfg(test)]
5652mod tests {
5653 use super::*;
5654
5655 fn edge(target: &str, micros: usize) -> EdgeAssertion {
5656 EdgeAssertion::new("src", target, "LINKS")
5657 .valid_from(format!("2026-01-01T00:00:00.{micros:06}Z"))
5658 .valid_to(format!("2026-01-01T00:00:00.{:06}Z", micros + 1))
5659 }
5660
5661 /// The estimate must **no longer** depend on the batch's shape (0.13.6).
5662 ///
5663 /// Its dependence on shape was correct for as long as the guard was
5664 /// quadratic and the constant differed 16× between the two paths through
5665 /// its inner loop. W7.5 removed that term, and measurement agrees: 1.94 s
5666 /// and 2.22 s for the two 20,000-edge batches that used to differ by 7×.
5667 /// A model that kept predicting a 7× spread would now be wrong in the
5668 /// *expensive* direction — warning loudly about a batch that is fine.
5669 #[test]
5670 fn two_batches_of_one_size_are_predicted_alike() {
5671 const N: usize = 20_000;
5672 let fanout: Vec<_> = (0..N).map(|i| edge(&format!("t{i:07}"), i)).collect();
5673 let history: Vec<_> = (0..N).map(|i| edge("t0", i)).collect();
5674
5675 assert_eq!(
5676 estimated_bulk_hold(&fanout),
5677 estimated_bulk_hold(&history),
5678 "the guard no longer reads the batch's shape, so neither may this"
5679 );
5680 }
5681
5682 /// Measured on libSQL 0.9.30 after W7.5: 1.94 s and 2.22 s for those two
5683 /// batches, against 2.6 s and 18.1 s before it. This pins that the model
5684 /// still tracks them — a coefficient edited without re-measuring fails here.
5685 #[test]
5686 fn the_estimate_matches_what_was_measured() {
5687 const N: usize = 20_000;
5688 let fanout: Vec<_> = (0..N).map(|i| edge(&format!("t{i:07}"), i)).collect();
5689 let history: Vec<_> = (0..N).map(|i| edge("t0", i)).collect();
5690
5691 for (batch, measured_ms, label) in
5692 [(fanout, 1_936u128, "fanout"), (history, 2_220, "history")]
5693 {
5694 let predicted = estimated_bulk_hold(&batch).as_millis();
5695 let ratio = predicted as f64 / measured_ms as f64;
5696 assert!(
5697 (0.8..1.25).contains(&ratio),
5698 "{label}: predicted {predicted} ms against a measured \
5699 {measured_ms} ms ({ratio:.2}x). Re-run \
5700 examples/bulk_atomic_diag.rs before changing the coefficients."
5701 );
5702 }
5703 }
5704
5705 /// `ilog2` panics on zero, and an empty batch is the caller asking whether
5706 /// a batch they have not built yet would be slow.
5707 #[test]
5708 fn an_empty_batch_estimates_nothing_rather_than_panicking() {
5709 assert_eq!(estimated_bulk_hold(&[]), std::time::Duration::ZERO);
5710 let one = [edge("t0", 0)];
5711 assert_eq!(
5712 estimated_bulk_hold(&one),
5713 std::time::Duration::from_nanos(7_400)
5714 );
5715 }
5716
5717 /// The model is used as a threshold test, so it must not go backwards.
5718 #[test]
5719 fn a_bigger_batch_never_predicts_a_shorter_hold() {
5720 let mut last = std::time::Duration::ZERO;
5721 for n in [1usize, 2, 3, 7, 8, 100, 511, 512, 513, 5_000, 20_000] {
5722 let batch: Vec<_> = (0..n).map(|i| edge(&format!("t{i:07}"), i)).collect();
5723 let now = estimated_bulk_hold(&batch);
5724 assert!(now >= last, "{n} rows predicts {now:?} after {last:?}");
5725 last = now;
5726 }
5727 }
5728
5729 /// The warning threshold sits well above the bound this path is exempt from.
5730 ///
5731 /// Warning at `CHUNK_BUDGET` would fire on batches working exactly as
5732 /// designed — the exemption is a contract (D-014), not a failure — and a
5733 /// warning that fires on correct behaviour gets filtered out, taking the
5734 /// 18-second case with it.
5735 #[test]
5736 fn the_warning_threshold_is_not_the_chunk_budget() {
5737 assert!(BULK_ATOMIC_WARN_HOLD > CHUNK_BUDGET * 10);
5738 }
5739
5740 // -----------------------------------------------------------------------
5741 // reject_overlaps_within — sorted and swept (0.13.6, W7.5, D-179)
5742 //
5743 // The pairwise version was obviously correct and too slow; this one is
5744 // neither, so what follows pins the cases where the obvious fix is wrong
5745 // rather than only the cases the guard already caught.
5746 // -----------------------------------------------------------------------
5747
5748 /// An edge over an explicit interval, all four key columns spelled out.
5749 fn span(target: &str, edge_type: &str, from: usize, to: Option<usize>) -> EdgeAssertion {
5750 let stamp = |n: usize| format!("2026-01-01T00:00:00.{n:06}Z");
5751 EdgeAssertion::new("src", target, edge_type)
5752 .valid_from(stamp(from))
5753 .valid_to(to.map_or_else(|| timestamp::OPEN_SENTINEL.to_string(), stamp))
5754 }
5755
5756 fn closed(from: usize, to: usize) -> EdgeAssertion {
5757 span("t0", "LINKS", from, Some(to))
5758 }
5759
5760 fn open_at(from: usize) -> EdgeAssertion {
5761 span("t0", "LINKS", from, None)
5762 }
5763
5764 /// The case that makes adjacent pairs insufficient.
5765 ///
5766 /// Sort by start and any overlap shows up between neighbours — but only if
5767 /// every neighbouring pair is eligible to be checked. `[5,20)` and `[5,6)`
5768 /// are not: identical `valid_from` is re-assertion. Skip them, and `[5,6)`
5769 /// against `[7,8)` is a clean gap, and the plain overlap between `[5,20)`
5770 /// and `[7,8)` never gets looked at.
5771 #[test]
5772 fn an_overlap_hidden_behind_an_equal_valid_from_is_still_found() {
5773 let batch = vec![closed(5, 20), closed(5, 6), closed(7, 8)];
5774 assert!(matches!(
5775 reject_overlaps_within(&batch),
5776 Err(DbError::OverlappingInterval { .. })
5777 ));
5778 }
5779
5780 /// The same trap in the other direction: skipped for being open.
5781 ///
5782 /// Two open intervals are `trg_links_single_open`'s case and are passed
5783 /// over here. A running maximum that counted them would take the sentinel
5784 /// as the widest reach and report every later open interval as overlapping
5785 /// it — inventing an error rather than missing one, which is why the sweep
5786 /// carries a second maximum restricted to closed intervals.
5787 #[test]
5788 fn two_open_intervals_are_left_to_the_trigger() {
5789 let batch = vec![closed(1, 5), open_at(10), open_at(20)];
5790 assert!(reject_overlaps_within(&batch).is_ok());
5791 }
5792
5793 /// An open interval still overlaps a closed one that reaches past its start.
5794 #[test]
5795 fn an_open_interval_over_a_closed_one_is_an_overlap() {
5796 let batch = vec![closed(1, 50), open_at(10)];
5797 assert!(matches!(
5798 reject_overlaps_within(&batch),
5799 Err(DbError::OverlappingInterval { .. })
5800 ));
5801 }
5802
5803 /// Same `valid_from`, different `valid_to`: a batch correcting itself.
5804 ///
5805 /// Last writer wins by `seq_id`, exactly as it does across batches. The
5806 /// guard has no opinion.
5807 #[test]
5808 fn equal_valid_from_is_re_assertion_not_overlap() {
5809 let batch = vec![closed(5, 20), closed(5, 6), closed(5, 900)];
5810 assert!(reject_overlaps_within(&batch).is_ok());
5811 }
5812
5813 /// Grouping is what makes the sweep sound, so it is pinned rather than read.
5814 #[test]
5815 fn edges_with_different_keys_do_not_see_each_other() {
5816 let batch = vec![
5817 span("t0", "LINKS", 1, Some(50)),
5818 span("t1", "LINKS", 10, Some(60)),
5819 span("t0", "CITES", 10, Some(60)),
5820 span("t0", "LINKS", 50, Some(60)),
5821 ];
5822 assert!(reject_overlaps_within(&batch).is_ok());
5823 }
5824
5825 /// Which of the two the report calls *existing* (0.13.6).
5826 ///
5827 /// Neither is older in transaction time — a batch lands under one stamp —
5828 /// so the pairwise version's answer was its input order, which means
5829 /// nothing. Valid-time order is the only ordering the two intervals have.
5830 #[test]
5831 fn the_report_names_the_earlier_interval_as_the_existing_one() {
5832 let batch = vec![closed(7, 8), closed(5, 20)];
5833 let Err(DbError::OverlappingInterval { overlap }) = reject_overlaps_within(&batch) else {
5834 panic!("the batch overlaps itself");
5835 };
5836 assert!(overlap.valid_from.ends_with(".000007Z"), "{overlap:?}");
5837 assert!(overlap.existing_from.ends_with(".000005Z"), "{overlap:?}");
5838 }
5839
5840 /// A guard whose answer depended on the caller's ordering would be a worse
5841 /// guard than the one it replaced, and sorting is exactly the change that
5842 /// could introduce that.
5843 #[test]
5844 fn the_answer_does_not_depend_on_the_order_the_caller_passed() {
5845 let mut batch = vec![closed(5, 20), closed(5, 6), closed(7, 8)];
5846 batch.reverse();
5847 assert!(reject_overlaps_within(&batch).is_err());
5848
5849 let mut clean = vec![closed(1, 5), closed(5, 6), closed(7, 8), open_at(8)];
5850 clean.reverse();
5851 assert!(reject_overlaps_within(&clean).is_ok());
5852 }
5853
5854 /// §2.6, and the reason the rewrite happened rather than the doc alone.
5855 ///
5856 /// Every edge shares a key, so the old loop reached `Interval::overlaps`
5857 /// on all n(n−1)/2 pairs — 50 million of them here, which is seconds even
5858 /// in release and considerably worse in the debug profile this runs under.
5859 /// The bound is loose on purpose: it is an order of magnitude, not a
5860 /// benchmark, and the only thing it can fail on is the quadratic term
5861 /// coming back.
5862 #[test]
5863 fn one_relationships_whole_history_is_no_longer_quadratic() {
5864 const N: usize = 10_000;
5865 let batch: Vec<_> = (0..N).map(|i| closed(i * 2, i * 2 + 1)).collect();
5866
5867 let started = std::time::Instant::now();
5868 assert!(reject_overlaps_within(&batch).is_ok());
5869 let took = started.elapsed();
5870
5871 assert!(
5872 took < std::time::Duration::from_secs(2),
5873 "{N} same-key edges took {took:?} in the guard"
5874 );
5875 }
5876
5877 // -----------------------------------------------------------------------
5878 // next_chunk_size — the control law (0.12.0, W2)
5879 //
5880 // All of these run without a database, a clock or an actor, which is why
5881 // W2 comes before W3: the loop that will use this function can only be
5882 // tested against a real write, and the properties below cannot be observed
5883 // there without also observing the machine.
5884 // -----------------------------------------------------------------------
5885
5886 use std::time::Duration;
5887
5888 /// `next_chunk_size` with the shipped budget and floor.
5889 fn step_to(current: usize, held_ms: f64, ceiling: usize) -> usize {
5890 next_chunk_size(
5891 current,
5892 Duration::from_nanos((held_ms * 1_000_000.0) as u64),
5893 CHUNK_BUDGET,
5894 CHUNK_FLOOR,
5895 ceiling,
5896 )
5897 }
5898
5899 /// The edge path, which is every test here that does not say otherwise.
5900 fn step(current: usize, held_ms: f64) -> usize {
5901 step_to(current, held_ms, chunk_rows::EDGES)
5902 }
5903
5904 /// Iterate the law against a machine that costs `per_row_us` per row plus a
5905 /// fixed `overhead_ms` per transaction — the two-term model D-142 measured.
5906 fn converge(
5907 start: usize,
5908 per_row_us: f64,
5909 overhead_ms: f64,
5910 ceiling: usize,
5911 steps: usize,
5912 ) -> Vec<usize> {
5913 let mut size = start;
5914 (0..steps)
5915 .map(|_| {
5916 let held = overhead_ms + per_row_us * size as f64 / 1000.0;
5917 size = step_to(size, held, ceiling);
5918 size
5919 })
5920 .collect()
5921 }
5922
5923 /// The reason the shrink is proportional rather than a halving: at 4× over
5924 /// budget, halving needs three steps and every one of them is a latency
5925 /// miss a caller can feel.
5926 ///
5927 /// Run on the annotations path, because it is the only one whose ceiling
5928 /// leaves room to start far above a size that is reachable — on the edge
5929 /// path a 4× miss lands under [`CHUNK_FLOOR`], which is a different test.
5930 #[test]
5931 fn a_chunk_far_over_budget_converges_from_above_in_at_most_two_steps() {
5932 const CEILING: usize = chunk_rows::ANNOTATIONS;
5933 let (per_row_us, overhead_ms) = (20.0, 0.05);
5934 let held = |n: usize| overhead_ms + per_row_us * n as f64 / 1000.0;
5935 assert!(
5936 held(CEILING) > 4.0 * 3.0,
5937 "the start is not far over budget"
5938 );
5939
5940 let trace = converge(CEILING, per_row_us, overhead_ms, CEILING, 4);
5941 let first_in_budget = trace
5942 .iter()
5943 .position(|&n| held(n) <= 3.0)
5944 .expect("never reached the budget");
5945 assert!(
5946 first_in_budget <= 1,
5947 "took {} steps to get under budget: {trace:?}",
5948 first_in_budget + 1
5949 );
5950 }
5951
5952 /// Growth is additive, so a size that is merely comfortable cannot leap the
5953 /// ceiling — and cannot overshoot the budget by more than a quarter.
5954 #[test]
5955 fn growth_is_slow_and_shrinking_is_fast() {
5956 let grown = step(40, 1.0);
5957 assert!(
5958 (41..=50).contains(&grown),
5959 "40 rows at 1 ms should grow by about a quarter, got {grown}"
5960 );
5961 let shrunk = step(90, 9.0);
5962 assert!(
5963 shrunk <= 40,
5964 "90 rows at 3x the budget should shrink proportionally, got {shrunk}"
5965 );
5966 }
5967
5968 /// The dead band. Between `budget / 2` and `budget` the size is right and
5969 /// moving it only costs a re-measurement; without this the law oscillates
5970 /// across the bound forever.
5971 #[test]
5972 fn a_chunk_inside_the_band_is_left_alone() {
5973 for held_ms in [1.6, 2.0, 2.5, 2.9, 3.0] {
5974 assert_eq!(step(60, held_ms), 60, "moved at {held_ms} ms");
5975 }
5976 assert_ne!(step(60, 1.4), 60, "did not grow at well under half budget");
5977 }
5978
5979 /// Both clamps, and the floor's violation stated as a test rather than only
5980 /// as a comment: a populated table drives this to `CHUNK_FLOOR` and holds it
5981 /// there **over budget**, which is [`CHUNK_FLOOR`]'s documented trade.
5982 #[test]
5983 fn the_floor_and_the_ceiling_both_hold() {
5984 // 118 µs/row + 0.03 ms fixed — the populated arm, where 35 rows is
5985 // ~4.1 ms and no size in range meets the bound.
5986 let trace = converge(chunk_rows::EDGES, 118.0, 0.03, chunk_rows::EDGES, 8);
5987 assert!(
5988 trace.iter().all(|&n| n >= CHUNK_FLOOR),
5989 "fell through the floor: {trace:?}"
5990 );
5991 assert_eq!(*trace.last().unwrap(), CHUNK_FLOOR, "settled off the floor");
5992
5993 // A free machine cannot grow past the path's constant.
5994 let fast = converge(CHUNK_FLOOR, 1.0, 0.01, chunk_rows::EDGES, 40);
5995 assert_eq!(*fast.last().unwrap(), chunk_rows::EDGES);
5996 assert!(fast.iter().all(|&n| n <= chunk_rows::EDGES));
5997 }
5998
5999 /// Zero is the one answer that cannot be recovered from: a loop asked for
6000 /// chunks of no rows makes no progress and never finishes. Degenerate
6001 /// inputs included, since `held` is a measurement and measurements arrive
6002 /// from a machine under load.
6003 #[test]
6004 fn the_law_never_returns_zero() {
6005 let cases = [
6006 (0usize, Duration::ZERO),
6007 (0, Duration::from_secs(60)),
6008 (1, Duration::from_secs(60)),
6009 (90, Duration::from_secs(3600)),
6010 (usize::MAX, Duration::from_nanos(1)),
6011 (1, Duration::ZERO),
6012 ];
6013 for (current, held) in cases {
6014 for (floor, ceiling) in [(35, 90), (1, 1), (0, 0), (90, 35)] {
6015 let n = next_chunk_size(current, held, CHUNK_BUDGET, floor, ceiling);
6016 assert!(
6017 n > 0,
6018 "returned 0 for current={current}, held={held:?}, \
6019 floor={floor}, ceiling={ceiling}"
6020 );
6021 }
6022 }
6023 }
6024
6025 /// A zero budget is not a configuration anyone should reach, but it is one
6026 /// division away from a panic, so it is pinned.
6027 #[test]
6028 fn a_zero_budget_shrinks_to_the_floor_rather_than_dividing_by_it() {
6029 assert_eq!(
6030 next_chunk_size(90, Duration::from_millis(1), Duration::ZERO, 35, 90),
6031 35
6032 );
6033 }
6034
6035 /// A panicked write actor is reported, and the report says so (W7.3, D-177).
6036 ///
6037 /// This is the branch `close()` actually has. Through 0.13.3 it sat beside
6038 /// `Ok(res) => res?` on an actor `Result` that could never be `Err`, and the
6039 /// pair looked like two failure paths under review — so the one that cannot
6040 /// fire was carried through two signatures and the one that can had no test.
6041 ///
6042 /// The `JoinError` is real rather than mocked: `JoinError` has no public
6043 /// constructor, and one built by hand would pin the mapping against a value
6044 /// tokio does not produce.
6045 #[tokio::test]
6046 async fn a_writer_that_panicked_is_reported_by_close() {
6047 // Swallow the panic's own output. The task is *meant* to panic, and a
6048 // backtrace in a green suite trains people to skim it.
6049 let prev = std::panic::take_hook();
6050 std::panic::set_hook(Box::new(|_| {}));
6051 let handle = tokio::spawn(async { panic!("the write connection is gone") });
6052 let joined = handle.await;
6053 std::panic::set_hook(prev);
6054
6055 assert!(
6056 joined.is_err(),
6057 "the task must have panicked for this to test anything"
6058 );
6059
6060 match writer_exit(joined) {
6061 Err(DbError::WriterStopped(reason)) => {
6062 assert!(
6063 reason.contains("did not exit cleanly"),
6064 "the message must say what happened: {reason}"
6065 );
6066 }
6067 other => panic!("a panicked actor must be WriterStopped, got {other:?}"),
6068 }
6069 }
6070
6071 /// An actor that ran to completion closes clean.
6072 ///
6073 /// The other half, and the one that must not acquire a failure mode by
6074 /// accident: `run_writer_actor` returns `()`, so the only way this can start
6075 /// reporting an error is if someone gives the actor a `Result` again.
6076 #[tokio::test]
6077 async fn a_writer_that_finished_normally_closes_clean() {
6078 let handle = tokio::spawn(async {});
6079 assert!(writer_exit(handle.await).is_ok());
6080 }
6081
6082 /// A token is a handle to one flag, not a value that is copied (0.13.8,
6083 /// W7.6). The clone the caller keeps and the clone the import holds have to
6084 /// be the same flag, or `cancel()` reaches nothing.
6085 #[test]
6086 fn a_cloned_token_cancels_the_original() {
6087 let token = CancelToken::new();
6088 let held_by_the_import = token.clone();
6089 assert!(!held_by_the_import.is_cancelled());
6090 token.cancel();
6091 assert!(held_by_the_import.is_cancelled());
6092 // And it stays cancelled: there is no un-cancel, deliberately, because
6093 // a token that could be reset would let a second import inherit a
6094 // decision made about the first.
6095 token.cancel();
6096 assert!(held_by_the_import.is_cancelled());
6097 }
6098
6099 /// The default control is the one the plain bulk methods pass, and it must
6100 /// never stop a write.
6101 #[test]
6102 fn the_default_control_neither_cancels_nor_reports() {
6103 let control = BulkControl::new();
6104 assert!(!control.is_cancelled());
6105 // No callback, so this is a no-op rather than a panic on an `unwrap`.
6106 control.report(BulkProgress {
6107 written: 1,
6108 total: 1,
6109 rows: 1,
6110 held: std::time::Duration::ZERO,
6111 });
6112 }
6113
6114 /// The callback receives what it was promised, once per call to `report`.
6115 #[test]
6116 fn progress_reaches_the_callback_unchanged() {
6117 let seen = Arc::new(std::sync::Mutex::new(Vec::new()));
6118 let control = BulkControl::new().on_progress({
6119 let seen = Arc::clone(&seen);
6120 move |p| seen.lock().unwrap().push(p)
6121 });
6122 let sample = BulkProgress {
6123 written: 180,
6124 total: 900,
6125 rows: 90,
6126 held: std::time::Duration::from_millis(12),
6127 };
6128 control.report(sample);
6129 assert_eq!(*seen.lock().unwrap(), vec![sample]);
6130 }
6131}