macrame/connection.rs
1use std::path::{Path, PathBuf};
2use std::sync::Arc;
3use tokio::sync::{mpsc, oneshot};
4
5use crate::error::{classify, BulkInterrupted, BulkResult, DbError, Result, WriteOp};
6use crate::graph::edge::EdgeAssertion;
7use crate::graph::lineage::{Ancestor, LineageShape, Lineages};
8use crate::integrity::{rebuild_current, RebuildReport};
9use crate::plan::ReadPlan;
10use crate::schema::migrations;
11use crate::temporal::archive::{archive, rehydrate, ArchiveReport, RehydrateReport};
12use crate::temporal::interval::Interval;
13use crate::temporal::snapshot::{self, SnapshotCadence};
14use crate::util::clock::FutureStampPolicy;
15use crate::util::clock::{Clock, SystemClock};
16use crate::util::timestamp;
17use crate::vector::ModelName;
18
19/// Rows per chunk on the background write paths (§5.1.5, D-011, D-014, D-058).
20///
21/// The Write Actor holds the sole write connection, so a single large statement
22/// blocks every other writer for its duration. Chunking bounds that stall; the
23/// cost is that a bulk import is *not* atomic across chunks, which is why
24/// all-or-nothing is [`Database::write_bulk_atomic`] — a separate entry point,
25/// with its own command on the actor's protocol — rather than a tuning
26/// parameter here.
27///
28/// # Why these are four constants and not one
29///
30/// Through 0.5.5 this was a single `CHUNK_ROWS = 1000` for all four bulk paths.
31/// The golden rule it was meant to serve is a bound on *duration* — a background
32/// chunk must commit fast enough that an interactive write queued behind it is
33/// not made to wait — and one row count cannot express one duration across paths
34/// whose measured per-row costs differ by 60× (D-058). At 1,000 rows the four
35/// paths took 3.5 ms, 24 ms, 89 ms and 143 ms: the same constant, four answers,
36/// three of them far outside the bound.
37///
38/// Each size below is derived from `benches/budgets.rs`'s `chunk_scaling`
39/// sweep against [`CHUNK_BUDGET`], then verified by measuring that size directly.
40/// They are *measurements of this machine*, not universal constants — D-055's
41/// reasoning about reference hardware applies here too, and re-deriving them on
42/// materially different storage is a `cargo bench` away.
43///
44/// # Sized for the tail, not the median
45///
46/// The first derivation solved `f + c·n = 3 ms` exactly and produced sizes whose
47/// *median* commit was 2.93 ms and whose upper estimate was 2.96 — inside the
48/// bound as reported and outside it for any chunk slower than typical. A latency
49/// bound is a statement about the chunk an unlucky interactive write actually
50/// queues behind, so these solve for ≈2.5 ms instead, leaving the remainder as
51/// headroom for the tail. That costs a few percent of throughput on the two
52/// linear paths and nothing on the two superlinear ones.
53///
54/// As measured by `chunk_budget`, each at its own size: edges **2.39 ms**,
55/// concepts **2.35 ms**, annotations **2.36 ms**, embeddings **2.06 ms**, no
56/// upper estimate above 2.42.
57///
58/// # Known limitation: these are empty-database figures
59///
60/// `chunk_budget` seeds concepts and starts with **no links and no vectors**,
61/// and D-059 established that per-row cost on the edge and embedding paths grows
62/// with the size of the structure being written, not with the chunk. The same
63/// 90-edge chunk takes **9.06 ms** into an 8,000-edge table. So the bound is met
64/// as measured here and *not* met on a populated database.
65///
66/// That gap was published as 47.7 ms until 0.10.0 and attributed to the schema
67/// defect D-059 documents. The defect was fixed by the `v5 → v6` rung and the
68/// figure was never updated. 9.08 ms is a 0.10.0 measurement, not D-059's 8.0 ms
69/// carried forward: `chunk_budget` gained a seeded arm, because until it did,
70/// nothing in the bench suite wrote a chunk into a populated table and this
71/// number was unfalsifiable. It agrees with D-059 once the session is accounted
72/// for — the empty arm read 2.69 and 2.65 ms beside it against the 2.39 ms
73/// published above, so the *ratio* is 3.4× here and 3.35× there.
74///
75/// **The residual is attributed as of 0.11.0 (D-142).** It is not the missing
76/// index, which shipped in 0.5.6; it is the `links_current` write. Dropping the
77/// three `links` insert triggers one at a time puts effectively all of the
78/// growth in `trg_links_current_sync` — the single-open guard contributes none,
79/// the log trigger and the base insert ~0.35 ms of a 4.15 ms rise — and within
80/// that trigger, 89% of the growth is maintenance of `idx_lc_traversal_cover`
81/// and `idx_lc_open_interval` rather than the upsert itself, which costs 0.49 ms
82/// run directly against the same table. Page-cache size, foreign keys and the
83/// fixture's key distribution were each tested and are each not the cause.
84///
85/// Knowing the cause does not by itself change the constant: the expensive index
86/// is D-042's covering index for the traversal, so narrowing it moves cost onto
87/// the read path it exists to protect. Re-deriving these constants against the
88/// D-088 fixture matrix is the named successor.
89///
90/// # These are ceilings as of 0.12.0, not sizes
91///
92/// D-143 re-derived all four against the D-088 matrix and the edge path came
93/// back **20** against a shipped 90 — and 20 would have been wrong at 80,000
94/// edges for the same reason 90 is wrong at 8,000, because per-row cost there
95/// grows with `links_current`. The finding was that no row count can bound a
96/// duration on such a path.
97///
98/// So the chunk loop stopped trying to pick one ahead of time. Each chunk is
99/// timed by the actor and its measured hold chooses the next size; these
100/// constants are the **largest** size that will ever be asked for, and every
101/// derivation below still applies to them as such. A path may run well under its
102/// constant on a populated database and at exactly it on an empty one, and both
103/// are the bound being met rather than a size being missed.
104pub mod chunk_rows {
105 /// Edge assertions (`bulk_import`).
106 ///
107 /// Per-row cost on this path rises with the size of `links_current`, not
108 /// with the chunk (D-059) — so cutting the chunk buys latency and costs
109 /// throughput, ~11% for 1,000 edges. An earlier version of this comment
110 /// claimed it was 3.3× *faster*; that came from multiplying eleven copies of
111 /// a chunk measured into an empty database.
112 ///
113 /// **This size does not meet the 3 ms bound on a populated database.** 90
114 /// edges into an 8,000-edge table take **9.06 ms** — measured, two sessions
115 /// at 9.08 and 9.05, against an empty-table arm of 2.69 and 2.65 beside
116 /// them (D-136).
117 ///
118 /// The reason given here until 0.10.0 — that `trg_links_single_open`'s
119 /// `EXISTS` scans the whole out-degree, "a schema defect with a proven fix,
120 /// recorded in D-059 and not applied here" — described 0.5.5. The fix *was*
121 /// applied, as the `v5 → v6` rung, and took this from 47.7 ms to ~8 ms.
122 /// What survives is the miss: the bound is still exceeded ~3×. Its cause is
123 /// no longer unknown — D-142 attributes it to `trg_links_current_sync`, and
124 /// within that to secondary-index maintenance on `links_current` — and the
125 /// guard this comment used to blame contributes **no** growth at all.
126 ///
127 /// **The constant is unchanged, and that is now a measured decision**
128 /// (D-143). Re-derived against all four D-088 shapes at 8,000 edges, they
129 /// agree that the largest size meeting the bound is **20**. It stays at 90
130 /// because 20 is the same miss at a larger population — per-row cost grows
131 /// with `links_current`, so a constant fitted at 8,000 edges is wrong at
132 /// 80,000 — while the throughput cost of turning eleven chunks into fifty
133 /// is certain and immediate (D-058). The fix is not a row count: it is for
134 /// the chunk loop to stop on elapsed time, **delivered in 0.12.0**. This
135 /// number is now the ceiling that loop starts from and never exceeds; on a
136 /// populated table it converges below it within a chunk or two.
137 ///
138 /// D-134 retired the growth claim on the neighbouring *single-assertion*
139 /// path and did not measure this one; D-136 is why this line now carries a
140 /// measurement rather than a figure quoted from 0.5.6.
141 pub const EDGES: usize = 90;
142
143 /// Concept upserts (`write_concepts`).
144 ///
145 /// Linear at ~23 µs per row, so unlike [`EDGES`] this size *is* a genuine
146 /// throughput sacrifice: 1,000-row chunks ran at 23.6 µs per row against
147 /// ~35 µs here. Paid deliberately — a 1,000-row chunk takes 24 ms, eight
148 /// times the bound.
149 pub const CONCEPTS: usize = 70;
150
151 /// Analytics annotations (`write_analytics_annotations`).
152 ///
153 /// The one path where the old constant was nearly right, and the only bulk
154 /// table with no triggers at all: ~2.5 µs per row, linear, so the bound buys
155 /// a large chunk. 1,000 rows would be 3.5 ms — over, but only just.
156 pub const ANNOTATIONS: usize = 600;
157
158 /// Embedding vectors (`upsert_embeddings`).
159 ///
160 /// The smallest by a wide margin, because DiskANN index maintenance makes an
161 /// embedding the most expensive row in the system. That cost grows with the
162 /// **corpus**, not the chunk (D-059): a fixed 30-vector chunk costs 49 µs per
163 /// vector into an empty corpus and 224 µs into an 8,000-vector one. Graph
164 /// insertion getting dearer as the graph grows is what DiskANN is, so unlike
165 /// [`EDGES`] there is nothing here to fix — but it does mean this size buys
166 /// latency at some throughput, not for free.
167 pub const EMBEDDINGS: usize = 30;
168}
169
170/// What one chunk transaction cost, reported by the actor to the caller-side
171/// chunk loop (0.12.0, W1).
172///
173/// `held` is measured **inside** the actor, around its own transaction, and
174/// therefore excludes the time the command spent queued. That exclusion is the
175/// point: queue time is what strict preemption *does*, and a controller fed
176/// `send + await` would shrink chunks as punishment for the actor correctly
177/// serving an interactive write first.
178///
179/// Crate-internal, along with the command enums that carry it. It was `pub`
180/// through 0.13.32 only because they were (D-206).
181#[derive(Debug, Clone, Copy, PartialEq, Eq)]
182pub(crate) struct ChunkOutcome {
183 /// Rows the transaction actually wrote.
184 pub rows: usize,
185 /// How long the actor held the write lock for them.
186 pub held: std::time::Duration,
187}
188
189/// A flag a caller can raise to stop a chunked bulk write (0.13.8, W7.6, D-181).
190///
191/// Cheap to clone and safe to set from any thread, which is the whole point: the
192/// task running the import is the one thing that cannot cancel it. Hand a clone
193/// to whatever *can* — a signal handler, a UI thread, a timeout task — and it
194/// takes effect at the next chunk boundary.
195///
196/// **A boundary, not an abort.** Nothing rolls back and no in-flight
197/// transaction is interrupted: the loop notices between chunks and stops
198/// sending. The chunks that committed stay committed, and
199/// [`BulkInterrupted::written`](crate::BulkInterrupted::written) says how many
200/// rows those were. That is the same per-chunk boundary
201/// [`Database::bulk_import`] already documents, so cancellation adds a reason to
202/// stop and no new failure mode.
203///
204/// Setting it after the last chunk has committed does nothing — a finished
205/// write reports success, because it succeeded.
206#[derive(Clone, Debug, Default)]
207pub struct CancelToken(Arc<std::sync::atomic::AtomicBool>);
208
209impl CancelToken {
210 /// A token that has not been cancelled.
211 pub fn new() -> Self {
212 Self::default()
213 }
214
215 /// Ask the bulk write holding a clone of this token to stop at its next
216 /// chunk boundary. Idempotent; a token never un-cancels.
217 pub fn cancel(&self) {
218 // `Relaxed` on both sides is sufficient and deliberate: nothing is
219 // published *through* this flag. The rows are ordered by the database
220 // and the chunk results by the response channel, so the only thing the
221 // reader needs is to observe the store eventually, which every ordering
222 // guarantees.
223 self.0.store(true, std::sync::atomic::Ordering::Relaxed);
224 }
225
226 /// Whether [`Self::cancel`] has been called on this token or any clone.
227 pub fn is_cancelled(&self) -> bool {
228 self.0.load(std::sync::atomic::Ordering::Relaxed)
229 }
230}
231
232/// One chunk's worth of progress, handed to the callback on
233/// [`BulkControl::on_progress`] (0.13.8, W7.6).
234///
235/// Reported *after* the chunk has committed, so `written` is a count of rows
236/// that are in the database and will stay there even if the next chunk fails.
237#[derive(Debug, Clone, Copy, PartialEq, Eq)]
238#[non_exhaustive]
239pub struct BulkProgress {
240 /// Rows committed so far, across every chunk including this one.
241 pub written: usize,
242 /// Rows in the batch the caller passed. `written` reaching this means the
243 /// last chunk has committed.
244 pub total: usize,
245 /// Rows this chunk wrote. Not a constant: the loop resizes chunks against
246 /// [`CHUNK_BUDGET`] as it measures them (D-058).
247 pub rows: usize,
248 /// How long the actor held the write lock for this chunk — the same figure
249 /// the controller steers on. Measured inside the actor, around its own
250 /// transaction, so it excludes the time the command spent queued.
251 pub held: std::time::Duration,
252}
253
254/// Cancellation and progress for the four chunked bulk paths (0.13.8, W7.6,
255/// D-181).
256///
257/// Default is "neither", which is what [`Database::bulk_import`] and its three
258/// siblings pass. The `_with` variants take one of these:
259///
260/// ```no_run
261/// # use macrame::{BulkControl, CancelToken, Database};
262/// # async fn f(db: &Database, edges: Vec<macrame::prelude::EdgeAssertion>) {
263/// let token = CancelToken::new();
264/// let stopper = token.clone();
265/// tokio::spawn(async move {
266/// tokio::time::sleep(std::time::Duration::from_secs(30)).await;
267/// stopper.cancel();
268/// });
269///
270/// let control = BulkControl::new()
271/// .cancel_with(token)
272/// .on_progress(|p| println!("{}/{} rows", p.written, p.total));
273///
274/// match db.bulk_import_with(edges, control).await {
275/// Ok(n) => println!("imported {n}"),
276/// Err(e) => println!("stopped after {}: {}", e.written, e.cause),
277/// }
278/// # }
279/// ```
280///
281/// **The callback runs on the importing task, between chunks.** It is therefore
282/// on the critical path: whatever it does is time the next chunk is not being
283/// sent in. Printing or updating a counter is what it is for; a blocking write
284/// is not, and neither is anything that calls back into the same `Database`,
285/// which would deadlock the loop against a channel it is itself draining.
286#[derive(Default, Clone)]
287pub struct BulkControl {
288 cancel: Option<CancelToken>,
289 on_progress: Option<Arc<dyn Fn(BulkProgress) + Send + Sync>>,
290}
291
292impl std::fmt::Debug for BulkControl {
293 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
294 f.debug_struct("BulkControl")
295 .field("cancel", &self.cancel)
296 .field("on_progress", &self.on_progress.is_some())
297 .finish()
298 }
299}
300
301impl BulkControl {
302 /// Neither cancellation nor progress — what the plain bulk methods pass.
303 pub fn new() -> Self {
304 Self::default()
305 }
306
307 /// Stop at the next chunk boundary when `token` is cancelled.
308 pub fn cancel_with(mut self, token: CancelToken) -> Self {
309 self.cancel = Some(token);
310 self
311 }
312
313 /// Call `f` after every chunk commits. See the note on [`BulkControl`]
314 /// about what this closure is allowed to do.
315 pub fn on_progress(mut self, f: impl Fn(BulkProgress) + Send + Sync + 'static) -> Self {
316 self.on_progress = Some(Arc::new(f));
317 self
318 }
319
320 fn is_cancelled(&self) -> bool {
321 self.cancel.as_ref().is_some_and(CancelToken::is_cancelled)
322 }
323
324 fn report(&self, progress: BulkProgress) {
325 if let Some(f) = &self.on_progress {
326 f(progress);
327 }
328 }
329}
330
331/// Smallest chunk the adaptive loop will fall to (0.12.0, W2).
332///
333/// # A floor is a deliberate, measured violation of [`CHUNK_BUDGET`]
334///
335/// Feedback alone converges to whatever size meets the budget, and on a
336/// populated `links` table that size keeps falling — per-row cost there grows
337/// with the table (D-059, D-142), so there is no size at which the *fixed* cost
338/// of a transaction stops dominating. Left unbounded the loop reaches chunks of
339/// one or two rows, where nearly all the work is `BEGIN`/`COMMIT` and the import
340/// no longer finishes.
341///
342/// 35 is measured, and **re-measured against the loop that uses it** — the
343/// difference matters, because the figure this constant shipped with was an
344/// extrapolation. `examples/chunk_matrix.rs -- converge` runs a 900-edge
345/// `bulk_import` into each of the four D-088 shapes at 8,000 edges and reports
346/// the actor's own per-transaction readings. A 35-row chunk costs **3.11–3.43 ms**
347/// across the four shapes, two sessions, excluding the run-up. The floor misses
348/// the 3 ms bound by 0.1–0.4 ms, not by the ~1.1 ms predicted from the sweep.
349///
350/// The miss is **steady state** — not a one-chunk transient on the way down —
351/// and the defense is the argument [`CHUNK_BUDGET`] is answerable to rather than
352/// the number itself: an interactive assertion arriving at the worst moment
353/// waits ~3.2 ms for the chunk in flight and then runs its own ≤ 5 ms write, so
354/// ~8.2 ms against a 16.7 ms frame.
355///
356/// What the same measurement says about the *size*: on this path at this
357/// population the loop goes `[90, 35, 35, …]` on all four shapes and never picks
358/// anything between. The proportional shrink from a 90-row chunk proposes ~31
359/// rows, which clamps here — so on the edge path the floor is not a safety net
360/// under the controller, it **is** the operating point, and this number is
361/// carrying more weight than a backstop normally would. Re-measure it, not the
362/// controller, when the edge path's per-row cost changes.
363const CHUNK_FLOOR: usize = 35;
364
365/// Size of the next chunk, from what the last one cost (0.12.0, W2).
366///
367/// Pure on purpose — no clock, no database, no actor — so the control law can be
368/// tested for the properties that matter without a fixture. Three regimes:
369///
370/// | last hold | response | why |
371/// |---|---|---|
372/// | over `budget` | shrink to `current · budget / held`, × 0.9 | back off *fast* from a bound already being exceeded; the 0.9 undershoots so the correction does not have to be repeated |
373/// | under `budget / 2` | grow by a quarter of `current`, at least one row | approach the bound *slowly*; the dead band above it stops a size that is merely comfortable from oscillating |
374/// | otherwise | hold | in band, and moving costs more than it buys |
375///
376/// The asymmetry is the whole design. Proportional shrinking converges from
377/// above in one or two steps, which matters because every step over budget is a
378/// latency miss a caller can feel; additive growth cannot overshoot by more than
379/// 25%, which matters because the ceiling is a throughput preference and not a
380/// bound.
381///
382/// `ceiling` is the path's [`chunk_rows`] constant, which is why those constants
383/// keep their values and their derivations: they are no longer the size, they
384/// are the largest size this will ever ask for. `floor` is [`CHUNK_FLOOR`] —
385/// see there for the budget it knowingly misses.
386///
387/// Never returns 0, at any input, including `held == 0` or `current == 0`.
388fn next_chunk_size(
389 current: usize,
390 held: std::time::Duration,
391 budget: std::time::Duration,
392 floor: usize,
393 ceiling: usize,
394) -> usize {
395 let held = held.as_nanos().max(1);
396 let budget_ns = budget.as_nanos().max(1);
397 let current = current.max(1);
398
399 let next = if held > budget_ns {
400 // Integer math, and the `max(1)` matters: a chunk 200× over budget
401 // would otherwise propose 0 and the loop would stop making progress.
402 let scaled = (current as u128) * budget_ns * 9 / (held * 10);
403 (scaled as usize).max(1)
404 } else if held * 2 < budget_ns {
405 // Saturating because `current` is a `usize` and this is the one branch
406 // that adds to it. Nothing sane reaches the boundary; the clamp below
407 // makes the answer correct anyway rather than a debug panic.
408 current.saturating_add((current / 4).max(1))
409 } else {
410 current
411 };
412
413 // Applied last and unconditionally, so a caller that passes a reversed pair
414 // gets the floor rather than a panic — and `max(1)` last of all, because a
415 // chunk of zero rows is the single answer no loop can make progress from.
416 next.clamp(floor.min(ceiling), ceiling).max(1)
417}
418
419/// The latency bound [`chunk_rows`] is derived from (§5.1.5, D-058).
420///
421/// This is the golden rule's actual content. §9 has carried it as a row count
422/// with a duration attached — "chunk commit, 500 rows ≤ 3 ms" — which reads as
423/// two requirements and is one: the duration is the requirement, and the row
424/// count is whatever satisfies it on a given path and machine.
425///
426/// 3 ms is §9's number, kept rather than renegotiated. What it buys, end to end:
427/// an interactive assertion arriving at the worst possible moment waits for the
428/// chunk in flight (≤ 3 ms — the SQLite write lock is not preemptible, so
429/// priority buys the *next* turn and not this one) and then runs its own write
430/// (≤ 5 ms, §9), so ≤ 8 ms
431/// worst case. That fits inside a 60 Hz frame with room, which is the standard
432/// this bound is ultimately answerable to.
433///
434/// # Some operations are exempt, and the exemption is a contract, not an oversight
435///
436/// This was recorded in three separate rustdoc notes and nowhere near the bound
437/// itself, which is where a reader looks for its scope (§8.6). Stated here, with
438/// Wave 3's measurements:
439///
440/// | Path | Bound | Why it cannot be chunked |
441/// |---|---|---|
442/// | [`Database::write_bulk_atomic`] | none — caller-sized `Vec` | D-014: the batch is *one act* under one stamp. Splitting it is the thing the method exists not to do |
443/// | [`Database::archive`] | measured **26.8 ms** for 2,000 archivable edges; see [`Database::archive_windowed`] | D-012: copy-then-delete must be atomic, or a crash between the phases duplicates or loses rows |
444/// | `rebuild_current` | measured **24.6 / 104 / 318 ms** at 4K / 16K / 40K rows in `links` (was "~50 s per 10M edges", which nothing had measured) | D-023: the window between `DELETE` and `INSERT` is the whole of current belief; a reader landing in it sees a graph with no edges and no error |
445/// | [`Database::rehydrate`] | unmeasured; a function of how many rows the caller named | D-012 backwards: the same copy-then-delete atomicity, in the other direction. **A row here since 0.12.9 only because it was previously invisible** — rehydration reported as `archive` and inherited its exemption without anyone deciding on it (W4.3, D-152) |
446/// | [`Database::archive_branch`] | unmeasured; a function of how much one lineage wrote | D-012 again, and D-230's chain: the links, the log entries and the `branches` row leave together or the ledger disagrees with itself about what is currently believed. There is no smaller unit — half a forgotten lineage is a lineage whose reads are answered by its parent |
447/// | the swap turn of [`Database::rebuild_current_chunked`], counted as `shadow_swap` | measured **46.8 ms** at the largest fixture (D-082), and it grows with the table | Index names are global and SQLite has no `ALTER INDEX … RENAME`, so the shadow cannot carry `idx_lc_traversal_cover` while the live table still holds it — all three indexes are built here, under the lock. This is the residual T1.2 could not remove, and there is no smaller unit: half a swapped projection is not a projection. **Exempt since 0.14.16** (W12.16, D-233). The *fill* half keeps its own kind and is deliberately absent from this table, which is what makes a violation there a regression rather than a constant |
448/// | [`Database::checkpoint`] | a function of the WAL's size, which is a function of how long since the last checkpoint — not of anything the caller passes | It is not a transaction at all. `PRAGMA wal_checkpoint` copies frames back into the main file and there is no unit smaller than the frame it is already working in; the caller asked for exactly this, and the alternative to a long checkpoint is a WAL that keeps growing (0.12.13, W5.2, D-156) |
449/// | the drop turn of [`Database::bulk_embeddings`], counted as `drop_embedding_index` (0.16.2, D-276) | µs-scale; one `DROP INDEX IF EXISTS` | One statement, no smaller unit — the same shape as [`Database::checkpoint`] by nature and [`Database::write_bulk_atomic`] by atomicity. Its kind exists for attribution beside `rebuild_embedding_index`, not for cost |
450/// | the rebuild turn of [`Database::bulk_embeddings`], counted as `rebuild_embedding_index` (0.16.2, D-276) | measured **2.61 / 19.7 / 39.0 s** for 2,000 vectors at dim 64 / 256 / 512, ~10 ms/vector at dim 256, growing with the corpus | One `CREATE INDEX` over the whole table — the one-pass DiskANN build is indivisible, exactly the criterion `shadow_swap` and `rebuild_current` meet. The difference is schedule: this hold is caller-scheduled and opt-in, so the docstring states the number instead of arguing it. Counted would add a permanent `N(bulk loads)` to every database that ever bulk-embedded — `shadow_swap`'s own argument, unchanged |
451/// | the toggle turn of [`Database::bulk_import_deferred`], counted as `links_current_mirror` (0.16.3, D-277) | µs-scale; one DDL statement either direction | Same shape as `drop_embedding_index`: one statement, no smaller unit, and its kind exists for attribution beside the load it wraps. The window's *cost* is the chunked rebuild that follows, which keeps its own counted kinds rather than hiding behind the toggle's exemption — the toggle is not where the time goes |
452///
453/// The `archive` figure is end-to-end through this method, so it **includes**
454/// the re-derivation `archive()` runs inside its transaction — but it does not
455/// attribute it, and until D-077 more than half of that re-derivation was an
456/// audit comparing `links_current` against the query that had just filled it.
457/// Note also which variable that cost scales with: `rebuild_within` reprojects
458/// **all of `links`**, so the archive's repair term grows with the *surviving*
459/// table and not with the batch being archived. A budget stated per "100K closed
460/// intervals" ([§9](../docs/architecture/s6-s10-flows-to-dependencies.md)) is
461/// therefore parameterised on the wrong quantity.
462///
463/// The first four are atomic **by contract**, which is why "cap the batch" and
464/// "add a third tier" were both considered and neither was taken: capping breaks the
465/// guarantee the operation exists to provide, and a third tier changes which
466/// caller waits without changing how long the lock is held. What was wrong was
467/// never the exemption — it was that the bound was stated as though it had none.
468///
469/// A caller who needs the latency bound and not the atomicity has
470/// [`Database::bulk_import`], which is the same write chunked at
471/// [`chunk_rows::EDGES`] and explicitly *not* atomic overall (D-011).
472///
473/// # One of them is no longer unbounded (T1.1, D-080)
474///
475/// `archive` was the worst of them, because its hold is a function of *how long
476/// since the last archive* rather than of anything the caller chose.
477/// [`Database::archive_windowed`] runs the same work as N sessions, each
478/// atomic, each its own actor turn. Measured on an 8,000-key fixture with four
479/// generations of superseded history: the longest single hold falls from
480/// **3.3 s to 0.77 s** at one-hour windows, for total wall time that is flat
481/// within this cycle's noise.
482///
483/// The same measurement at 2,000 keys goes the other way — the hold falls
484/// 260 ms → 117 ms while total time rises 260 ms → 671 ms — so windowing is a
485/// trade and not a free improvement. It pays when the backlog is large, which
486/// is when the unwindowed hold is a problem in the first place. `archive` is
487/// kept, not deprecated, for exactly that reason.
488pub const CHUNK_BUDGET: std::time::Duration = std::time::Duration::from_millis(3);
489
490/// Predicted hold above which [`Database::write_bulk_atomic`] warns (T1.3).
491///
492/// 250 ms is fifteen frames at 60 Hz: not a hitch, a visible freeze. It is well
493/// above [`CHUNK_BUDGET`] on purpose — this path is exempt from that bound by
494/// contract, so warning at 3 ms would fire on batches that are working exactly
495/// as designed and train the reader to filter the message out.
496pub const BULK_ATOMIC_WARN_HOLD: std::time::Duration = std::time::Duration::from_millis(250);
497
498/// Roughly how long [`Database::write_bulk_atomic`] will hold the actor for
499/// this batch (T1.3, D-081; re-fitted 0.13.6, W7.5, D-179).
500///
501/// # Two terms, and the batch's shape is no longer one of them
502///
503/// T1.3 asks for "rows × measured per-row cost". Through 0.13.5 that was wrong
504/// in a way worth a paragraph: `write_edges_atomic` opened with a
505/// `reject_overlaps_within` that compared **every pair**, and the quadratic
506/// term's constant depended on the batch's *shape* rather than its size, so two
507/// 20,000-edge batches held the actor for **2.6 s** and **18.1 s** — a size-only
508/// model was off by 7× between them, in the under-predicting direction.
509///
510/// W7.5 sorts and sweeps instead, and the 18.1 s batch now holds for **2.2 s**.
511/// The shape term is gone from the code and therefore from here: measured on
512/// the same machine, the two shapes are within 15% of each other at every size
513/// from 100 to 20,000 rows, which is inside the noise this model claims.
514///
515/// What is left is not flat either, and the second term is why. Per-row cost
516/// rises from ~36 µs at 100 rows to ~111 µs at 20,000, because each insert
517/// maintains indexes and two triggers against a table the batch is itself
518/// growing:
519///
520/// ```text
521/// hold ≈ rows · (7.4 µs + 7.24 µs · ⌊log₂ rows⌋)
522/// ```
523///
524/// # What this is calibrated against, and where it will be wrong
525///
526/// libSQL 0.9.30, one machine, best of three, 100–20,000 rows in both shapes;
527/// within 15% from 500 rows up. Below that it under-predicts by up to 3×, which
528/// is harmless in the same way the old 3× over-prediction was — nothing that
529/// small approaches [`BULK_ATOMIC_WARN_HOLD`].
530///
531/// **The log term reads the batch because the batch is all it has.** It stands
532/// for the depth of a structure the batch is loading, and this signature never
533/// sees the table. That is exact for the bulk import this warns about, and
534/// optimistic for a small batch appended to an already-large table — the same
535/// blind spot the flat per-row model had, now visible instead of averaged away.
536///
537/// It is machine-specific and says nothing about disk. It exists to turn
538/// "uncapped" into an order of magnitude a caller can act on, and should not be
539/// read more precisely than that. `examples/bulk_atomic_diag.rs` prints
540/// predicted against measured, so the model's drift is visible rather than
541/// assumed.
542pub fn estimated_bulk_hold(edges: &[EdgeAssertion]) -> std::time::Duration {
543 let rows = edges.len() as u64;
544 if rows == 0 {
545 return std::time::Duration::ZERO;
546 }
547
548 // Nanoseconds throughout, saturating: a caller who passes a batch large
549 // enough to overflow this has a problem the arithmetic cannot express, and
550 // saturating to ~584 years still crosses every threshold above.
551 let per_row = 7_400u64.saturating_add((rows.ilog2() as u64).saturating_mul(7_240));
552 std::time::Duration::from_nanos(rows.saturating_mul(per_row))
553}
554
555/// Most sessions [`Database::archive_windowed`] will run for one call (T1.1).
556///
557/// A limit exists because the session count is a function of *transaction-time
558/// span divided by window*, and both come from the caller — a one-second window
559/// over a decade of history is ten million actor turns, each opening a
560/// transaction and writing a horizon row. That is not a slow archive, it is a
561/// caller who meant something else.
562///
563/// 4,096 is chosen against the operation it bounds rather than against a clock:
564/// at the measured 26.8 ms for a session with work in it, a full run of this
565/// many is about two minutes of background writing, and the whole point of
566/// windowing is that those two minutes are interruptible. It is a refusal
567/// rather than a clamp — see [`DbError::ArchiveWindow`] for why.
568pub const MAX_ARCHIVE_SESSIONS: usize = 4_096;
569
570/// A concept assertion: the payload of an upsert.
571///
572/// `#[non_exhaustive]` since 0.14.8 for
573/// [`EdgeAssertion`]'s reason: `branch` is the
574/// first field added since it was written, and one break is better than a
575/// recurring one.
576#[derive(Debug, Clone, PartialEq)]
577#[non_exhaustive]
578pub struct ConceptUpsert {
579 pub id: String,
580 pub title: String,
581 pub content: String,
582 pub embedding_model: Option<String>,
583 pub valid_from: String,
584 pub valid_to: String,
585 pub retired: bool,
586 /// The lineage this concept is minted on, or `None` for the trunk (§15.2,
587 /// D-225).
588 ///
589 /// **The rule here is narrower than the edge's, and it is the schema's
590 /// rather than this crate's.** `concepts` is a current-state projection
591 /// keyed by identity — `id` is `NOT NULL UNIQUE` — so two lineages holding
592 /// different beliefs about one concept is two rows with one `id`, which the
593 /// unique index refuses on its own. `trg_concepts_cross_lineage` turns that
594 /// refusal into [`DbError::CrossLineage`] so it says which rule was broken.
595 ///
596 /// So a branch **inherits** its parent's concepts and cannot restate them;
597 /// what this field is for is a concept the branch *mints*, which is the
598 /// case the trunk has no row for. A branch that needs to disagree with its
599 /// parent about a concept's content is asking for the overlay design, which
600 /// is deferred with its reopen trigger named (D-214).
601 pub branch: Option<crate::branch::BranchId>,
602}
603
604impl ConceptUpsert {
605 pub fn new(id: impl Into<String>, title: impl Into<String>) -> Self {
606 Self {
607 id: id.into(),
608 title: title.into(),
609 content: String::new(),
610 embedding_model: None,
611 valid_from: String::new(),
612 valid_to: timestamp::OPEN_SENTINEL.to_string(),
613 retired: false,
614 branch: None,
615 }
616 }
617
618 pub fn content(mut self, content: impl Into<String>) -> Self {
619 self.content = content.into();
620 self
621 }
622
623 pub fn embedding_model(mut self, model: impl Into<String>) -> Self {
624 self.embedding_model = Some(model.into());
625 self
626 }
627
628 pub fn valid_from(mut self, ts: impl Into<String>) -> Self {
629 self.valid_from = ts.into();
630 self
631 }
632
633 pub fn valid_to(mut self, ts: impl Into<String>) -> Self {
634 self.valid_to = ts.into();
635 self
636 }
637
638 /// Mint this concept on `branch` rather than on the trunk (0.14.8).
639 ///
640 /// See [`branch`](Self::branch) for why a branch may mint a concept and may
641 /// not restate one it inherited.
642 pub fn on_branch(mut self, branch: crate::branch::BranchId) -> Self {
643 self.branch = Some(branch);
644 self
645 }
646
647 /// The lineage this upsert names, spelled out. See
648 /// [`EdgeAssertion::branch_name`](crate::graph::EdgeAssertion).
649 pub(crate) fn branch_name(&self) -> &str {
650 self.branch
651 .as_ref()
652 .map_or(crate::schema::ddl::MAIN_BRANCH, |b| b.as_str())
653 }
654
655 pub fn retired(mut self, retired: bool) -> Self {
656 self.retired = retired;
657 self
658 }
659
660 /// Put the timestamps in canonical form (D-029) before they cross the channel.
661 pub fn normalized(mut self) -> Result<Self> {
662 crate::util::ids::validate_id(&self.id)?;
663 self.valid_from = timestamp::normalize(&self.valid_from)?;
664 self.valid_to = timestamp::normalize(&self.valid_to)?;
665 Ok(self)
666 }
667}
668
669/// One derived analytics result for one concept (§5.4, D-041).
670///
671/// Not a `ConceptUpsert`. The distinction is the whole of D-041: a concept
672/// upsert is a statement about the world and belongs in the ledger, while an
673/// annotation is a function of an algorithm applied to a graph and belongs in
674/// `analytics_annotations`, which carries no log trigger. Writing one as the
675/// other overwrote the concept's `content` with the label and recorded every
676/// analytics rerun as a fresh version of the world.
677#[derive(Debug, Clone, PartialEq, Eq)]
678#[non_exhaustive]
679pub struct Annotation {
680 pub concept_id: String,
681 /// Namespaced by convention, e.g. `louvain.community`, `kcore.shell`.
682 pub label: String,
683 /// JSON-encoded payload. Opaque to this crate.
684 pub value: String,
685}
686
687impl Annotation {
688 pub fn new(
689 concept_id: impl Into<String>,
690 label: impl Into<String>,
691 value: impl Into<String>,
692 ) -> Self {
693 Self {
694 concept_id: concept_id.into(),
695 label: label.into(),
696 value: value.into(),
697 }
698 }
699}
700
701/// Commands sent to the Write Actor on the high-priority channel (UI-driven work).
702pub(crate) enum HighPriCommand {
703 AssertEdge {
704 edge: EdgeAssertion,
705 responder: oneshot::Sender<Result<()>>,
706 },
707 RetireEdge {
708 source: String,
709 target: String,
710 edge_type: String,
711 valid_from: String,
712 valid_to: String,
713 /// The lineage doing the retiring, or `None` for the trunk (0.14.8).
714 branch: Option<crate::branch::BranchId>,
715 responder: oneshot::Sender<Result<()>>,
716 },
717 UpsertConcept {
718 concept: ConceptUpsert,
719 responder: oneshot::Sender<Result<()>>,
720 },
721 WriteBulkAtomic {
722 edges: Vec<EdgeAssertion>,
723 responder: oneshot::Sender<Result<usize>>,
724 },
725 RebuildCurrent {
726 responder: oneshot::Sender<Result<RebuildReport>>,
727 },
728 /// Create a model's embedding table and its DiskANN index (D-037, D-048).
729 ///
730 /// High priority despite being setup work: it is one small transaction, and
731 /// every embedding write for the model blocks on it, so queueing it behind a
732 /// bulk job would stall the thing it gates.
733 RegisterModel {
734 model: ModelName,
735 dim: usize,
736 responder: oneshot::Sender<Result<()>>,
737 },
738 /// Move WAL frames back into the main database file (§4.5, F-30, D-156).
739 ///
740 /// High priority, and for once the reason is not latency: a caller asking
741 /// for a checkpoint is asking for it *now*, usually at the end of a bulk
742 /// load or before taking a copy of the file, and queueing it behind the
743 /// background work it was meant to follow inverts the intent. It is also
744 /// the only command here that is not a transaction.
745 Checkpoint {
746 responder: oneshot::Sender<Result<CheckpointReport>>,
747 },
748 /// Register a lineage (0.14.7, §15.4).
749 ///
750 /// High priority, and not because it is urgent: it is one insert into a
751 /// table with no secondary indices, so it is the cheapest turn the actor
752 /// takes. What makes it high priority is that everything the caller does
753 /// next is a write *on* this branch, and queueing a fork behind a bulk
754 /// import would stall the work it exists to enable — `RegisterModel`'s
755 /// argument, for the same reason.
756 ///
757 /// It goes through the actor rather than the read connection for the
758 /// ordinary reason every write does, plus one specific to it: the duplicate
759 /// and parent checks are only sound if nothing can register a colliding
760 /// name between the check and the insert, and the actor is what makes the
761 /// pair one turn.
762 Fork {
763 name: crate::branch::BranchId,
764 parent: crate::branch::BranchId,
765 responder: oneshot::Sender<Result<crate::branch::Branch>>,
766 },
767 Shutdown {
768 responder: oneshot::Sender<Result<()>>,
769 },
770}
771
772/// What `PRAGMA wal_checkpoint` returned (0.12.13, W5.2, D-156).
773///
774/// The three columns SQLite gives back, named, rather than `()` — a checkpoint
775/// that did nothing and a checkpoint that reclaimed a 400 MB WAL are the same
776/// `Ok(())`, and the difference is the entire reason a caller asked.
777#[derive(Debug, Clone, Copy, PartialEq, Eq)]
778#[non_exhaustive]
779pub struct CheckpointReport {
780 /// `true` when SQLite could not complete the requested mode because a
781 /// reader or writer was in the way.
782 ///
783 /// **This is not an error, and it is not ignorable.** `TRUNCATE` waits for
784 /// readers only as long as `busy_timeout` allows; past that it gives up and
785 /// says so, having possibly still copied frames. A caller checkpointing
786 /// before copying the file away must read this, because a busy checkpoint
787 /// means the main file is not self-contained yet.
788 pub busy: bool,
789 /// Frames left in the WAL at the end. `0` when the checkpoint completed,
790 /// since the mode run is `TRUNCATE`.
791 pub log_frames: u64,
792 /// Frames moved back into the database file.
793 ///
794 /// Read from a `FULL` pass rather than from the `TRUNCATE` — see
795 /// `run_checkpoint` for why a truncating checkpoint cannot report this
796 /// number itself.
797 pub checkpointed_frames: u64,
798}
799
800impl CheckpointReport {
801 /// The WAL was fully reclaimed: nothing blocked, and nothing is left.
802 pub fn is_complete(&self) -> bool {
803 !self.busy && self.log_frames == 0
804 }
805}
806
807/// Commands sent to the Write Actor on the low-priority channel (background work).
808pub(crate) enum LowPriCommand {
809 /// One chunk of **concepts** — a ledger write, logged and versioned.
810 WriteConceptsChunk {
811 chunk: Vec<ConceptUpsert>,
812 responder: oneshot::Sender<Result<ChunkOutcome>>,
813 },
814 /// One chunk of **derived annotations** — off-ledger, no log trigger (D-041).
815 ///
816 /// The pair is named apart deliberately: this variant was `WriteAnalyticsChunk`
817 /// beside a `WriteAnnotationsChunk` that carried concepts, which is the
818 /// crossing D-075 undid.
819 WriteAnalyticsChunk {
820 chunk: Vec<Annotation>,
821 responder: oneshot::Sender<Result<ChunkOutcome>>,
822 },
823 /// One chunk of vectors for one model (§5.9, D-048).
824 ///
825 /// Low priority: embedding is bulk derived work and must never preempt an
826 /// interactive assertion.
827 UpsertEmbeddingChunk {
828 model: ModelName,
829 chunk: Vec<(String, Vec<f32>)>,
830 responder: oneshot::Sender<Result<ChunkOutcome>>,
831 },
832 /// Drop a model's DiskANN index — bulk-embedding setup (D-276).
833 ///
834 /// Low priority like the chunk it serves: it is a caller's bulk load that
835 /// wants the index gone, and nothing interactive should queue behind it.
836 /// One `DROP INDEX IF EXISTS` statement, atomic by nature.
837 DropEmbeddingIndex {
838 model: ModelName,
839 responder: oneshot::Sender<Result<()>>,
840 },
841 /// One-pass rebuild of a model's DiskANN index — bulk-embedding finish
842 /// (D-276).
843 ///
844 /// One `CREATE INDEX` statement, and the indivisible half of the recipe:
845 /// measured at 19.7 s for 2,000 vectors at dim 256 on the reference box.
846 /// Budget-exempt by the same criterion as `ShadowSwap` — atomic by
847 /// necessity — with the difference that this one is *caller-scheduled*, so
848 /// a slow build is a choice the caller made knowingly.
849 RebuildEmbeddingIndex {
850 model: ModelName,
851 responder: oneshot::Sender<Result<()>>,
852 },
853 BulkImportChunk {
854 chunk: Vec<EdgeAssertion>,
855 responder: oneshot::Sender<Result<ChunkOutcome>>,
856 },
857 /// Drop or restore `trg_links_current_sync` — the links_current mirror's
858 /// window, opened and closed by [`Database::bulk_import_deferred`]
859 /// (D-277).
860 ///
861 /// Low priority beside the chunks it serves. One DDL statement either way
862 /// (`DROP TRIGGER IF EXISTS` down, `CREATE TRIGGER IF NOT EXISTS` up — the
863 /// same idempotent pair the v19 rung used on `trg_links_single_open`), so
864 /// the toggle is safe to send twice and the restore cannot fail on an
865 /// already-present trigger.
866 LinksCurrentMirror {
867 present: bool,
868 responder: oneshot::Sender<Result<()>>,
869 },
870 Archive {
871 cutoff: String,
872 archive_path: PathBuf,
873 responder: oneshot::Sender<Result<ArchiveReport>>,
874 },
875 /// Forget one lineage, moving its whole ledger to the cold file (0.14.13,
876 /// §15.4, D-230).
877 ///
878 /// Low priority for `Archive`'s reason and one of its own: it is bulk
879 /// physical movement holding the write lock for its whole transaction, and
880 /// it is the least urgent write in the crate — the rows it moves belong to
881 /// a lineage nobody is reading.
882 ArchiveBranch {
883 branch: String,
884 archive_path: PathBuf,
885 responder: oneshot::Sender<Result<ArchiveReport>>,
886 },
887 /// Move named concepts back out of the cold file (0.9.0, C3).
888 ///
889 /// Low priority for the same reason `Archive` is: it is bulk physical
890 /// movement with no latency bound, and it holds the write lock for its whole
891 /// transaction.
892 Rehydrate {
893 ids: Vec<String>,
894 archive_path: PathBuf,
895 responder: oneshot::Sender<Result<RehydrateReport>>,
896 },
897 /// Reconstruct the FTS index from `concepts` (§5.9, D-036, D-051).
898 ///
899 /// Low priority: it is maintenance on a derivative table, and a search index
900 /// that is a few seconds stale is a smaller cost than an interactive write
901 /// that waits behind a full reindex.
902 RebuildFts {
903 responder: oneshot::Sender<Result<()>>,
904 },
905 /// Refresh or top up the query planner's statistics (0.12.4, D-149).
906 ///
907 /// Low priority, and not a close call: statistics being a few seconds stale
908 /// costs a plan that was already the plan a moment ago, where preempting an
909 /// interactive assertion costs a caller their latency bound. It is a write —
910 /// it writes `sqlite_stat1` — so it takes the write lock like anything else,
911 /// and `PRAGMA analysis_limit` in `configure` is what keeps the hold a
912 /// function of the index count instead of the table size.
913 Analyze {
914 /// `true` runs `PRAGMA optimize`, which re-analyses only what SQLite
915 /// believes has gone stale; `false` runs `ANALYZE` unconditionally.
916 incremental: bool,
917 responder: oneshot::Sender<Result<()>>,
918 },
919 /// One step of a chunked shadow rebuild (§5.8, T1.2, D-082).
920 ///
921 /// Low priority, and one command per step rather than one per rebuild: the
922 /// whole value of building beside the live table is that the actor returns
923 /// here between chunks. See [`Database::rebuild_current_chunked`].
924 ShadowRebuild {
925 step: crate::integrity::ShadowStep,
926 responder: oneshot::Sender<Result<crate::integrity::ShadowOutcome>>,
927 },
928}
929
930/// What the write actor knows between turns (0.15.6, W14.3, [D-248]).
931///
932/// [`run_writer_actor`] owned a connection and nothing else. Every command was
933/// handed `&conn`, and every fact a command established died with it — so a
934/// single-edge assertion asked `branches` how many lineages exist, compiled the
935/// overlap guard, and compiled `INSERT_LINK`, on every call, having done all
936/// three on the previous one. Measured on the trunk that is 76 µs of a 160 µs
937/// write; once the database has forked it is 155 µs of a 343 µs write, because
938/// the statement being compiled each time is the guard's resolved form.
939///
940/// **Everything here is cached for one reason and invalidated by name for the
941/// same one: the actor is the only writer** (D-014). `branches` is written by
942/// `Fork` and by `ArchiveBranch` and by nothing else in the crate; the
943/// statements are bound to a connection this task owns for the process
944/// lifetime. A cache whose only writer is holding it cannot go stale behind its
945/// own back, which is why this is a plain `&mut` and not an epoch or a lock.
946///
947/// Built lazily rather than at open. Eager construction pays on a database that
948/// never asserts an edge, and it puts fallible work in the spawn path, where
949/// there is no caller to hand the error to.
950///
951/// # What is deliberately not here
952///
953/// The **hot-log intactness verdict** (review C-5). It is read on `read_conn`
954/// by every recorded-time read, not by the actor, so caching it here would put
955/// it on the wrong side of the process. It needs a shared cell and an
956/// invalidation argument about a *reader* seeing a stale answer, which is a
957/// different argument from this one and gets its own release.
958///
959/// The **batch paths' statements**. `write_edges_atomic` prepares inside its
960/// own transaction and drops before it commits, because a live statement is
961/// what makes SQLite refuse to end one. It already prepares once per chunk
962/// rather than once per row (D-056, §8.8), which is where that path's cost was.
963///
964/// [D-248]: ../../docs/architecture/s13-decision-register.md#d-248
965struct ActorState {
966 /// `branches`, as this actor last left it.
967 lineages: Option<Lineages>,
968 /// `INSERT_LINK`, compiled against the actor's connection.
969 insert_link: Option<libsql::Statement>,
970 /// The overlap guard, with the shape it was compiled for.
971 guard: Option<OverlapGuard>,
972}
973
974// `Lineages` moved to `graph::lineage` in 0.15.17 ([D-259]). It held
975// `(branch_id, is_root)` here, because the shape was all the actor needed;
976// resolving ancestry in Rust needs `parent_id` and `forked_at` as well, on
977// both sides of the crate, and two structs answering one question from one
978// table is what D-030 is about.
979//
980// [D-259]: ../docs/architecture/s13-decision-register.md#d-259
981
982impl ActorState {
983 fn new() -> Self {
984 Self {
985 lineages: None,
986 insert_link: None,
987 guard: None,
988 }
989 }
990
991 /// Forget `branches`. Called by the two commands that write it.
992 fn forget_lineages(&mut self) {
993 self.lineages = None;
994 }
995
996 /// Drop the compiled statements.
997 ///
998 /// Called by the commands that `ATTACH`, `DETACH`, or otherwise move the
999 /// schema under the connection. SQLite recompiles a statement across a
1000 /// schema change on its own and this does not rely on that: a statement
1001 /// dropped here costs one prepare on the next write, against a class of bug
1002 /// whose symptom would be a stale plan on the archive path in production.
1003 fn forget_statements(&mut self) {
1004 self.insert_link = None;
1005 self.guard = None;
1006 }
1007
1008 /// Both of the above, for a command that does both.
1009 fn forget_everything(&mut self) {
1010 self.forget_lineages();
1011 self.forget_statements();
1012 }
1013
1014 async fn lineages(&mut self, conn: &libsql::Connection) -> Result<&Lineages> {
1015 if self.lineages.is_none() {
1016 self.lineages = Some(Lineages::load(conn).await?);
1017 }
1018 Ok(self
1019 .lineages
1020 .as_ref()
1021 .expect("loaded on the line above or already present"))
1022 }
1023
1024 /// [`Lineages::shape_of`], against the cache.
1025 ///
1026 /// This is what replaced `check_lineages`, and the round trip it replaced
1027 /// is the one the review counted (C-6): one `SELECT` over `branches` per
1028 /// write, for an answer that changes when a lineage is forked or forgotten.
1029 async fn shape_of(
1030 &mut self,
1031 conn: &libsql::Connection,
1032 names: &[&str],
1033 ) -> Result<LineageShape> {
1034 self.lineages(conn).await?.shape_of(names)
1035 }
1036
1037 /// The overlap guard, compiled at most once per shape.
1038 ///
1039 /// Keyed on the shape rather than on the statement text because that is the
1040 /// thing [`check_prepared`] reads: it binds four parameters for `Trunk` and
1041 /// five otherwise, so a guard held under one shape and used under another
1042 /// would bind the wrong row even where the SQL happened to match.
1043 async fn guard(
1044 &mut self,
1045 conn: &libsql::Connection,
1046 shape: LineageShape,
1047 branch: &str,
1048 ) -> Result<&OverlapGuard> {
1049 // Keyed on the lineage as well as the shape since 0.15.17: the bound
1050 // ancestry is one reader's answer, so a guard held for `main` cannot
1051 // serve a branch that happens to share its shape. A caller writing to
1052 // one lineage — which is every caller this crate has — still prepares
1053 // once and keeps it across turns.
1054 if !self
1055 .guard
1056 .as_ref()
1057 .is_some_and(|g| g.answers_for(shape, branch))
1058 {
1059 // Resolved against the actor's own cache, then dropped, so the
1060 // borrow ends before the assignment. `lineages` refreshes it when
1061 // `Fork` or `ArchiveBranch` forgot it.
1062 let lineages = self.lineages(conn).await?.clone();
1063 self.guard = Some(OverlapGuard::prepare(conn, shape, &lineages, branch).await?);
1064 }
1065 Ok(self
1066 .guard
1067 .as_ref()
1068 .expect("prepared on the line above or already present"))
1069 }
1070
1071 /// `INSERT_LINK`, compiled once.
1072 ///
1073 /// The single-edge path ran `conn.execute(INSERT_LINK, …)`, which compiles
1074 /// the statement on every call — 61 µs of it, because `links` carries the
1075 /// projection triggers and they are compiled with the insert. This is D-056
1076 /// and D-057's lesson, which was learned on the batch path and never
1077 /// carried across to the path a caller actually waits on.
1078 async fn insert_link(&mut self, conn: &libsql::Connection) -> Result<&libsql::Statement> {
1079 if self.insert_link.is_none() {
1080 self.insert_link = Some(conn.prepare(INSERT_LINK).await?);
1081 }
1082 Ok(self
1083 .insert_link
1084 .as_ref()
1085 .expect("prepared on the line above or already present"))
1086 }
1087}
1088
1089enum LoopCtl {
1090 Continue,
1091 Break,
1092}
1093
1094/// Primary database handle for Macrame bitemporal ledger.
1095///
1096/// # Why this is not `Clone`, and what a multi-consumer caller uses instead
1097///
1098/// **Share it as `Arc<Database>`.** Every method here but one takes `&self`, so
1099/// an `Arc` is a complete handle and not a workaround: reads run concurrently
1100/// off `read_conn`, writes queue behind the actor's channel exactly as they do
1101/// through a `&Database`, and nothing becomes serialised that was not
1102/// serialised already. The exception is [`Database::close`], which takes `self`,
1103/// so the last owner closes with
1104/// `Arc::into_inner(db).expect("last handle").close().await`.
1105///
1106/// That exception is the whole reason `Clone` is absent. Cloning would have to
1107/// duplicate **the right to shut down**, and each field carrying that right
1108/// breaks differently when duplicated:
1109///
1110/// - `writer` is a [`tokio::task::JoinHandle`], which is not `Clone` at all —
1111/// so a hand-written impl would have to give the copy a `None`, and
1112/// `close()` on that copy returns `Ok(())` without ever checking the actor's
1113/// exit status. That status is one of the two reasons [`Drop`] tells callers
1114/// to prefer `close()`.
1115/// - `cadence_stop` is a [`tokio::sync::watch::Sender`], which **is** `Clone`,
1116/// and that is the worse case. Its contract is that *dropping* it stops the
1117/// snapshot task; a watch channel closes when the last sender goes, so one
1118/// surviving copy keeps that task running against a database that is going
1119/// away. Nothing returns an error, which is why this is the argument rather
1120/// than the `JoinHandle`.
1121/// - `closed` is per-handle, so two copies disagree about whether the ledger
1122/// was closed: `Drop` warns about a database that *was* closed, or stays
1123/// silent about one that was not.
1124///
1125/// And the ordering `close()` documents — cadence stopped, actor joined, *then*
1126/// the final snapshot, so that no write can land between the fold and the file
1127/// — is only enforceable while one handle can perform it. A second `close()`
1128/// writes a "final" snapshot with the actor still alive.
1129///
1130/// So the missing impl is the type saying shutdown has exactly one owner. The
1131/// Python binding reached the same shape from the other side and for the same
1132/// reason: `PyDatabase` holds a `RwLock<Option<Database>>` rather than a copy
1133/// per caller (0.13.30, W11.1, D-203).
1134pub struct Database {
1135 db: libsql::Database,
1136 /// The file this handle opened, kept so [`Database::diagnostic_conn`] can
1137 /// open it again under different flags (T5.1, D-091). `archive_path` and
1138 /// `snapshots_dir` are derived from it and were previously the only trace
1139 /// of it on the struct.
1140 path: PathBuf,
1141 read_conn: libsql::Connection,
1142 highpri_tx: mpsc::Sender<HighPriCommand>,
1143 lowpri_tx: mpsc::Sender<LowPriCommand>,
1144 clock: Arc<dyn Clock>,
1145 archive_path: PathBuf,
1146 snapshots_dir: PathBuf,
1147 schema_version: u32,
1148 /// Kept so [`Database::diagnostic_conn`] can configure the connections it
1149 /// mints the same way `open()` configured the internal readers (0.12.16,
1150 /// W5.5, D-159). Before that split, each one ran with SQLite's defaults.
1151 reader_cache_size: Option<i32>,
1152 /// The `SQLITE_OPEN_READ_ONLY` connection behind
1153 /// [`Database::diagnostic_conn`], opened on first use and dropped with this
1154 /// handle (0.15.14, W15.4, review C-9, [D-256]).
1155 ///
1156 /// **The connection, not the `libsql::Database` handle**, and the
1157 /// difference is the measurement rather than a preference. The first shape
1158 /// written here cached the handle and minted a connection per call, on the
1159 /// argument that `diagnostic_conn` promises a connection the *caller* owns.
1160 /// `examples/diagnostic_conn_probe.rs` says `Builder::…build()` costs
1161 /// **0.10 µs and opens nothing** — it succeeds against a path that does not
1162 /// exist — while `connect()` costs **51.5 µs** and is where
1163 /// `SQLITE_CANTOPEN` arrives for a missing file. The handle cache removes a
1164 /// call that does no work.
1165 ///
1166 /// **`Mutex<Option<_>>` rather than the `OnceCell` 0.15.14 shipped**
1167 /// (0.15.15, W15.5, [D-257]). A `OnceCell` can be filled and never
1168 /// emptied, and this connection is handed to arbitrary SQL, so it acquires
1169 /// state that the *next* caller must not inherit — a leaked `BEGIN` above
1170 /// all, which pins a WAL read snapshot and makes both this surface's reads
1171 /// stale and [`Database::checkpoint`] a no-op. The slot has to be
1172 /// clearable for the dirty ones to be replaced, and clearable is what a
1173 /// `OnceCell` is not. `tokio`'s mutex rather than `std`'s because it is
1174 /// held across the open.
1175 ///
1176 /// A failed first attempt leaves the slot empty, so a database whose file
1177 /// appears later is not poisoned by the call that came too early — the
1178 /// property the `OnceCell` had, kept.
1179 ///
1180 /// [D-256]: ../../docs/architecture/s13-decision-register.md#d-256
1181 /// [D-257]: ../../docs/architecture/s13-decision-register.md#d-257
1182 diagnostic_conn: tokio::sync::Mutex<Option<libsql::Connection>>,
1183 writer: Option<tokio::task::JoinHandle<()>>,
1184 /// Stops the snapshot cadence. Dropping it stops the task too, which is what
1185 /// keeps a `Database` that is dropped rather than closed from leaving a task
1186 /// running against a connection whose database is going away.
1187 cadence_stop: Option<tokio::sync::watch::Sender<bool>>,
1188 cadence: Option<tokio::task::JoinHandle<()>>,
1189 /// Set by [`Database::close`]. Read only by [`Drop`], which warns when it is
1190 /// still false — see that impl for why the omission is worth a warning.
1191 closed: bool,
1192 /// Shared with the actor (T1.4, T1.2). Held here rather than behind
1193 /// `#[cfg(feature = "metrics")]` so `open_inner` has one shape; with the
1194 /// feature off the metrics half is a zero-sized type and only
1195 /// [`Database::metrics`] is gated — which is also why the field is unread in
1196 /// the default build: the actor holds the other `Arc` and does the writing.
1197 #[cfg_attr(not(feature = "metrics"), allow(dead_code))]
1198 shared: Arc<ActorShared>,
1199}
1200
1201/// What the snapshot cadence should do, for [`Tuning::cadence`].
1202///
1203/// # Why this is not `Option<SnapshotCadence>`
1204///
1205/// [`Database::open_with_cadence`] takes `Option<SnapshotCadence>`, where `None`
1206/// means *no cadence at all*. Carrying that field into [`Tuning`] unchanged
1207/// would have made it the one field in the struct whose `None` is a request to
1208/// change the behaviour rather than a request to leave it alone — and since
1209/// `Tuning` derives `Default`, `open_tuned(path, Tuning::default())` would then
1210/// have silently disabled snapshots, while `open(path)` runs them. Two calls
1211/// that read as synonyms, one of which stops writing anchors.
1212///
1213/// So the tri-state is written out. `Default` is the default cadence, matching
1214/// [`Database::open`]; `Disabled` is `open_with_cadence(path, None)`, and has to
1215/// be asked for by name.
1216#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
1217#[non_exhaustive]
1218pub enum CadencePolicy {
1219 /// [`SnapshotCadence::default`], as [`Database::open`] uses.
1220 #[default]
1221 Default,
1222 /// No cadence task. `close()` is then the only thing that writes an anchor
1223 /// (§5.5, D-053).
1224 Disabled,
1225 /// An explicit cadence.
1226 Every(SnapshotCadence),
1227}
1228
1229impl CadencePolicy {
1230 /// Collapse to the `Option` the open path has always taken.
1231 fn resolve(self) -> Option<SnapshotCadence> {
1232 match self {
1233 Self::Default => Some(SnapshotCadence::default()),
1234 Self::Disabled => None,
1235 Self::Every(cadence) => Some(cadence),
1236 }
1237 }
1238}
1239
1240/// When SQLite should checkpoint the WAL on its own, for
1241/// [`Tuning::wal_autocheckpoint`] (0.12.14, W5.3, D-157).
1242///
1243/// # Why this is not `Option<u32>`
1244///
1245/// The same reason [`CadencePolicy`] is not `Option<SnapshotCadence>`, and the
1246/// plan for this wave specified `Option<u32>` here too. In a struct that derives
1247/// `Default`, a field whose `None` means *turn the mechanism off* is a field
1248/// that turns the mechanism off for everyone who did not mention it. Absence
1249/// means "leave it alone" everywhere in [`Tuning`], and disabling the automatic
1250/// checkpointer — which is not safe without an explicit
1251/// [`Database::checkpoint`] to replace it — has to be asked for by name.
1252///
1253/// **The default does not change.** 1,000 pages is SQLite's default and stays
1254/// SQLite's default; F-30 is a control-loop perturbation, not a correctness bug,
1255/// and changing a default is a behaviour change for every existing caller.
1256///
1257/// # What disabling it actually buys, measured (0.12.14, W5.3, D-157)
1258///
1259/// F-30 says the automatic checkpointer is an unbudgeted hold *inside* 0.12.0's
1260/// adaptive chunk controller: a checkpoint firing during a chunk transaction is
1261/// charged to that chunk, and since D-146 made the measured hold the input to
1262/// `next_chunk_size`, the controller shrinks in response to work the chunk did
1263/// not do. Three rounds, 6,000 concepts of 1 KB each through `write_concepts`,
1264/// release build:
1265///
1266/// | | longest chunk hold | mean | chunks | over budget | wall |
1267/// |---|---|---|---|---|---|
1268/// | autocheckpoint on (default) | **9.3–10.3 ms** | 2.40–2.44 ms | 125–130 | 24–28 | 304–321 ms |
1269/// | autocheckpoint off | **4.50 ms** | 2.08–2.20 ms | 142–153 | 18–27 | 298–339 ms |
1270///
1271/// **The tail is the finding, and it is real and reproducible.** The longest
1272/// hold roughly halves, and the >10 ms histogram bucket is populated only with
1273/// the checkpointer on — that bucket is the checkpoint, landing inside somebody
1274/// else's transaction and being charged to it. Every round agrees.
1275///
1276/// **What it does not buy is a calmer controller.** `over_budget` overlaps
1277/// between the arms, and total wall time is the same within noise. The
1278/// controller works near the budget boundary either way, because
1279/// [D-090](../docs/architecture/s13-decision-register.md)'s ~0.8 ms
1280/// per-transaction floor and the convergence cost do not go anywhere. So the
1281/// honest statement is that disabling autocheckpoint removes an outlier, not an
1282/// oscillation.
1283///
1284/// **And the cost is deferred, not removed.** The explicit
1285/// [`Database::checkpoint`] at the end of the same fixture moved **8,400–9,100
1286/// frames in 41–45 ms** with the checkpointer off, against **~860 frames in
1287/// 5.5–6.2 ms** with it on. That is the whole trade in one line: the same work,
1288/// moved out of the latency-bounded path and into one hold the caller chose the
1289/// moment for. It is a good trade for a bulk importer and a bad one for an
1290/// interactive process, which is why this is a knob and not a new default.
1291#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
1292#[non_exhaustive]
1293pub enum WalCheckpointPolicy {
1294 /// SQLite's own default: checkpoint once the WAL passes 1,000 pages.
1295 #[default]
1296 Default,
1297 /// No automatic checkpointing.
1298 ///
1299 /// **Only correct if you call [`Database::checkpoint`] yourself.** Without
1300 /// one, the WAL grows for the life of the process and the database file is
1301 /// never brought up to date.
1302 Disabled,
1303 /// Checkpoint once the WAL passes this many pages.
1304 ///
1305 /// `0` is not special-cased to [`Self::Disabled`] even though SQLite treats
1306 /// it that way, because a caller who computed a threshold and got zero has
1307 /// a bug, and inheriting SQLite's overload would turn it into a silently
1308 /// unbounded WAL.
1309 EveryPages(u32),
1310}
1311
1312impl WalCheckpointPolicy {
1313 /// The pragma to run, or `None` to leave the connection at SQLite's
1314 /// default.
1315 fn pragma(self) -> Option<String> {
1316 match self {
1317 Self::Default => None,
1318 Self::Disabled => Some("PRAGMA wal_autocheckpoint = 0".to_string()),
1319 Self::EveryPages(pages) => Some(format!("PRAGMA wal_autocheckpoint = {pages}")),
1320 }
1321 }
1322}
1323
1324/// Everything [`Database::open_tuned`] can be told, in one growable struct
1325/// (0.12.12, W5.1, D-155).
1326///
1327/// # Why a struct rather than a fourth constructor
1328///
1329/// There were three — [`Database::open`], [`Database::open_with_cadence`],
1330/// [`Database::open_with_clock`] — and each new knob added one more, with the
1331/// combinatorics of the ones before it. 0.13.0 alone wanted three knobs
1332/// (`wal_autocheckpoint`, and a page cache each for the writer and the
1333/// readers), which is the point at which the naming stops being possible.
1334///
1335/// **Setters plus `#[non_exhaustive]` are the whole design** (0.15.13, W15.3,
1336/// [C-11], [D-255]). They make a new knob an additive change: callers write
1337/// `Tuning::default().cadence(..)`, and the fields that arrive after them are
1338/// the ones they did not ask about. That is not a hypothetical — W5.1 shipped
1339/// this struct with two fields, and W5.3/W5.4 added three more without
1340/// touching a caller.
1341///
1342/// # Why the attribute needed the setters, and why 0.5.1's answer was the
1343/// other one
1344///
1345/// D-155 specified `#[non_exhaustive]` for this struct and then could not
1346/// ship it, on a fact that is still true: a `#[non_exhaustive]` **struct**
1347/// cannot be built with literal syntax outside its own crate *at all*, and
1348/// the functional-update form is literal syntax, so
1349/// `Tuning { cadence, ..Default::default() }` is `E0639` for every external
1350/// caller — the exact expression the attribute was there to protect. (The rule
1351/// differs from `#[non_exhaustive]` on an enum, which only forces a wildcard
1352/// arm; [`CadencePolicy`] has kept it for that reason since W4.2.) D-155 named
1353/// the two ways to have both — *a builder with setters, or plain `Default`* —
1354/// and chose `Default`, because the field-literal form is the legible one.
1355///
1356/// **What changed is not the argument but the deadline.** `Default` alone
1357/// leaves the growth additive only for callers who wrote
1358/// `..Default::default()`; a caller who wrote the exhaustive literal breaks on
1359/// the next field. Before 1.0 that is a compile error with an obvious fix.
1360/// After it, it is a major version — and this struct is the one in the crate
1361/// whose whole documented purpose is to keep acquiring fields. So the release
1362/// that is still allowed to break callers pays D-155's other price and writes
1363/// the setters, which is the half of its own analysis it declined at the time.
1364///
1365/// **The fields stay `pub` and stay readable**, and on a value you own they
1366/// stay assignable: `let mut t = Tuning::default(); t.cadence = ..;` compiles
1367/// outside this crate exactly as it did. What the attribute forbids is the
1368/// *literal*, which is the one form that enumerates every field and therefore
1369/// the one form a new field can break.
1370///
1371/// [C-11]: ../../docs/Macrame%20Update%20Plan%20v0.16.0.md
1372/// [D-255]: ../../docs/architecture/s13-decision-register.md#d-255
1373///
1374/// # The three constructors stay
1375///
1376/// They delegate here and are not deprecated. `open(path)` is the right call for
1377/// most callers and should not acquire a warning for being the common case; the
1378/// consolidation is about where the *next* knob goes, not about moving anyone.
1379///
1380/// ```no_run
1381/// # use macrame::prelude::*;
1382/// # async fn f() -> macrame::Result<()> {
1383/// let db = Database::open_tuned(
1384/// "graph.db",
1385/// Tuning::default().cadence(CadencePolicy::Disabled),
1386/// )
1387/// .await?;
1388/// # Ok(()) }
1389/// ```
1390#[derive(Clone, Default)]
1391#[non_exhaustive]
1392pub struct Tuning {
1393 /// What the snapshot cadence should do. Defaults to
1394 /// [`SnapshotCadence::default`], as [`Database::open`] does.
1395 pub cadence: CadencePolicy,
1396 /// A clock to stamp `recorded_at` with, for tests (§5.1.2, D-062). `None`
1397 /// is [`SystemClock`]. Floored against the database exactly as
1398 /// [`Database::open_with_clock`] describes — read that before injecting
1399 /// one against a non-empty file.
1400 pub clock: Option<Arc<dyn Clock>>,
1401 /// When SQLite checkpoints the WAL on its own (0.12.14, W5.3, F-30).
1402 ///
1403 /// Applied to the **write connection**, which is the only connection in
1404 /// this crate that commits, and therefore the only one whose autocheckpoint
1405 /// setting can ever fire. Pair [`WalCheckpointPolicy::Disabled`] with an
1406 /// explicit [`Database::checkpoint`] or the WAL grows without bound.
1407 pub wal_autocheckpoint: WalCheckpointPolicy,
1408 /// Page cache for the **write** connection, as SQLite's `cache_size`
1409 /// (0.12.15, W5.4).
1410 ///
1411 /// `None` leaves SQLite's default of −2000, which is −2000 *kibibytes*, or
1412 /// 2 MB. **Negative values are KiB and positive values are pages** — that
1413 /// is SQLite's convention and it is preserved rather than smoothed over,
1414 /// because a caller who knows the pragma should not have to discover that
1415 /// this crate redefined it. `Some(-64_000)` is 64 MB; `Some(64_000)` is
1416 /// 64,000 pages, which at the 4 KiB page size this crate gets is 256 MB.
1417 ///
1418 /// The writer wants a large cache: it is one connection, it holds the write
1419 /// lock while it works, and every page it has to re-read from disk is time
1420 /// no other writer can use.
1421 ///
1422 /// # Unlike the two above, `None` here is not a policy enum
1423 ///
1424 /// Because SQLite's default is a *value* rather than a mechanism. Absence
1425 /// still means "leave it alone" — it just happens that leaving this alone
1426 /// is expressible as not running a pragma, where leaving the automatic
1427 /// checkpointer alone required saying which of two things "alone" meant.
1428 pub writer_cache_size: Option<i32>,
1429 /// Page cache for every **read-only** connection: the shared
1430 /// [`Database::read_conn`], the snapshot cadence's own connection, and
1431 /// (since W5.5) each [`Database::diagnostic_conn`] (0.12.15, W5.4).
1432 ///
1433 /// Same units as [`Self::writer_cache_size`], and the same `None`.
1434 ///
1435 /// Split from the writer's because the profiles are opposite and one number
1436 /// cannot serve both. There is exactly one writer and it is long-lived, so
1437 /// its cache is a fixed cost paid once. Read-only connections are plural —
1438 /// the shared reader and the cadence's — so a large value here is
1439 /// multiplied by however many exist, which is the wrong size for the one
1440 /// connection that holds the write lock.
1441 ///
1442 /// **The multiplier used to be unbounded** and is not since 0.15.14
1443 /// (W15.4, [D-256]): `diagnostic_conn` minted a connection per call, so a
1444 /// caller in a loop multiplied this number by their own call count. There
1445 /// is one such connection per `Database` now, so the count is three.
1446 ///
1447 /// [D-256]: ../../docs/architecture/s13-decision-register.md#d-256
1448 pub reader_cache_size: Option<i32>,
1449 /// What to do about a stored `recorded_at` in the future (0.13.5, W7.4,
1450 /// §3.4).
1451 ///
1452 /// The clock floors itself at `MAX(recorded_at)` so stamps stay strictly
1453 /// increasing across restarts, which means one row from the future becomes
1454 /// this process's floor and every stamp it issues inherits it — into rows
1455 /// the next open reads back. Defaults to refusing beyond
1456 /// [`crate::DEFAULT_FUTURE_STAMP_TOLERANCE`], a day.
1457 ///
1458 /// Like [`Self::wal_autocheckpoint`] and unlike the two cache sizes, this
1459 /// is a policy enum rather than an `Option`, for
1460 /// [D-155](../../docs/architecture/s13-decision-register.md)'s reason: it
1461 /// guards an invariant, and a `None` that switches it off would switch it
1462 /// off for every caller who never heard of it.
1463 pub future_stamps: FutureStampPolicy,
1464}
1465
1466impl Tuning {
1467 // The setters below are what make `#[non_exhaustive]` payable (0.15.13,
1468 // W15.3, D-255). One per field, named after it, taking `self` — so
1469 // `Tuning::default().cadence(x).writer_cache_size(y)` is an expression, and
1470 // a field added later is a method added later rather than a break. They are
1471 // deliberately not clever: no `Into`, no grouping of two knobs under one
1472 // name, nothing that would have to be redesigned the first time a field
1473 // does not fit the pattern.
1474
1475 /// What the snapshot cadence should do — the [`cadence`](Self::cadence)
1476 /// field.
1477 pub fn cadence(mut self, cadence: CadencePolicy) -> Self {
1478 self.cadence = cadence;
1479 self
1480 }
1481
1482 /// Inject a clock — the [`clock`](Self::clock) field.
1483 ///
1484 /// Takes the clock rather than an `Option`, because `None` is what
1485 /// [`Tuning::default`] already holds and a setter whose argument can undo
1486 /// itself invites `clock(None)` as a way of saying nothing.
1487 /// [`Database::open_with_clock`] documents the flooring this is subject to;
1488 /// read it before injecting one against a non-empty file.
1489 pub fn clock(mut self, clock: Arc<dyn Clock>) -> Self {
1490 self.clock = Some(clock);
1491 self
1492 }
1493
1494 /// When SQLite checkpoints the WAL on its own — the
1495 /// [`wal_autocheckpoint`](Self::wal_autocheckpoint) field.
1496 pub fn wal_autocheckpoint(mut self, policy: WalCheckpointPolicy) -> Self {
1497 self.wal_autocheckpoint = policy;
1498 self
1499 }
1500
1501 /// Page cache for the write connection, in SQLite's units — the
1502 /// [`writer_cache_size`](Self::writer_cache_size) field, which documents
1503 /// why negative means KiB and positive means pages.
1504 pub fn writer_cache_size(mut self, size: i32) -> Self {
1505 self.writer_cache_size = Some(size);
1506 self
1507 }
1508
1509 /// Page cache for every read-only connection — the
1510 /// [`reader_cache_size`](Self::reader_cache_size) field, which documents
1511 /// why this is not the same number as the writer's.
1512 pub fn reader_cache_size(mut self, size: i32) -> Self {
1513 self.reader_cache_size = Some(size);
1514 self
1515 }
1516
1517 /// What to do about a stored `recorded_at` in the future — the
1518 /// [`future_stamps`](Self::future_stamps) field.
1519 pub fn future_stamps(mut self, policy: FutureStampPolicy) -> Self {
1520 self.future_stamps = policy;
1521 self
1522 }
1523
1524 /// The `Option<SnapshotCadence>` the three older constructors take, mapped
1525 /// onto the tri-state. `None` there means *disabled*, which is why
1526 /// [`CadencePolicy`] exists — see its docs.
1527 fn from_legacy(cadence: Option<SnapshotCadence>, clock: Option<Arc<dyn Clock>>) -> Self {
1528 Self {
1529 cadence: match cadence {
1530 Some(cadence) => CadencePolicy::Every(cadence),
1531 None => CadencePolicy::Disabled,
1532 },
1533 clock,
1534 wal_autocheckpoint: WalCheckpointPolicy::default(),
1535 writer_cache_size: None,
1536 reader_cache_size: None,
1537 future_stamps: FutureStampPolicy::default(),
1538 }
1539 }
1540}
1541
1542// `Clock` is not `Debug` — it is a behavioural trait with two methods and
1543// requiring `Debug` of every implementor to print a handle here would be the
1544// tail wagging the dog. So the field is reported as present-or-absent, which is
1545// the only part of it a reader of a `Tuning` dump can act on.
1546impl std::fmt::Debug for Tuning {
1547 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1548 f.debug_struct("Tuning")
1549 .field("cadence", &self.cadence)
1550 .field("clock", &self.clock.as_ref().map(|_| "<injected>"))
1551 .field("wal_autocheckpoint", &self.wal_autocheckpoint)
1552 .field("writer_cache_size", &self.writer_cache_size)
1553 .field("reader_cache_size", &self.reader_cache_size)
1554 .finish()
1555 }
1556}
1557
1558impl Database {
1559 /// Open a database file at `path`, configuring pragmas, running migrations, and spawning the Write Actor.
1560 ///
1561 /// The snapshot cadence runs with [`SnapshotCadence::default`]. Use
1562 /// [`Database::open_with_cadence`] to tune or disable it.
1563 pub async fn open(path: impl AsRef<Path>) -> Result<Self> {
1564 Self::open_with_cadence(path, Some(SnapshotCadence::default())).await
1565 }
1566
1567 /// Open with an explicit snapshot cadence, or `None` to run without one
1568 /// (§5.5, D-053).
1569 ///
1570 /// `None` restores the pre-0.5.5 behaviour, where `close()` is the only
1571 /// thing that ever writes an anchor. That is the right setting for a
1572 /// short-lived process that will not accumulate a delta worth bounding, and
1573 /// for tests that assert on the contents of the snapshot directory.
1574 pub async fn open_with_cadence(
1575 path: impl AsRef<Path>,
1576 cadence: Option<SnapshotCadence>,
1577 ) -> Result<Self> {
1578 Self::open_inner(path.as_ref(), Tuning::from_legacy(cadence, None)).await
1579 }
1580
1581 /// Open with an injected clock (§5.1.2, **defect K**, D-062).
1582 ///
1583 /// The reason this exists is testing: `recorded_at` is the transaction-time
1584 /// axis, and until now every test that wanted to assert on one had to either
1585 /// avoid it or drive a raw connection, because `open()` hardcoded
1586 /// [`SystemClock`]. `FakeClock` has been public and constructed in the test
1587 /// harness since 0.5.2 with nothing to inject it into — the compiler warned
1588 /// about the dead field on every build for three releases.
1589 ///
1590 /// **The clock is floored against the database before the actor starts.**
1591 /// [`Clock::raise_floor`] is called with the newest `recorded_at` in the
1592 /// ledger, so an injected clock cannot issue a stamp below what is already
1593 /// stored — which would abort the next concept write on
1594 /// `trg_concepts_monotonic_ra` rather than merely being odd. This is the
1595 /// step whose absence kept the defect open: the obvious implementation
1596 /// (take an `Arc<dyn Clock>`, use it) produces a `Database` that fails on
1597 /// its first write against any non-empty file.
1598 ///
1599 /// On a fresh database there is no floor, so an injected `FakeClock` issues
1600 /// exactly the stamps it was given.
1601 pub async fn open_with_clock(
1602 path: impl AsRef<Path>,
1603 cadence: Option<SnapshotCadence>,
1604 clock: Arc<dyn Clock>,
1605 ) -> Result<Self> {
1606 Self::open_inner(path.as_ref(), Tuning::from_legacy(cadence, Some(clock))).await
1607 }
1608
1609 /// Open with an explicit [`Tuning`] (0.12.12, W5.1, D-155).
1610 ///
1611 /// The consolidated form of the three constructors above, and the one that
1612 /// grows: every knob 0.13.0 adds arrives as a field here rather than as a
1613 /// fourth `open_*`. See [`Tuning`] for why the struct is
1614 /// `#[non_exhaustive]` and why that makes the growth additive.
1615 ///
1616 /// That sentence was written at 0.12.12 and was false until 0.15.13: the
1617 /// struct was *not* `#[non_exhaustive]`, and [`Tuning`]'s own docs carried
1618 /// a section arguing at length that it should not be. Two documents in one
1619 /// file contradicting each other for eleven releases, which is the shape
1620 /// C-16 is about; W15.3 resolved it by making this one true.
1621 pub async fn open_tuned(path: impl AsRef<Path>, tuning: Tuning) -> Result<Self> {
1622 Self::open_inner(path.as_ref(), tuning).await
1623 }
1624
1625 async fn open_inner(path: &Path, tuning: Tuning) -> Result<Self> {
1626 let Tuning {
1627 cadence,
1628 clock: injected,
1629 wal_autocheckpoint,
1630 writer_cache_size,
1631 reader_cache_size,
1632 future_stamps,
1633 } = tuning;
1634 let cadence = cadence.resolve();
1635 let db = libsql::Builder::new_local(path).build().await?;
1636 let write_conn = configure(db.connect()?, writer_cache_size).await?;
1637 // The writer is the only connection that commits, so it is the only one
1638 // whose `wal_autocheckpoint` can ever fire. Setting it on the readers
1639 // would be a pragma with no path to running (0.12.14, W5.3, D-157).
1640 if let Some(pragma) = wal_autocheckpoint.pragma() {
1641 let _ = write_conn.query(&pragma, ()).await?;
1642 }
1643 let read_conn = configure(db.connect()?, reader_cache_size).await?;
1644
1645 // PRAGMA query_only = ON on reader connection (§5.1.2)
1646 read_conn.execute("PRAGMA query_only = ON", ()).await?;
1647
1648 let migration = migrations::run(&write_conn).await?;
1649
1650 let (highpri_tx, highpri_rx) = mpsc::channel(256);
1651 let (lowpri_tx, lowpri_rx) = mpsc::channel(64);
1652
1653 // Floored after `migrations::run`, so the tables the floor is read from
1654 // are guaranteed to exist.
1655 let clock: Arc<dyn Clock> = match injected {
1656 Some(clock) => {
1657 if let Some(floor) =
1658 crate::util::clock::recorded_at_floor(&read_conn, future_stamps).await?
1659 {
1660 clock.raise_floor(floor);
1661 }
1662 clock
1663 }
1664 None => Arc::new(SystemClock::new(&read_conn, future_stamps).await?),
1665 };
1666 let shared = Arc::new(ActorShared::default());
1667 let writer = tokio::spawn(run_writer_actor(
1668 write_conn,
1669 Arc::clone(&clock),
1670 highpri_rx,
1671 lowpri_rx,
1672 Arc::clone(&shared),
1673 ));
1674
1675 let archive_path = derive_archive_path(path);
1676 let snapshots_dir = derive_snapshots_dir(path);
1677
1678 // **The cadence gets its own connection (Wave 4.1).** It used to share
1679 // `read_conn`, on the reasoning that `libsql::Connection` is an
1680 // Arc-backed handle and R15 makes every extra local connection a cost worth
1681 // not paying for nothing. The cost it was not paying for turned out to be
1682 // real: `reconstruct` brackets a fold with `ATTACH cold … DETACH cold`,
1683 // that region is per-connection state, and it is not synchronised. Two
1684 // folds on one connection can therefore interleave so that one DETACHes
1685 // the handle the other is mid-fold on.
1686 //
1687 // Recorded in §8.5 as a hazard rather than a defect because it **did not
1688 // reproduce**: 200 concurrent reconstructions against a 1 ms cadence with
1689 // an archive present produced zero errors, since the cadence anchors at
1690 // `MAX(recorded_at)` and so almost always takes the hot path. Narrow, and
1691 // real — a write landing between `log_head` and the fold opens it.
1692 //
1693 // Separate connections remove the interleaving rather than ordering it,
1694 // which is why this is preferred to a mutex around the region: there is
1695 // no shared state left to race on, and nothing to remember to hold. The
1696 // R15 objection does not apply — that fault is about *concurrent* opens,
1697 // and this is one more sequential open during `open()`.
1698 let (cadence_stop, cadence) = match cadence {
1699 Some(cadence) => {
1700 let cadence_conn = configure(db.connect()?, reader_cache_size).await?;
1701 cadence_conn.execute("PRAGMA query_only = ON", ()).await?;
1702 let (tx, rx) = tokio::sync::watch::channel(false);
1703 let handle = tokio::spawn(snapshot::run_cadence(
1704 cadence_conn,
1705 snapshots_dir.clone(),
1706 archive_path.clone(),
1707 cadence,
1708 rx,
1709 Arc::clone(&shared) as Arc<dyn snapshot::CommittedTurns>,
1710 ));
1711 (Some(tx), Some(handle))
1712 }
1713 None => (None, None),
1714 };
1715
1716 let handle = Self {
1717 db,
1718 path: path.to_path_buf(),
1719 read_conn,
1720 highpri_tx,
1721 lowpri_tx,
1722 clock,
1723 archive_path,
1724 snapshots_dir,
1725 schema_version: migrations::current_version(),
1726 reader_cache_size,
1727 diagnostic_conn: tokio::sync::Mutex::new(None),
1728 writer: Some(writer),
1729 cadence_stop,
1730 cadence,
1731 closed: false,
1732 shared,
1733 };
1734
1735 // **Re-anchor after a migration (Wave 4.4).**
1736 //
1737 // D-043 makes a `SCHEMA_VERSION` bump invalidate every snapshot on disk,
1738 // which is correct — a snapshot is a serialised `MaterializedState` and a
1739 // schema change can change what that means. What was missing is the other
1740 // half: nothing wrote a replacement, so the first `reconstruct` after an
1741 // upgrade skipped every file as incompatible and folded from genesis. On
1742 // a database with a large log that is the difference between reading one
1743 // snapshot and folding the whole history, and the only trace was a
1744 // `warn!` per skipped file.
1745 //
1746 // Written here rather than left to the cadence because the cadence fires
1747 // on log *growth* (D-053): an upgraded database that is then read but not
1748 // written would never re-anchor at all.
1749 //
1750 // Failure is logged, not returned. A missing anchor costs time and no
1751 // information — snapshots are derivative under Doctrine VI — so refusing
1752 // to open a database because its optimisation could not be rebuilt would
1753 // trade a real capability for a performance one.
1754 //
1755 // Gated on the cadence being enabled, as well as on an actual upgrade:
1756 // `open_with_cadence(None)` means *this handle writes no snapshots except
1757 // at close()*, and a one-off write at open would contradict that for a
1758 // caller who asked for the quiet mode precisely to control when files
1759 // appear. They still get an anchor from `close()`.
1760 if migration.upgraded() && handle.cadence.is_some() {
1761 let ts = handle.clock.now();
1762 let archive = crate::temporal::archive::archive_present(&handle.archive_path)
1763 .then_some(handle.archive_path.as_path());
1764 match snapshot::write_final(&handle.read_conn, &handle.snapshots_dir, &ts, archive)
1765 .await
1766 {
1767 Ok(path) => tracing::info!(
1768 "schema moved v{} -> v{}; re-anchored snapshots at {:?}",
1769 migration.from,
1770 migration.to,
1771 path
1772 ),
1773 Err(e) => tracing::warn!(
1774 "schema moved v{} -> v{} but the re-anchor failed: {e}. \
1775 Reconstruction stays correct and folds from genesis until the \
1776 cadence writes one.",
1777 migration.from,
1778 migration.to
1779 ),
1780 }
1781 }
1782
1783 Ok(handle)
1784 }
1785
1786 /// Read connection handle for queries, traversals, and folds.
1787 pub fn read_conn(&self) -> &libsql::Connection {
1788 &self.read_conn
1789 }
1790
1791 /// The file this handle opened.
1792 pub fn path(&self) -> &Path {
1793 &self.path
1794 }
1795
1796 /// The **OS-level read-only** connection to this database, for diagnostics
1797 /// (§4.7, T5.1, D-091).
1798 ///
1799 /// # Why this exists when `read_conn()` already does
1800 ///
1801 /// Two different things, and the difference is the point:
1802 ///
1803 /// * `read_conn()` returns a shared `&Connection` carrying
1804 /// `PRAGMA query_only = ON`. That pragma is **per-connection and
1805 /// reversible by its holder in one statement**, so it is a guardrail
1806 /// against accident, not a capability boundary. And because it is the
1807 /// connection the crate's own traversals and folds run on, a caller who
1808 /// runs a long reporting query there is competing with all of them.
1809 /// * This returns a connection opened with `SQLITE_OPEN_READ_ONLY`, which is
1810 /// enforced by the engine below the pragma layer, and which nothing inside
1811 /// the crate runs on.
1812 ///
1813 /// # One connection per `Database`, shared between callers (0.15.14, W15.4)
1814 ///
1815 /// Through 0.15.13 this minted a connection per call and the sentence above
1816 /// read *"a **new, independently owned** … connection"*. Review item C-9
1817 /// asked for the file to be opened once per handle instead, and the
1818 /// measurement behind it (`examples/diagnostic_conn_probe.rs`) was sharper
1819 /// than the ask: `connect()` is **51.5 µs of an 82.7 µs call**, and
1820 /// `Builder::…build()` — which every document in this crate called *the
1821 /// open* — is **0.10 µs and opens nothing**, succeeding against a path that
1822 /// does not exist. Caching the handle would have removed 0.10 µs. What
1823 /// ships caches the connection: **82.7 µs → 19.9 µs**, and what remains is
1824 /// the `stat` below, not the connection.
1825 ///
1826 /// **So per-connection state is shared between diagnostic callers.** An
1827 /// `ATTACH`, a `PRAGMA`, a temp table one caller creates is visible to the
1828 /// next — which matters here and nowhere else in this API, because
1829 /// `diagnostic_query` is the one arbitrary-SQL surface the crate exposes.
1830 /// Asserted in `diagnostic_callers_share_one_connection_and_its_state`
1831 /// rather than left to this paragraph.
1832 ///
1833 /// What that costs is isolation between *diagnostic* callers. What it does
1834 /// not cost is the thing D-091 was for: this is still not `read_conn()`, a
1835 /// reporting query here still does not compete with the crate's traversals
1836 /// and folds, and the read-only boundary below is untouched.
1837 ///
1838 /// # What is scrubbed before you get it, and what is not (0.15.15, W15.5)
1839 ///
1840 /// 0.15.14 shipped the sharing and documented it. [D-257] measured what it
1841 /// actually admits, and one of the four is not an isolation nuisance but a
1842 /// correctness hole that leaves this surface entirely: **a leaked `BEGIN`
1843 /// pins a WAL read snapshot.** Measured — 200 writes through the typed
1844 /// surface while a diagnostic caller held an unclosed read transaction —
1845 /// later diagnostic reads answered **1 row instead of 201**, silently, and
1846 /// [`Database::checkpoint`] became a no-op with the WAL stuck at 8.5 MB
1847 /// until the transaction was rolled back. Stale answers on the surface a
1848 /// caller reaches for when they already distrust the typed one, and an
1849 /// unbounded WAL on a method that has nothing to do with diagnostics.
1850 ///
1851 /// So this method scrubs on entry rather than trusting the caller, and the
1852 /// prices are from `examples/diagnostic_hygiene_probe.rs`:
1853 ///
1854 /// | left behind | how it is found | what happens |
1855 /// |---|---|---|
1856 /// | an open transaction | `is_autocommit()`, **0.04 µs** | `ROLLBACK`, 2.4 µs |
1857 /// | a temp table or view | `PRAGMA temp.schema_version` | connection dropped, re-minted at 56.6 µs |
1858 /// | an `ATTACH` | `PRAGMA database_list` | connection dropped, re-minted |
1859 /// | `busy_timeout`, `cache_size` | not detected — restated, 1.0 µs | reset to the crate's values |
1860 /// | any other pragma | **not detected** | **inherited by the next caller** |
1861 ///
1862 /// The two dirt questions are asked as pragmas because the same two asked
1863 /// over `temp.sqlite_master` and `pragma_database_list` cost **7.8 µs**
1864 /// against **2.4 µs**, on a call whose entire warm cost is the `stat`.
1865 ///
1866 /// The last row is the honest residue. SQLite has no cheap enumeration of
1867 /// connection-scoped pragma state, so the crate restates the two pragmas
1868 /// *it* set (D-159's `busy_timeout` above all — a caller who sets it to 0
1869 /// would otherwise remove the 5 s margin from every later diagnostic call)
1870 /// and leaves the rest. A caller who sets `case_sensitive_like` or
1871 /// `recursive_triggers` changes what **later diagnostic queries on this
1872 /// handle** see, and nothing else: the crate's own readers and writer are
1873 /// different connections, so no typed answer can move. That is a smaller
1874 /// blast radius than 0.15.14 had and a larger one than zero, and it is
1875 /// written down rather than rounded off.
1876 ///
1877 /// That paragraph is about pragmas that belong to a *connection*, which is
1878 /// what "residue" means and what the scrub is for. **Not every pragma
1879 /// reachable here is one**, and the section below is the exception —
1880 /// measured after D-257 claimed this one "cannot change any typed answer"
1881 /// without checking (0.15.16, [D-258]).
1882 ///
1883 /// **Scrubbed on entry, not on exit**, because there is no exit: this
1884 /// returns a `Connection` clone the caller keeps for as long as it likes,
1885 /// and the crate is never told they are done. Entry is the one place that
1886 /// covers both this method and `diagnostic_query`. The consequence is that
1887 /// a caller who leaks a transaction and never calls again holds the pin
1888 /// until the handle drops — so the Python binding, which *does* know when
1889 /// a query is over, scrubs on exit as well.
1890 ///
1891 /// **Measured on libSQL 0.9.30 rather than assumed**
1892 /// (`examples/readonly_open_probe.rs`), against a live WAL database with the
1893 /// write actor running:
1894 ///
1895 /// | | `read_conn()` | `diagnostic_conn()` |
1896 /// |---|---|---|
1897 /// | `SELECT`, `EXPLAIN QUERY PLAN` | allowed | allowed |
1898 /// | `INSERT` | refused | refused |
1899 /// | `PRAGMA query_only = OFF` | **allowed** | allowed |
1900 /// | `INSERT` after that | **allowed** | **refused** |
1901 /// | `ATTACH` an existing file | allowed | allowed |
1902 /// | `INSERT` into the attachment | refused¹ | **refused** |
1903 /// | `ATTACH` a path that does not exist | — | refused (`SQLITE_CANTOPEN`) |
1904 ///
1905 /// The third and fourth rows are the whole difference: turning the pragma
1906 /// off restores writes on `read_conn()` and does not here. That is what
1907 /// "boundary rather than guardrail" means, and it is now a number rather
1908 /// than a claim.
1909 ///
1910 /// ¹ On `read_conn()` that refusal is `query_only` — the same reversible
1911 /// thing as row 2. On `diagnostic_conn()` it is the open flags, and the
1912 /// probe runs it *after* `query_only = OFF` so that the pragma cannot be
1913 /// what is doing the work.
1914 ///
1915 /// # `ATTACH` is permitted, and does not widen the write boundary
1916 ///
1917 /// Checked because `diagnostic_query` (Python) is the only arbitrary-SQL
1918 /// surface this crate exposes, and an attachment is a second `open` whose
1919 /// flags it does not obviously inherit. It does inherit them: the
1920 /// attachment is read-only, and a nonexistent path is `SQLITE_CANTOPEN`
1921 /// rather than a new file, because `SQLITE_OPEN_CREATE` is dropped for the
1922 /// attachment as it is for `main`. So `SQLITE_OPEN_READ_ONLY` bounds the
1923 /// **connection**, not just the one file it names (0.10.0, W4.3).
1924 ///
1925 /// What it does widen is *reading*: an `ATTACH` can name any file the
1926 /// process can open, so this connection is a read surface over the
1927 /// filesystem, not over this database. That is a property of arbitrary SQL
1928 /// rather than of the flags, and it is unchanged by them.
1929 ///
1930 /// # One pragma here can end the process, and sharing is not why
1931 ///
1932 /// `PRAGMA hard_heap_limit = 1` through this connection leaves the whole
1933 /// **process** unable to use SQLite. Not this connection, not this handle:
1934 /// measured (`tests_py/probes/diagnostic_global_pragmas.py`), the next
1935 /// ordinary write, the next read, `checkpoint()`, `close()`, and opening a
1936 /// *different* database file all fail with `out of memory`, permanently.
1937 ///
1938 /// `SQLITE_OPEN_READ_ONLY` does not stand in the way because setting it is
1939 /// not a write to the database file, and the scrub above does not help
1940 /// because there is nothing left on the connection to scrub: the limit
1941 /// lives in the SQLite library, one per process. **Re-measured with a
1942 /// connection minted per call — the 0.15.13 shape, before any of the
1943 /// sharing this method now does — the outcome is identical.** So this is
1944 /// not a cost of [D-256]'s shared connection and no amount of hygiene
1945 /// addresses it.
1946 ///
1947 /// Six other candidates were measured and are harmless: `soft_heap_limit`
1948 /// (a hint, not a wall), `locking_mode = EXCLUSIVE` (accepted; the writer
1949 /// kept working), `temp_store_directory`, `max_page_count` (clamped, and
1950 /// per-connection), `case_sensitive_like` (the control), and
1951 /// `wal_checkpoint`, which is refused outright because this connection is
1952 /// read-only.
1953 ///
1954 /// It belongs with the `ATTACH` note above rather than with the scrub: both
1955 /// are properties of handing a caller **arbitrary SQL**, not of the flags
1956 /// the connection was opened with. The practical form is one sentence —
1957 /// *`diagnostic_query` is not a safe place to put a string that came from
1958 /// somewhere else* — which `ATTACH` already made true and this makes
1959 /// sharper. Not blocked by refusing statements that look like this one,
1960 /// because matching SQL text is guesswork wearing the costume of a
1961 /// guarantee, and it would do nothing for a Rust caller holding the
1962 /// connection directly ([D-258]).
1963 ///
1964 /// [D-258]: ../../docs/architecture/s13-decision-register.md#d-258
1965 ///
1966 /// # One way this is *more* permissive, which is worth knowing
1967 ///
1968 /// `CREATE TEMP TABLE` **succeeds** here and is refused by `read_conn()`.
1969 /// Temp tables live in a separate temporary database that is writable
1970 /// regardless of how the main one was opened, whereas `query_only` refuses
1971 /// them outright — which is the mechanism [D-050] measured when it removed
1972 /// `TwoPhaseTempTable` for returning `SQLITE_READONLY (8)` on the read
1973 /// connection. So the stronger boundary is not uniformly stronger, and a
1974 /// strategy that needs a temp table has a connection it could run on. That
1975 /// is recorded, not acted on: D-050 removed the strategy for two reasons and
1976 /// this addresses one of them.
1977 ///
1978 /// # Calling this concurrently was R15's shape, and 0.15.14 is why it is not
1979 ///
1980 /// Through 0.15.13 this section said *"this is the one method on `Database`
1981 /// that opens the file … each call is a fresh `libsql::Builder::…build()`,
1982 /// so *N* threads calling it at once are *N* concurrent opens"*. The first
1983 /// half was true and the second was wrong about which call does it:
1984 /// `build()` opens nothing, and `connect()` is the open. The conclusion
1985 /// happened to be right for the wrong reason, which is why it took a
1986 /// measurement to move.
1987 ///
1988 /// **Measured through the unlocked Python binding**, 48 threads on a
1989 /// barrier, 30 runs per arm (`tests_py/probes/r15_diagnostic_path.py`):
1990 ///
1991 /// | arm | bad runs |
1992 /// |---|---|
1993 /// | a connection per call, as before 0.15.14 | **3 / 30** |
1994 /// | the `libsql::Database` handle cached, a connection per call | 2 / 30 |
1995 /// | the `connect()` serialised behind a mutex | 1 / 18 |
1996 /// | **one connection, as shipped** | **0 / 30** |
1997 ///
1998 /// Rows two and three are why the shape that preserved the old contract was
1999 /// not taken: caching the handle leaves the crash because it leaves the
2000 /// `connect()`, and serialising the `connect()` alone does not reach zero —
2001 /// the race is between minting a connection and the *use* of the others,
2002 /// not between two mintings.
2003 ///
2004 /// **There is nothing left on this path to bound**, because after the first
2005 /// call it no longer opens anything. `Database::open` from many threads is
2006 /// still R15's shape and `examples/r15_soak.rs` still reproduces it; this
2007 /// method is no longer a way to reach it. The Python binding keeps its
2008 /// mutex as margin rather than as a measured necessity — see
2009 /// `PyDatabase::diagnostic_rows`.
2010 ///
2011 /// # Errors
2012 ///
2013 /// The file must already exist. `SQLITE_OPEN_READ_ONLY` drops
2014 /// `SQLITE_OPEN_CREATE` with it, so a missing file is `SQLITE_CANTOPEN`
2015 /// rather than a fresh empty database — which is the right failure, and is
2016 /// surfaced as a typed error rather than as libSQL's error 14.
2017 ///
2018 /// The check is a `stat` on **every** call, not only the first, and it is
2019 /// still most of what a warm call costs (18.6 µs of the 22 µs a clean one
2020 /// takes since 0.15.15). It is kept at that price because the alternative
2021 /// is the worst failure a *diagnostic* surface can have: a cached
2022 /// connection whose file has been deleted and replaced answers from the old
2023 /// inode, silently, on the one method a caller reaches for when they
2024 /// already doubt the typed answer.
2025 pub async fn diagnostic_conn(&self) -> Result<libsql::Connection> {
2026 let fail = |reason: String| DbError::DiagnosticConn {
2027 path: self.path.display().to_string(),
2028 reason,
2029 };
2030 // Checked per call rather than once, because it is the documented
2031 // error of *this* method and costs a `stat`. The handle below is
2032 // opened once; the question "is the file there" is asked every time,
2033 // so a caller who deletes the file still gets the typed refusal on the
2034 // next call rather than a connection to an inode nothing can name.
2035 if !self.path.exists() {
2036 return Err(fail(
2037 "the file does not exist, and a read-only open cannot create it".to_string(),
2038 ));
2039 }
2040 let mut slot = self.diagnostic_conn.lock().await;
2041
2042 // Scrub what the last caller left, before this one can inherit it
2043 // (0.15.15, W15.5, D-257). Free on a clean connection: the transaction
2044 // check is a C call at 0.04 us and the two dirt pragmas are 2.4 us,
2045 // against a `stat` of 18.6 that has already happened above.
2046 scrub(&mut slot).await;
2047
2048 match slot.as_ref() {
2049 // Restated per call since 0.15.15 rather than once at mint. The
2050 // pragmas are per-connection, and there is one connection now, so
2051 // 0.15.14 set them once — which was right until the shared
2052 // connection was also something a caller could move them on. No
2053 // dirt check can see a pragma, so the crate restores its own
2054 // instead of detecting that they went: 1.0 us, against a
2055 // `busy_timeout` of 0 outliving the call that set it.
2056 Some(conn) => configure_common(conn, self.reader_cache_size).await?,
2057 None => {
2058 *slot = Some(self.open_diagnostic_conn(&fail).await?);
2059 }
2060 }
2061
2062 Ok(slot
2063 .as_ref()
2064 .expect("the slot was filled above or the open returned Err")
2065 .clone())
2066 }
2067
2068 /// Roll back and discard whatever the last diagnostic caller left behind,
2069 /// without handing a connection out (0.15.15, W15.5, [D-257]).
2070 ///
2071 /// [`Database::diagnostic_conn`] does this on the way *in*, which is the
2072 /// only place the crate can do it: the method returns a `Connection` clone
2073 /// and is never told the caller is finished with it. That covers every
2074 /// caller and leaves one gap — somebody who leaks a transaction and then
2075 /// never calls again holds the WAL read snapshot until the handle drops,
2076 /// which makes [`Database::checkpoint`] a no-op for that whole time.
2077 ///
2078 /// This is for the callers that *do* know when they are done. It costs the
2079 /// scrub and not the `stat` — around 3.5 µs on a clean connection — because
2080 /// it hands nothing back and so has nothing to promise about the file still
2081 /// being there.
2082 ///
2083 /// **The Python binding does not call it**, though the first draft did. A
2084 /// mutation deleting that call left the whole suite green, and the reason is
2085 /// that the gap is not reachable from there: `diagnostic_query` runs one
2086 /// statement, a bare `BEGIN` pins nothing — the snapshot is taken by the
2087 /// first *read* inside the transaction — and any statement that would take
2088 /// it arrives through the same method, whose entry scrub has already rolled
2089 /// the transaction back. The gap is real for a Rust caller holding a clone
2090 /// across both, which is what this method and
2091 /// `scrubbing_releases_the_pin_without_handing_out_a_connection` are for.
2092 ///
2093 /// Infallible by construction: everything it might have reported is
2094 /// something it responds to by discarding the connection, and the next
2095 /// [`Database::diagnostic_conn`] opens a fresh one.
2096 ///
2097 /// [D-257]: ../../docs/architecture/s13-decision-register.md#d-257
2098 pub async fn scrub_diagnostic_conn(&self) {
2099 let mut slot = self.diagnostic_conn.lock().await;
2100 scrub(&mut slot).await;
2101 }
2102
2103 /// The cold half of [`Database::diagnostic_conn`]: the actual open.
2104 ///
2105 /// Reached on the first call and on any call whose predecessor left the
2106 /// connection dirty enough to discard. It costs 56.5 µs against the warm
2107 /// path's scrub, and it is split out for reading rather than for speed:
2108 /// `Box::pin`ning it here — on the theory that carrying `Builder::build()`'s
2109 /// state machine inside the warm path's was what made a clean call 29.8 µs
2110 /// rather than the 21.8 its parts measure — changed nothing at all
2111 /// (0.15.15, W15.5, [D-257]).
2112 ///
2113 /// [D-257]: ../../docs/architecture/s13-decision-register.md#d-257
2114 async fn open_diagnostic_conn(
2115 &self,
2116 fail: &dyn Fn(String) -> DbError,
2117 ) -> Result<libsql::Connection> {
2118 let db = libsql::Builder::new_local(&self.path)
2119 .flags(libsql::OpenFlags::SQLITE_OPEN_READ_ONLY)
2120 .build()
2121 .await
2122 .map_err(|e| fail(e.to_string()))?;
2123 let conn = db.connect().map_err(|e| fail(e.to_string()))?;
2124 // Configured since 0.12.16 (W5.5, D-159). Until then this connection
2125 // ran with SQLite's defaults while every other connection in the
2126 // process ran with the crate's — most consequentially a `busy_timeout`
2127 // of 0 against everyone else's 5 s, on the one surface whose job is to
2128 // answer questions when the typed path is already suspect. Only the
2129 // common half: `SQLITE_OPEN_READ_ONLY` cannot set `journal_mode`, and
2130 // the rest govern writes this connection cannot make.
2131 configure_common(&conn, self.reader_cache_size).await?;
2132 // The `libsql::Database` is dropped here and the connection outlives
2133 // it, which is the ownership libSQL's own API implies: `connect()`
2134 // returns a `Connection` that does not borrow the builder's handle.
2135 Ok(conn)
2136 }
2137
2138 /// Cross-check the snapshot chain against a fold from genesis (§5.5, T5.3,
2139 /// D-092).
2140 ///
2141 /// `write_final` composes onto the previous snapshot, so snapshot *n* is
2142 /// derived from snapshot *n−1* and nothing in the chain ever folds the whole
2143 /// log. An error at any link propagates forward forever and every read
2144 /// agrees with it, because every read descends from it. This is the check
2145 /// that would notice.
2146 ///
2147 /// # When to run it
2148 ///
2149 /// **Not on a schedule this crate chooses.** A genesis fold is precisely the
2150 /// cost snapshots exist to avoid, so running it periodically by default
2151 /// would give every application the bill snapshots were bought to remove —
2152 /// on a database whose log is large enough for snapshots to matter, which is
2153 /// the only kind where this is worth doing. The plan calls it a scheduling
2154 /// problem and it is the caller's schedule: an idle period, a nightly job,
2155 /// or once per *N* anchors, chosen against a log size this crate cannot see.
2156 ///
2157 /// The cadence is deliberately left alone for the same reason — it runs on a
2158 /// connection shared with nothing and a fold there would compete with
2159 /// interactive reads at a moment nobody chose.
2160 ///
2161 /// # It reports; it does not repair
2162 ///
2163 /// A divergence means the snapshots are a wrong **cache**, not that the
2164 /// ledger is corrupt: [Doctrine VI] makes them disposable, so deleting
2165 /// [`Self::snapshots_dir`] restores correctness and costs only speed.
2166 /// Rewriting the file here would destroy the evidence that composition has a
2167 /// defect, which is the only thing this can tell you that you did not
2168 /// already know.
2169 ///
2170 /// Pair it with the actor counters ([`Self::metrics`], D-079) so a
2171 /// divergence found by a scheduled run is visible beside the write latency
2172 /// of the period that produced it.
2173 ///
2174 /// [Doctrine VI]: ../../docs/architecture/s0-s3-foundations.md#doctrine-vi
2175 pub async fn verify_snapshot_chain(&self, ts: &str) -> Result<crate::temporal::ChainCheck> {
2176 let archive = crate::temporal::archive::archive_present(&self.archive_path)
2177 .then_some(self.archive_path.as_path());
2178 crate::temporal::verify_snapshot_chain(&self.read_conn, ts, archive, &self.snapshots_dir)
2179 .await
2180 }
2181
2182 /// Check the newest link of the snapshot chain (0.15.19, review C-18).
2183 ///
2184 /// The affordable half of [`Self::verify_snapshot_chain`]: re-derive the
2185 /// newest snapshot from the one before it and compare, which is one
2186 /// anchored delta rather than a fold from genesis. `Ok(None)` when there
2187 /// are not two snapshots yet.
2188 ///
2189 /// The snapshot cadence already runs this after every anchor it writes and
2190 /// logs a divergence at `warn`, so a caller reaching for it directly is
2191 /// usually one that wants the [`crate::temporal::ChainCheck`] itself — the
2192 /// disagreeing ids — rather than a yes or no.
2193 ///
2194 /// **It reports; it does not repair.** A snapshot is derivative
2195 /// (Doctrine VI), so the repair is to delete the snapshot directory, which
2196 /// is the caller's call and one line. What this cannot tell you is whether
2197 /// the chain went wrong further back than one link; that is what
2198 /// [`Self::verify_snapshot_chain`] is for, and why it stays.
2199 pub async fn verify_last_link(&self) -> Result<Option<crate::temporal::ChainCheck>> {
2200 let archive = crate::temporal::archive::archive_present(&self.archive_path)
2201 .then_some(self.archive_path.as_path());
2202 crate::temporal::verify_last_link(&self.read_conn, archive, &self.snapshots_dir).await
2203 }
2204
2205 /// The clock every write is stamped with (§5.1.1).
2206 pub fn clock(&self) -> &Arc<dyn Clock> {
2207 &self.clock
2208 }
2209
2210 /// Schema version this handle opened against.
2211 pub fn schema_version(&self) -> u32 {
2212 self.schema_version
2213 }
2214
2215 /// Cold database path, derived by convention from the main file.
2216 pub fn archive_path(&self) -> &Path {
2217 &self.archive_path
2218 }
2219
2220 /// Snapshot directory, derived by convention from the main file.
2221 pub fn snapshots_dir(&self) -> &Path {
2222 &self.snapshots_dir
2223 }
2224
2225 /// What the write actor has done since this handle was opened (T1.4, D-079).
2226 ///
2227 /// Requires the `metrics` feature. The counters are per-handle and start at
2228 /// zero on `open()` — they are not read from the database, because the thing
2229 /// being measured is *this process's* actor and merging two processes'
2230 /// histograms would produce a number about neither.
2231 ///
2232 /// The intended first question is [`crate::metrics::MetricsSnapshot::budget_violations`]:
2233 ///
2234 /// ```no_run
2235 /// # async fn f(db: ¯ame::Database) {
2236 /// # #[cfg(feature = "metrics")] {
2237 /// for k in db.metrics().budget_violations() {
2238 /// eprintln!("{} broke the 3 ms bound {} times", k.kind, k.over_budget);
2239 /// }
2240 /// # }
2241 /// # }
2242 /// ```
2243 ///
2244 /// Reading this does not stop the actor — see
2245 /// [`crate::metrics::ActorMetrics::snapshot`] for what that costs in
2246 /// consistency, and why the trade goes that way.
2247 #[cfg(feature = "metrics")]
2248 pub fn metrics(&self) -> crate::metrics::MetricsSnapshot {
2249 self.shared.metrics.snapshot()
2250 }
2251
2252 /// The underlying libSQL database, for callers that need their own connection.
2253 ///
2254 /// # Actor containment is a convention above this line, not a guarantee
2255 ///
2256 /// **Kept public, and the honest statement of what that costs (Wave 4.3).**
2257 /// §5.1 says the write actor is the sole writer, and two mechanisms make that
2258 /// true of the handle: every write method goes through a channel, and
2259 /// [`Self::read_conn`] carries `PRAGMA query_only = ON`. **Nothing protects a
2260 /// connection obtained from here.** A caller can open one, write to `links`
2261 /// directly, and the actor will not know — the triggers still fire and the
2262 /// ledger stays internally consistent, but the single-writer property that
2263 /// [`crate::CHUNK_BUDGET`]'s latency argument rests on is gone, and so is the
2264 /// serialisation the overlap guard (D-060) relies on.
2265 ///
2266 /// This is the same shape as the limit stated in §4.2 for that guard, and it
2267 /// is one fact rather than two: **the storage layer permits what this API
2268 /// refuses.** Making it private would not change that — the database file is
2269 /// reachable by any SQLite client on the machine — it would only remove the
2270 /// supported way to do the thing, which is how escape hatches become
2271 /// `unsafe`-adjacent folklore.
2272 ///
2273 /// The free functions [`crate::register_model`] and
2274 /// [`crate::upsert_embedding`] take a bare connection for the same reason and
2275 /// carry the same caveat; prefer [`Self::register_model`] and
2276 /// [`Self::upsert_embeddings`], which go through the actor.
2277 ///
2278 /// # The legitimate-use list is now one item long (T5.1, D-091)
2279 ///
2280 /// It used to read: `EXPLAIN QUERY PLAN` and other diagnostics, read-only
2281 /// reporting queries wanting their own connection rather than sharing the
2282 /// reader, and provoking a guard in a test. The first two are exactly what
2283 /// [`Self::diagnostic_conn`] now does, and it does them behind an OS-level
2284 /// read-only open rather than on a handle that can write. **Use that.**
2285 ///
2286 /// What is left is the one use that genuinely requires write access through
2287 /// a connection the actor does not own: *provoking a guard* — writing the
2288 /// state §4.7 says the storage layer permits and this API refuses, so a test
2289 /// can assert the gap is still where the document says it is. That is the
2290 /// only thing this crate's own suite uses it for.
2291 ///
2292 /// # Why `#[doc(hidden)]` and not a `raw-access` feature
2293 ///
2294 /// T5.1 offers either. The feature is the stronger declaration — it shows up
2295 /// in the consumer's `Cargo.toml`, where a reviewer sees it — and it was
2296 /// **not** taken, for a reason specific to what uses this:
2297 ///
2298 /// Cargo features are additive and cannot be *required* by a test target
2299 /// except through `required-features`, which makes a plain `cargo test`
2300 /// **skip** that binary silently. The binaries that call this are
2301 /// `storage_boundary_tests` and `wave1_regression_tests` — the §4.7
2302 /// tripwires, whose entire job is to fail when a documented gap moves. Gating
2303 /// them behind a feature would mean the ordinary `cargo test` stopped running
2304 /// the tests that enforce the section this item is about, to make a
2305 /// declaration about a hatch. That trade is the wrong way round, and it is
2306 /// the same failure the project already names: a suite that quietly does less
2307 /// than it appears to.
2308 ///
2309 /// So the hatch stays reachable and stops being *discoverable*: it is absent
2310 /// from the docs, and the documented path for every non-write use is
2311 /// [`Self::diagnostic_conn`]. [D-068] is unchanged — removing it would buy
2312 /// the appearance of a guarantee, since the file is reachable by any SQLite
2313 /// client on the machine.
2314 ///
2315 /// [D-068]: ../../docs/architecture/s13-decision-register.md#d-068
2316 // convention (D-068/D-091): `raw()` is #[doc(hidden)] and is NOT exposed by
2317 // any binding. Everything above this line is invisible on docs.rs and
2318 // invisible to a contributor reading the Python surface list, which is where
2319 // the decision to expose it would actually be taken — hence this sentinel and
2320 // its twin in `bindings/python/src/lib.rs` (0.10.0, W4.10). The documented
2321 // path for every non-write use is `diagnostic_conn`.
2322 #[doc(hidden)]
2323 pub fn raw(&self) -> &libsql::Database {
2324 &self.db
2325 }
2326
2327 // -- write surface (§5.1, Appendix A) --
2328 //
2329 // Every method here validates and canonicalises before the value crosses the
2330 // channel, so a bad edge type or a second-precision timestamp is a typed
2331 // error at the call site rather than an engine `CHECK` failure surfacing
2332 // from the far side of an actor with no context attached.
2333 //
2334 // NOTE (§5.1.8, D-028): awaiting one of these waits on a Rust channel, not
2335 // in SQLite, so `busy_timeout` does not bound it. During an in-flight
2336 // `rebuild_current` or `archive` the caller stalls for that transaction's
2337 // duration. Wrap in `tokio::time::timeout` if you need a bound — but a
2338 // timeout is not a cancellation: the command stays queued and commits when
2339 // the actor reaches it.
2340
2341 /// Assert an edge (Doctrine III: a new row, never an update).
2342 ///
2343 /// # One row costs a transaction, so N rows cost N transactions
2344 ///
2345 /// This is the correct method for a caller who genuinely has one edge, and
2346 /// it is the wrong one in a loop. Each call is its own transaction and pays
2347 /// the ~0.8 ms per-transaction floor (D-090) whole, so a thousand edges
2348 /// asserted one at a time spend roughly **0.8 s in transaction overhead
2349 /// alone** — before any of the work — and mint a thousand distinct
2350 /// `recorded_at` stamps for what the caller probably means as one act.
2351 ///
2352 /// There are two bulk forms and the difference between them is the one to
2353 /// get right:
2354 ///
2355 /// - [`Self::bulk_import`] is **chunked** against [`CHUNK_BUDGET`] and
2356 /// atomic per chunk. It amortises the transaction floor across the batch
2357 /// while still yielding to interactive work at every chunk boundary. This
2358 /// is the one a loop should almost always become.
2359 /// - [`Self::write_bulk_atomic`] is one transaction under one stamp and is
2360 /// **the one write with no latency bound** — the hold is a function of
2361 /// `edges.len()`, tabulated in its own docs, and is time every other
2362 /// writer spends waiting. Reach for it when the batch is genuinely one
2363 /// act that must not be observable half-applied, not for speed.
2364 ///
2365 /// The choice is the caller's and neither form is deprecated. Doctrine III
2366 /// makes "one act, one stamp" a semantic claim rather than a performance
2367 /// one, and only the caller knows whether their thousand edges are one act.
2368 pub async fn assert_edge(&self, edge: EdgeAssertion) -> Result<()> {
2369 let edge = edge.normalized()?;
2370 self.high(|responder| HighPriCommand::AssertEdge { edge, responder })
2371 .await
2372 }
2373
2374 /// Close an open interval by asserting its replacement (Doctrine III).
2375 pub async fn retire_edge(
2376 &self,
2377 source: impl Into<String>,
2378 target: impl Into<String>,
2379 edge_type: impl Into<String>,
2380 valid_from: &str,
2381 valid_to: &str,
2382 ) -> Result<()> {
2383 let edge_type = edge_type.into();
2384 crate::graph::edge::validate_edge_type(&edge_type)?;
2385 let valid_from = timestamp::normalize(valid_from)?;
2386 let valid_to = timestamp::normalize(valid_to)?;
2387 let (source, target) = (source.into(), target.into());
2388
2389 self.high(|responder| HighPriCommand::RetireEdge {
2390 source,
2391 target,
2392 edge_type,
2393 valid_from,
2394 valid_to,
2395 branch: None,
2396 responder,
2397 })
2398 .await
2399 }
2400
2401 /// Retire an edge **on a lineage**, which is a different write (0.14.8).
2402 ///
2403 /// The `_on` suffix is the crate's established spelling for the
2404 /// branch-taking variant of a call whose trunk form predates branching —
2405 /// [`query_as_of_edges_on`](crate::temporal::query_as_of_edges_on) is the
2406 /// other one. A sixth positional `Option<BranchId>` on
2407 /// [`Self::retire_edge`] would have made every existing call site read as
2408 /// though it had made a lineage decision it never made.
2409 ///
2410 /// # This closes a row; it does not close *the* row
2411 ///
2412 /// Retiring an edge the branch **inherited** writes the branch's own row at
2413 /// the ancestor's key, carrying the closed interval and this lineage's id.
2414 /// The ancestor's row is untouched, and the read prefers the nearer one, so
2415 /// the edge is gone from this lineage's view and unchanged in its parent's.
2416 /// That is **shadow retirement**, and it is the only retirement across
2417 /// lineages that does not commit the parent corruption
2418 /// [Doctrine III](../../docs/architecture/s0-s3-foundations.md#doctrine-iii)
2419 /// forbids — which is not a rule this method obeys but a shape the ledger
2420 /// cannot express: `links` is append-only and no statement in this crate
2421 /// closes a row in place.
2422 ///
2423 /// `weight` and `properties` are carried over from the visible row rather
2424 /// than restated, which is what makes this a retirement rather than a new
2425 /// assertion that happens to be closed.
2426 ///
2427 /// # Errors
2428 ///
2429 /// - [`DbError::UnknownBranch`] when `branch` is not registered.
2430 /// - [`DbError::NotFound`] when this lineage can see no open row at that
2431 /// `valid_from`. On a branch that includes *never inherited it* and
2432 /// *inherited it and already shadowed it*, which are one answer here
2433 /// because they are one answer to the question asked: there is nothing
2434 /// at that key to retire.
2435 pub async fn retire_edge_on(
2436 &self,
2437 source: impl Into<String>,
2438 target: impl Into<String>,
2439 edge_type: impl Into<String>,
2440 valid_from: &str,
2441 valid_to: &str,
2442 branch: crate::branch::BranchId,
2443 ) -> Result<()> {
2444 let edge_type = edge_type.into();
2445 crate::graph::edge::validate_edge_type(&edge_type)?;
2446 let valid_from = timestamp::normalize(valid_from)?;
2447 let valid_to = timestamp::normalize(valid_to)?;
2448 let (source, target) = (source.into(), target.into());
2449
2450 self.high(|responder| HighPriCommand::RetireEdge {
2451 source,
2452 target,
2453 edge_type,
2454 valid_from,
2455 valid_to,
2456 branch: Some(branch),
2457 responder,
2458 })
2459 .await
2460 }
2461
2462 /// Insert or update a concept.
2463 ///
2464 /// # One row costs a transaction
2465 ///
2466 /// The same trade [`Self::assert_edge`] describes, for the same reason and
2467 /// with the same ~0.8 ms floor (D-090): correct for one concept, wrong in a
2468 /// loop. [`Self::write_concepts`] takes a `Vec` and commits it as one
2469 /// transaction under one stamp.
2470 ///
2471 /// There is no atomic-across-chunks concept path and none is needed to make
2472 /// the choice: `write_concepts` is chunked against [`CHUNK_BUDGET`] and
2473 /// atomic per chunk, so a large `Vec` is cooperative rather than a stall.
2474 /// The responsiveness argument for writing one row at a time therefore does
2475 /// not apply — the bulk form already yields at every chunk boundary.
2476 pub async fn upsert_concept(&self, concept: ConceptUpsert) -> Result<()> {
2477 let concept = concept.normalized()?;
2478 self.high(|responder| HighPriCommand::UpsertConcept { concept, responder })
2479 .await
2480 }
2481
2482 /// A handle on one lineage (§15.4, 0.14.9, [D-226]).
2483 ///
2484 /// Takes `&Arc<Self>` rather than `&self` because the view holds the handle
2485 /// and must not be able to end it: `close` takes `self` by value and an
2486 /// `Arc` cannot surrender that while a clone survives, so the restriction
2487 /// is structural rather than documented. Sharing the handle is already
2488 /// `Arc<Database>` (§5.1.11), so this asks for nothing a caller did not
2489 /// have.
2490 ///
2491 /// Does no I/O and cannot fail. Whether the lineage is *registered* is
2492 /// asked by every operation on the view, which is where
2493 /// [`DbError::UnknownBranch`] names it.
2494 ///
2495 /// [D-226]: ../../docs/architecture/s13-decision-register.md#d-226
2496 pub fn view(
2497 self: &std::sync::Arc<Self>,
2498 branch: crate::branch::BranchId,
2499 ) -> crate::branch::BranchView {
2500 crate::branch::BranchView::new(std::sync::Arc::clone(self), branch)
2501 }
2502
2503 /// Cut a new lineage from an existing one (§15.2, §15.4).
2504 ///
2505 /// # A fork is O(1) in rows written
2506 ///
2507 /// One row in `branches`, and nothing else. No ledger table is read, copied
2508 /// or touched: a branch inherits its parent's history by *resolution at
2509 /// read* rather than by owning a copy of it, which is what
2510 /// [`TraversalBuilder::on_branch`](crate::graph::TraversalBuilder::on_branch)
2511 /// resolves and 0.14.6 bounds by the fork point. The cost of that choice is
2512 /// on the read side and is measured — [D-220] for the resolution, [D-223]
2513 /// for the cutoff — and the cost of the alternative would be here, as an
2514 /// O(rows) fork and storage multiplied by branch count (§15.3, option 3).
2515 ///
2516 /// # The fork point is *now*, and that is a bound on this release rather
2517 /// than on the design
2518 ///
2519 /// `forked_at` is stamped from the same clock as every other write, so the
2520 /// new lineage sees its parent's history up to this instant. Forking from a
2521 /// *past* instant is a coherent thing to want and the schema has always
2522 /// allowed it — `branches` carries `forked_at` and `created_at` as separate
2523 /// columns under `CHECK (forked_at <= created_at)` — but it is not in this
2524 /// release and is additive when it is.
2525 ///
2526 /// # What this lineage can do
2527 ///
2528 /// It can be **read**: every traversal entry point takes a branch, and on a
2529 /// forked ledger the read resolves along the ancestry and stops at the fork
2530 /// point. Since 0.14.8 it can also be **written** — [`EdgeAssertion`] and
2531 /// [`ConceptUpsert`] carry a lineage, and [`Self::retire_edge_on`] shadows
2532 /// an inherited edge (D-225). Through 0.14.7 they did not, and a caller who
2533 /// forked and then called `assert_edge` got a successful write **on the
2534 /// trunk**; that is fixed rather than documented now.
2535 ///
2536 /// What a branch still may not do is **restate an inherited concept**.
2537 /// `concepts` is keyed by identity, so that is refused as
2538 /// [`DbError::CrossLineage`] — see [`ConceptUpsert::branch`]. Edges are the
2539 /// thing a lineage may hold its own belief about, and superseding one is a
2540 /// row written *beside* the ancestor's rather than over it.
2541 ///
2542 /// # Errors
2543 ///
2544 /// - [`DbError::UnknownBranch`] when `from` is not registered. Named rather
2545 /// than left to the foreign key, because the caller asked about a branch.
2546 /// - [`DbError::BranchExists`] when `name` is taken — including `"main"`,
2547 /// which every database has from its first migration.
2548 /// - [`DbError::ForkPrecedesParent`] when the clock would place this fork
2549 /// point before the parent's *own* — not before the parent's
2550 /// `created_at`, which is what the schema comment promised until 0.14.7
2551 /// and is not checkable: the trunk's `created_at` is stamped during
2552 /// migration from the wall clock, before an injected clock exists, so
2553 /// that rule refuses every fork on every `FakeClock` database (D-224).
2554 /// Reachable with [`FakeClock`](crate::util::FakeClock), and the one
2555 /// refusal here that no `CHECK` could have made — it is cross-row, and a
2556 /// `CHECK` sees one row.
2557 ///
2558 /// # Example
2559 ///
2560 /// ```no_run
2561 /// # use macrame::prelude::*;
2562 /// # async fn f(db: &Database) -> Result<()> {
2563 /// let alt = db.fork(BranchId::new("turn/17/alt/1")?, BranchId::main()).await?;
2564 /// let seen = TraversalBuilder::new("socrates")
2565 /// .on_branch(alt.id.clone())
2566 /// .execute_ids(db.read_conn(), "2026-08-29T00:00:00.000000Z")
2567 /// .await?;
2568 /// # let _ = seen;
2569 /// # Ok(())
2570 /// # }
2571 /// ```
2572 ///
2573 /// [D-220]: ../../docs/architecture/s13-decision-register.md#d-220
2574 /// [D-223]: ../../docs/architecture/s13-decision-register.md#d-223
2575 pub async fn fork(
2576 &self,
2577 name: crate::branch::BranchId,
2578 from: crate::branch::BranchId,
2579 ) -> Result<crate::branch::Branch> {
2580 self.high(|responder| HighPriCommand::Fork {
2581 name,
2582 parent: from,
2583 responder,
2584 })
2585 .await
2586 }
2587
2588 /// Every lineage the ledger knows about, trunk first (§15.4).
2589 ///
2590 /// Read through [`Self::read_conn`] rather than the write actor, which is
2591 /// the difference between this and [`Self::fork`] and is deliberate:
2592 /// `branches` is append-only, so the only way this listing can be stale is
2593 /// by missing a branch created after it was taken, and a caller who wanted
2594 /// to know about that branch would have had to create it. Queueing a read
2595 /// behind the write actor would make listing branches wait on a bulk import
2596 /// for no answer it could change.
2597 ///
2598 /// A database that has never forked returns exactly one row: the trunk,
2599 /// with no parent and no fork point.
2600 pub async fn branches(&self) -> Result<Vec<crate::branch::Branch>> {
2601 crate::branch::list(self.read_conn()).await
2602 }
2603
2604 /// The beliefs `a` holds that `b` does not (§15.4, 0.14.11, D-228).
2605 ///
2606 /// One [`Divergence`](crate::branch::Divergence) per edge key the two
2607 /// lineages disagree about, in key order: `b` holds no belief about it, or
2608 /// holds one with a different interval or weight. Not symmetric —
2609 /// `diff(b, a)` is the other half, and composing the two is *two* snapshots
2610 /// even though each is one.
2611 ///
2612 /// Read through the read connection rather than the actor, like
2613 /// [`Self::branches`], and taken at one snapshot rather than two: see
2614 /// `graph::lineage::diff_sql` for why that decides the shape of the query.
2615 ///
2616 /// There is no instant parameter. A diff filtered to a valid-time instant
2617 /// cannot report the one divergence that is *about* an instant having
2618 /// passed — a branch that retired an edge its parent still holds open — so
2619 /// this compares the whole of both views.
2620 ///
2621 /// # Errors
2622 ///
2623 /// [`DbError::UnknownBranch`], naming whichever of the two is not
2624 /// registered, and `a` first when neither is.
2625 pub async fn diff(
2626 &self,
2627 a: &crate::branch::BranchId,
2628 b: &crate::branch::BranchId,
2629 ) -> Result<Vec<crate::branch::Divergence>> {
2630 crate::branch::diff(self.read_conn(), a, b).await
2631 }
2632
2633 /// Assert many edges in one transaction under one stamp (D-014).
2634 ///
2635 /// # This is the one write with no latency bound, and here is what it costs
2636 ///
2637 /// The batch is one act under one `recorded_at`, so it cannot be chunked —
2638 /// splitting it is the thing this method exists not to do. That makes the
2639 /// actor's hold a function of `edges.len()`, and until now the only
2640 /// statement of that anywhere was the prose "uncapped" in
2641 /// [`CHUNK_BUDGET`]'s table. A caller who stalls every other writer for
2642 /// eight seconds should have been able to predict it from the signature.
2643 ///
2644 /// Measured on libSQL 0.9.30 (T1.3, D-081), holding the actor for:
2645 ///
2646 /// | rows | hold |
2647 /// |---|---|
2648 /// | 500 | ~34 ms |
2649 /// | 2,000 | ~155 ms |
2650 /// | 10,000 | ~1.0 s |
2651 /// | 20,000 | ~2.6 s |
2652 ///
2653 /// [`estimated_bulk_hold`] is that curve as a function, and this method
2654 /// emits a `tracing::warn!` when it predicts more than
2655 /// [`BULK_ATOMIC_WARN_HOLD`]. **The estimate is a shape, not a promise** —
2656 /// see [`estimated_bulk_hold`] for what it is calibrated against and where
2657 /// it will be wrong.
2658 ///
2659 /// A caller who needs the latency bound and not the atomicity wants
2660 /// [`Self::bulk_import`], which is the same write chunked and explicitly not
2661 /// atomic overall (D-011).
2662 pub async fn write_bulk_atomic(&self, edges: Vec<EdgeAssertion>) -> Result<usize> {
2663 let estimate = estimated_bulk_hold(&edges);
2664 if estimate > BULK_ATOMIC_WARN_HOLD {
2665 // Warned here rather than in the actor, and before the send: this is
2666 // the caller's own task, so the log line lands with their span
2667 // attached and names the call site that chose the batch size. By the
2668 // time the actor has it, the only context left is "a large batch".
2669 tracing::warn!(
2670 rows = edges.len(),
2671 estimated_hold_ms = estimate.as_millis() as u64,
2672 "write_bulk_atomic will hold the write actor for roughly \
2673 {estimate:?} — it is atomic by contract (D-014) and cannot be \
2674 chunked. Every other writer waits that long. Use bulk_import \
2675 if the batch does not need to be all-or-nothing."
2676 );
2677 }
2678
2679 let edges = normalize_all(edges)?;
2680 self.high(|responder| HighPriCommand::WriteBulkAtomic { edges, responder })
2681 .await
2682 }
2683
2684 /// Move the WAL back into the main database file (§4.5, F-30, 0.12.13,
2685 /// W5.2, D-156).
2686 ///
2687 /// Runs `PRAGMA wal_checkpoint(FULL)` and then `(TRUNCATE)` on the write
2688 /// connection, as one actor turn, and returns what SQLite reported. **Read
2689 /// [`CheckpointReport::busy`]** — a checkpoint that could not run is an
2690 /// `Ok` whose WAL is still there.
2691 ///
2692 /// Two passes rather than one because **a truncating checkpoint cannot
2693 /// report its own work**: the counts describe the WAL *after* the
2694 /// operation, and after a truncation there is nothing left to describe, so
2695 /// `TRUNCATE` alone answers `busy=0, log=0, checkpointed=0` on success —
2696 /// indistinguishable from having done nothing. `FULL` supplies the frame
2697 /// count and `TRUNCATE` resets the file; `busy` is the union of the two.
2698 ///
2699 /// # When a caller needs this
2700 ///
2701 /// Three cases, and only three:
2702 ///
2703 /// - **Before copying the database file elsewhere.** In WAL mode the `.db`
2704 /// file alone is not the database; recent commits live in the `-wal`. A
2705 /// complete checkpoint is what makes the main file self-contained.
2706 /// - **At the end of a bulk load that turned the automatic checkpointer
2707 /// off.** That is the pairing this method exists for — see
2708 /// [`Tuning::wal_autocheckpoint`]. Disabling autocheckpoint without
2709 /// calling this leaves a WAL that grows for the life of the process.
2710 /// - **Before a long idle period**, to give back the disk.
2711 ///
2712 /// Nobody else should call it on a timer. SQLite checkpoints automatically
2713 /// every 1,000 pages and that default is not changed by this method
2714 /// existing; a periodic explicit checkpoint on top of it buys nothing and
2715 /// takes the write lock to do so.
2716 ///
2717 /// # It takes the write lock, and it is budget-exempt
2718 ///
2719 /// The hold is a function of how many frames have accumulated, which is a
2720 /// function of how long since the last checkpoint — not of anything passed
2721 /// in. It is on [`CHUNK_BUDGET`]'s exemption table for that reason, and it
2722 /// is the one entry there that is not a transaction: there is no smaller
2723 /// unit to chunk into, because the operation *is* the copy.
2724 pub async fn checkpoint(&self) -> Result<CheckpointReport> {
2725 self.high(|responder| HighPriCommand::Checkpoint { responder })
2726 .await
2727 }
2728
2729 /// Rebuild `links_current` from `links` and verify zero drift (§5.8).
2730 ///
2731 /// One transaction holding the write lock for its whole duration, because
2732 /// [D-023] will not let the `DELETE` and the `INSERT` be split: a reader
2733 /// landing between them would see a graph with no edges and no error.
2734 /// [`Self::rebuild_current_chunked`] is the same result with a different
2735 /// latency profile, and is what a populated database wants.
2736 ///
2737 /// The report's `drift_after` is the audit run inside the same transaction,
2738 /// so a repair that did not converge is reported by the call that made it
2739 /// rather than by the next one to look.
2740 ///
2741 /// [D-023]: ../docs/architecture/s13-decision-register.md#d-023
2742 pub async fn rebuild_current(&self) -> Result<RebuildReport> {
2743 self.high(|responder| HighPriCommand::RebuildCurrent { responder })
2744 .await
2745 }
2746
2747 /// Rebuild `links_current` beside itself, in chunks (§5.8, T1.2, D-082).
2748 ///
2749 /// Same result as [`Self::rebuild_current`], different latency profile.
2750 /// `rebuild_current` is one transaction holding the write lock for its whole
2751 /// duration, because D-023 will not let the `DELETE` and the `INSERT` be
2752 /// split: a reader landing between them sees a graph with no edges and no
2753 /// error. This builds the replacement in a shadow table instead — the live
2754 /// table stays live and trigger-maintained throughout — and swaps it in at
2755 /// the end.
2756 ///
2757 /// Each step is its own actor turn, so an interactive assertion can jump the
2758 /// queue between chunks. That is the whole of the improvement, and it is why
2759 /// the loop is here rather than inside the actor's arm (the same reasoning
2760 /// as [`Self::archive_windowed`] and [`Self::bulk_import`]).
2761 ///
2762 /// # What the swap still costs
2763 ///
2764 /// Not microseconds. Index names are global and SQLite has no `ALTER INDEX
2765 /// … RENAME`, so the shadow cannot be built carrying `links_current`'s index
2766 /// names while `links_current` still holds them — and building it under
2767 /// other names would leave the table permanently indexed under names absent
2768 /// from [`CREATE_INDICES`](crate::schema::ddl::CREATE_INDICES), so the next
2769 /// migration would create a second copy of each.
2770 /// `DROP TABLE` frees the names, so the swap transaction is where
2771 /// the three indexes get built. What the chunking moves off the lock is the
2772 /// **projection** — the window function over all of `links` — which is the
2773 /// O(E log E) term.
2774 ///
2775 /// # When this returns an error rather than a repair
2776 ///
2777 /// [`DbError::RebuildInterrupted`] means an archive committed while the
2778 /// shadow was being built. Its deletions are invisible to a catch-up pass
2779 /// keyed on `recorded_at` — a deleted row has no `recorded_at` left to find
2780 /// it by — so the work is discarded rather than swapped in. `links_current`
2781 /// is untouched and the call can simply be retried.
2782 ///
2783 /// Use [`Self::rebuild_current`] when the repair must be one atomic act, or
2784 /// when nothing else is contending for the actor and the extra turns are
2785 /// pure overhead.
2786 pub async fn rebuild_current_chunked(&self) -> Result<RebuildReport> {
2787 use crate::integrity::{ShadowOutcome, ShadowStep};
2788
2789 // Each `else` arm is unreachable: the actor maps each step to its own
2790 // outcome variant. Written as a refutable pattern rather than an
2791 // `unwrap` so that adding a step cannot turn a mismatch into a panic on
2792 // the write path — and `WriterDroppedResponder` is the honest name for
2793 // "the actor answered with something this cannot use".
2794 let ShadowOutcome::Started { build_start, epoch } =
2795 self.shadow_step(ShadowStep::Begin).await?
2796 else {
2797 return Err(DbError::WriterDroppedResponder);
2798 };
2799
2800 let mut after: Option<String> = None;
2801 loop {
2802 let ShadowOutcome::Filled { last } = self
2803 .shadow_step(ShadowStep::Fill {
2804 after: after.take(),
2805 })
2806 .await?
2807 else {
2808 return Err(DbError::WriterDroppedResponder);
2809 };
2810 match last {
2811 Some(last) => after = Some(last),
2812 None => break,
2813 }
2814 }
2815
2816 let ShadowOutcome::Swapped { rows } = self
2817 .shadow_step(ShadowStep::Swap { build_start, epoch })
2818 .await?
2819 else {
2820 return Err(DbError::WriterDroppedResponder);
2821 };
2822
2823 Ok(RebuildReport {
2824 rows_rebuilt: rows,
2825 // Not audited. The chunked path's whole argument is that the
2826 // expensive work happens off the lock, and `audit_current` is two
2827 // `EXCEPT` passes over the projection — the cost D-077 removed from
2828 // the archive for the same reason. A caller who wants the check has
2829 // `audit_current` on the read connection, where it costs nobody the
2830 // write lock.
2831 drift_after: 0,
2832 })
2833 }
2834
2835 /// Run one step of a chunked rebuild, for a caller doing its own scheduling.
2836 ///
2837 /// [`Self::rebuild_current_chunked`] is this in a loop and is what almost
2838 /// everyone wants. This exists because that loop offers no seam: it drives
2839 /// `Begin`, then `Fill` to exhaustion, then `Swap`, and a caller who needs to
2840 /// do something *between* steps — pace them against a frame budget, abandon
2841 /// a rebuild that has run long enough, or provoke the archive interlock in a
2842 /// test — cannot get in.
2843 ///
2844 /// The obligation that comes with it: `epoch` from
2845 /// [`ShadowOutcome::Started`](crate::integrity::ShadowOutcome) must be handed
2846 /// back to [`ShadowStep::Swap`](crate::integrity::ShadowStep), or the
2847 /// archive interlock is defeated and a stale projection can be swapped in.
2848 /// The looping version cannot get that wrong; this one can.
2849 pub async fn shadow_step(
2850 &self,
2851 step: crate::integrity::ShadowStep,
2852 ) -> Result<crate::integrity::ShadowOutcome> {
2853 self.low(|responder| LowPriCommand::ShadowRebuild { step, responder })
2854 .await
2855 }
2856
2857 /// Import edges on the background channel, chunked (D-011).
2858 ///
2859 /// Atomic *per chunk*, not overall: a failure partway leaves earlier chunks
2860 /// committed. That is the tradeoff [`chunk_rows`] documents — use
2861 /// [`Database::write_bulk_atomic`] when the batch must be all-or-nothing.
2862 ///
2863 /// Chunked adaptively, at most [`chunk_rows::EDGES`] rows at a time: that
2864 /// constant is where the loop starts and the largest chunk it will send, and
2865 /// each chunk's measured hold sizes the next against [`CHUNK_BUDGET`]. It is
2866 /// also faster in total than the larger chunks this used through 0.5.5
2867 /// (D-058).
2868 ///
2869 /// A consequence worth planning for: the chunk boundaries — and so the
2870 /// `recorded_at` stamps this import writes — depend on how fast the machine
2871 /// was, not only on how many edges were passed (§5.1.6).
2872 ///
2873 /// # The WAL during a bulk (measured, 0.16.1 — see D-274's diagnostics)
2874 ///
2875 /// Every chunk's commit runs WAL pages back into the database file when
2876 /// SQLite's own autocheckpoint threshold (1,000 pages) is reached, and on a
2877 /// bulk that fires every few chunks. Measured on the 16,000-edge random-pair
2878 /// ladder (medians of 3, `benchmarks/diagnostics/`): 5.0 s with the default,
2879 /// 3.7 s with the threshold at 10,000 pages (`wal_autocheckpoint = 10_000`
2880 /// at open), 3.6 s with it disabled outright — and the checkpoint the bulk
2881 /// was paying is 240 ms once at the end instead. The WAL is the price of the
2882 /// third: ~554 MB for that fixture against 43 MB at the 10,000-page
2883 /// threshold and ~6 MB at the default, so the 10,000-page setting is the
2884 /// recipe unless the disk is known to be large. On an ordered (chain-shaped)
2885 /// import the sweep is a wash — the recipe is for the random-order shape a
2886 /// real importer produces. The knob is per-handle and set at
2887 /// [`Database::open`](crate::Database::open); call
2888 /// [`Self::checkpoint`] once after a disabled-checkpoint bulk.
2889 ///
2890 /// Returns [`BulkInterrupted`] rather than [`DbError`] on failure, because
2891 /// a path that is not all-or-nothing owes its caller the count of what
2892 /// landed (0.13.8, W7.6). `?` into a `Result<_, DbError>` still compiles
2893 /// and drops the count, which is the caller's decision to take.
2894 ///
2895 /// [`Self::bulk_import_with`] adds cancellation and per-chunk progress.
2896 pub async fn bulk_import(&self, edges: Vec<EdgeAssertion>) -> BulkResult<usize> {
2897 self.bulk_import_with(edges, BulkControl::new()).await
2898 }
2899
2900 /// [`Self::bulk_import`] with cancellation and progress (0.13.8, W7.6,
2901 /// D-181).
2902 ///
2903 /// The chunk boundaries this path already has are what make both possible:
2904 /// the loop is between transactions several times a second, which is where
2905 /// a token can be read and a callback run without holding anything.
2906 pub async fn bulk_import_with(
2907 &self,
2908 edges: Vec<EdgeAssertion>,
2909 control: BulkControl,
2910 ) -> BulkResult<usize> {
2911 let edges = normalize_all(edges).map_err(before_any_chunk)?;
2912 self.low_chunked(edges, chunk_rows::EDGES, control, |chunk, responder| {
2913 LowPriCommand::BulkImportChunk { chunk, responder }
2914 })
2915 .await
2916 }
2917
2918 /// Load edges **without the maintained projection in the way**, then
2919 /// re-derive it in one chunked rebuild (D-277, plan §8.1 / F1).
2920 ///
2921 /// Three phases through the actor: drop `trg_links_current_sync` (the
2922 /// per-row upsert into `links_current`), load every edge through the same
2923 /// chunked path [`Self::bulk_import`] uses, restore the trigger, then
2924 /// [`Self::rebuild_current_chunked`] — the re-derivation D-082 made
2925 /// chunked, whose fill costs ~104 ms per 16k rows and whose swap is the
2926 /// exempt 46.8 ms turn. Measured on the reference box, 16,000 random-pair
2927 /// edges (medians of three):
2928 ///
2929 /// | arm | bulk | rebuild + restore | total |
2930 /// |---|---|---|---|
2931 /// | shipped (`bulk_import`) | 5.08 s | — | 5.08 s |
2932 /// | this | 2.29 s | 0.19 s | **2.48 s (2.05×)** |
2933 ///
2934 /// The mirror is what the ladder showed growing with the graph on random
2935 /// pairs (plan §9.2): the near-chain shape pays it too, just flatter, so
2936 /// this is the lever for both — and the rebuild's cost is a function of
2937 /// the table, not the shape, which is why the skip arm's total is flat
2938 /// where the shipped arm's is not.
2939 ///
2940 /// # What the window actually touches, stated precisely
2941 ///
2942 /// **The ledger is complete at every instant of the window.** The log
2943 /// mirror and the single-open guard stay up: `links` and
2944 /// `transaction_log` gain every row this call loads, versioned and
2945 /// guarded exactly as the shipped path does. What lags is
2946 /// `links_current` — Doctrine VI's derivative state, which the crate
2947 /// has always maintained as *the projection you could fold from the
2948 /// ledger*. Current-time reads during the window (`traverse`,
2949 /// `query_as_of_edges` without an instant) see a partial projection and
2950 /// are not wrong about the past — they are stale about the present, the
2951 /// one state Doctrine VI calls disposable.
2952 ///
2953 /// The window **ends when this method returns**, success, failure, or
2954 /// cancellation: the restore runs before the rebuild, so any write that
2955 /// lands after it mirrors again, and the rebuild then sweeps the backlog.
2956 /// If the future is dropped or the process unwinds mid-load, the window
2957 /// stays open — the projection stays stale but the ledger stays complete,
2958 /// `audit_current` reports the drift as [`DbError::CurrentDrift`], and
2959 /// [`Self::rebuild_current_chunked`] closes it. That is the honest doc
2960 /// entry plan §8.1 asked for, and why this is a signature rather than a
2961 /// default: the shipped path never has a window at all.
2962 ///
2963 /// # What a failure does
2964 ///
2965 /// The load's failure — [`BulkInterrupted`] with its `written` count —
2966 /// propagates only **after** the trigger is restored and the projection
2967 /// rebuilt from what did commit, so a failed deferred bulk is in exactly
2968 /// the state a caller expects: the prefix is committed, the projection is
2969 /// true, the mirror is on. An empty `edges` is a no-op: no DDL, no
2970 /// rebuild, nothing attributed.
2971 pub async fn bulk_import_deferred(
2972 &self,
2973 edges: Vec<EdgeAssertion>,
2974 ) -> BulkResult<usize> {
2975 self.bulk_import_deferred_with(edges, BulkControl::new())
2976 .await
2977 }
2978
2979 /// [`Self::bulk_import_deferred`] with cancellation and progress, on the
2980 /// load half — the same chunked loop, the same token, the same callbacks
2981 /// [`Self::bulk_import_with`] takes. The toggle and the rebuild are
2982 /// single turns around it.
2983 pub async fn bulk_import_deferred_with(
2984 &self,
2985 edges: Vec<EdgeAssertion>,
2986 control: BulkControl,
2987 ) -> BulkResult<usize> {
2988 let edges = normalize_all(edges).map_err(before_any_chunk)?;
2989 if edges.is_empty() {
2990 return Ok(0);
2991 }
2992
2993 if let Err(cause) = self
2994 .low(|responder| LowPriCommand::LinksCurrentMirror {
2995 present: false,
2996 responder,
2997 })
2998 .await
2999 {
3000 // A toggle that failed has left the mirror exactly where it was —
3001 // on. Stopping here is safe: nothing was loaded, nothing is stale.
3002 return Err(BulkInterrupted { written: 0, cause });
3003 }
3004
3005 let loaded = self
3006 .low_chunked(edges, chunk_rows::EDGES, control, |chunk, responder| {
3007 LowPriCommand::BulkImportChunk { chunk, responder }
3008 })
3009 .await;
3010
3011 // Restore before reporting, and before rebuilding: any write that
3012 // lands after the restore maintains the projection itself, and the
3013 // rebuild then sweeps whatever the window left behind.
3014 let restored = self
3015 .low(|responder| LowPriCommand::LinksCurrentMirror {
3016 present: true,
3017 responder,
3018 })
3019 .await;
3020 let rebuilt = self.rebuild_current_chunked().await;
3021
3022 // The mirror's absence out-ranks the rebuild's failure out-ranks the
3023 // load's: a caller must not read success beside a mirror that is off,
3024 // and a load error beside a restored-and-rebuilt projection is just
3025 // the load's answer.
3026 let written = match &loaded {
3027 Ok(n) => *n,
3028 Err(e) => e.written,
3029 };
3030 match (loaded, restored, rebuilt) {
3031 (Ok(n), Ok(()), Ok(_)) => Ok(n),
3032 // A failed load with the mirror restored and the projection
3033 // rebuilt is the load's own answer — the prefix committed, the
3034 // projection is true, the mirror is on.
3035 (Err(e), Ok(()), Ok(_)) => Err(e),
3036 (_, Err(cause), _) => Err(BulkInterrupted { written, cause }),
3037 (Err(e), Ok(()), Err(report)) => Err(BulkInterrupted {
3038 written: e.written,
3039 cause: report,
3040 }),
3041 (Ok(n), Ok(()), Err(report)) => Err(BulkInterrupted {
3042 written: n,
3043 cause: report,
3044 }),
3045 }
3046 }
3047
3048 /// Upsert many **concepts** on the background channel, chunked (D-011).
3049 ///
3050 /// This is the bulk concept path, and every row it writes is a ledger write:
3051 /// it versions the concept and lands in `transaction_log`. Derived analytics
3052 /// output does not belong here — see
3053 /// [`Database::write_analytics_annotations`] and D-041.
3054 ///
3055 /// Called `write_annotations` through 0.5.6, from when the two writes were
3056 /// one call. D-041 split them and the name stayed on the wrong one for three
3057 /// releases, so the crate had a `write_annotations` that wrote concepts
3058 /// sitting beside a `write_analytics_annotations` that wrote annotations
3059 /// (D-075).
3060 ///
3061 /// Chunked, so it returns [`BulkInterrupted`] and its `written` count on
3062 /// failure (0.13.8, W7.6); [`Self::write_concepts_with`] adds cancellation
3063 /// and progress.
3064 pub async fn write_concepts(&self, concepts: Vec<ConceptUpsert>) -> BulkResult<usize> {
3065 self.write_concepts_with(concepts, BulkControl::new()).await
3066 }
3067
3068 /// [`Self::write_concepts`] with cancellation and progress (0.13.8, W7.6).
3069 pub async fn write_concepts_with(
3070 &self,
3071 concepts: Vec<ConceptUpsert>,
3072 control: BulkControl,
3073 ) -> BulkResult<usize> {
3074 let concepts: Vec<ConceptUpsert> = concepts
3075 .into_iter()
3076 .map(ConceptUpsert::normalized)
3077 .collect::<Result<_>>()
3078 .map_err(before_any_chunk)?;
3079 self.low_chunked(
3080 concepts,
3081 chunk_rows::CONCEPTS,
3082 control,
3083 |chunk, responder| LowPriCommand::WriteConceptsChunk { chunk, responder },
3084 )
3085 .await
3086 }
3087
3088 /// State as believed at `ts` (§5.5, D-026, D-049).
3089 ///
3090 /// A read: it runs on `read_conn` and never touches the Write Actor, so a
3091 /// reconstruction and a full-speed write-back do not slow each other.
3092 ///
3093 /// Prefer this to calling [`crate::temporal::reconstruct`] directly. The
3094 /// free function takes the archive path and the snapshot directory as
3095 /// arguments, and a caller who passes `None` for the second gets a correct
3096 /// answer that folds the whole log every time — the composition is opt-in
3097 /// at that layer and easy to leave off by accident. Here both come from the
3098 /// handle, so the fast path is the default one.
3099 pub async fn reconstruct(&self, ts: &str) -> Result<crate::temporal::MaterializedState> {
3100 let ts = timestamp::normalize(ts)?;
3101 crate::temporal::reconstruct(
3102 &self.read_conn,
3103 &ts,
3104 Some(&self.archive_path),
3105 Some(&self.snapshots_dir),
3106 )
3107 .await
3108 }
3109
3110 /// State at `ts` as `branch` saw it (0.15.17, [D-259], review C-10).
3111 ///
3112 /// [`Self::reconstruct`] with the ancestry resolved: each ancestor bounded
3113 /// at its fork point, one belief per edge key from the nearest lineage
3114 /// holding it. See [`crate::temporal::reconstruct_on`] for how it is
3115 /// assembled, what it costs, and the two things it does **not** do —
3116 /// concepts are not resolved by lineage, and the result must not be saved
3117 /// as a snapshot.
3118 ///
3119 /// A read, on `read_conn`, like [`Self::reconstruct`]. The archive path and
3120 /// the snapshot directory come from the handle, so snapshot composition is
3121 /// on by default for each of the folds this runs.
3122 ///
3123 /// [D-259]: ../../docs/architecture/s13-decision-register.md#d-259
3124 pub async fn reconstruct_on(
3125 &self,
3126 ts: &str,
3127 branch: &str,
3128 ) -> Result<crate::temporal::MaterializedState> {
3129 let ts = timestamp::normalize(ts)?;
3130 crate::temporal::reconstruct_on(
3131 &self.read_conn,
3132 &ts,
3133 branch,
3134 Some(&self.archive_path),
3135 Some(&self.snapshots_dir),
3136 )
3137 .await
3138 }
3139
3140 /// `branch`'s ancestry, nearest first, each with its fork-point cutoff.
3141 ///
3142 /// The input [`crate::temporal::resolve_beliefs`] takes. Resolved from
3143 /// `branches` in Rust since 0.15.17 ([D-259]) — the walk is a few
3144 /// microseconds and the table is tiny and append-only, so this is a read
3145 /// like any other rather than something to cache.
3146 ///
3147 /// The trunk of an unforked database answers with one row and no cutoff,
3148 /// which is its true ancestry. A lineage that is not registered is refused
3149 /// by name with [`DbError::UnknownBranch`].
3150 ///
3151 /// [D-259]: ../../docs/architecture/s13-decision-register.md#d-259
3152 pub async fn ancestry(&self, branch: &str) -> Result<Vec<crate::branch::Ancestor>> {
3153 let lineages = crate::graph::lineage::Lineages::load(&self.read_conn).await?;
3154 // Checked before it is walked: `resolve` answers for a name it has never
3155 // seen with a one-row ancestry, which is the right answer for a root and
3156 // the D-069 wrong-looking-right answer for a typo.
3157 lineages.shape(branch)?;
3158 Ok(lineages.ancestry(branch))
3159 }
3160
3161 /// Every edge one [`ReadPlan`] names (0.15.9, W13.4, [D-251]).
3162 ///
3163 /// The whole projection filtered to the plan's instants and lineage —
3164 /// topology only, no start node, and no budget on the answer. On a large
3165 /// ledger that is a large `Vec`; [`Self::load_subgraph`] is the bounded
3166 /// neighbourhood read and [`crate::graph::TraversalBuilder`] is the
3167 /// anchored one.
3168 ///
3169 /// # What this can express that nothing else could
3170 ///
3171 /// [`crate::temporal::query_as_of_edges_on`] is the same read at a
3172 /// valid-time instant, and it takes no transaction-time one: before this
3173 /// release, *"which edges did we believe existed, as of March, as they
3174 /// stood in January"* had exactly two answers available — walk it from a
3175 /// start node, or fold the entire log with [`Self::reconstruct`] and filter
3176 /// the result. The first needs an anchor the question does not have and the
3177 /// second is a different order of work. The fold this uses is the
3178 /// traversal's own, so the bitemporal cell is now readable whole at the
3179 /// cost of reading it.
3180 ///
3181 /// The two functions share one statement, which is why neither can drift
3182 /// from the other; `query_as_of_edges_on` is this with `recorded` unset and
3183 /// the lineage dropped from each row.
3184 ///
3185 /// # Errors
3186 ///
3187 /// [`DbError::UnknownBranch`] naming a
3188 /// lineage that was never registered — refused rather than answered for the
3189 /// trunk, for `graph::lineage::Lineages::shape`'s reason.
3190 /// [`DbError::RecordedInstantUnreachable`]
3191 /// when [`ReadPlan::recorded`] is below what the hot log still covers
3192 /// ([D-247](../../docs/architecture/s13-decision-register.md#d-247)).
3193 /// [`DbError::InvalidTimestamp`] for a
3194 /// stamp that is not canonical, from the same normaliser every other read
3195 /// uses — a plan is inert and validates nothing, so this is where a
3196 /// malformed instant is noticed.
3197 ///
3198 /// [D-251]: ../../docs/architecture/s13-decision-register.md#d-251
3199 pub async fn edges(&self, plan: ReadPlan) -> Result<Vec<crate::temporal::EdgeBelief>> {
3200 // `None` is now, and now is this handle's clock rather than the
3201 // system's: a database opened on a `FakeClock` reads at the instant it
3202 // is writing at, which is the whole reason the clock is a handle
3203 // property (§5.1.1).
3204 let valid = match plan.valid.as_deref() {
3205 Some(ts) => timestamp::normalize(ts)?,
3206 None => self.clock.now(),
3207 };
3208 let recorded = plan
3209 .recorded
3210 .as_deref()
3211 .map(timestamp::normalize)
3212 .transpose()?;
3213 crate::plan::edges_at(
3214 &self.read_conn,
3215 &valid,
3216 recorded.as_deref(),
3217 plan.branch_name(),
3218 plan.limit,
3219 )
3220 .await
3221 }
3222
3223 /// Create a model's embedding table and DiskANN index (§5.9, D-048).
3224 ///
3225 /// Idempotent: registering a model that already exists at the same
3226 /// dimension succeeds, and at a different dimension fails with
3227 /// [`DbError::DimMismatch`] naming both, rather than no-opping through
3228 /// `IF NOT EXISTS` and leaving the caller believing the dimension they
3229 /// asked for is the one in force.
3230 ///
3231 /// This issues DDL, which everywhere else in the crate is the migration
3232 /// runner's exclusive business (D-032). The exception is bounded and
3233 /// deliberate: a model's table is created once, by an explicit call, and
3234 /// the alternative — a caller-supplied write connection — is the very thing
3235 /// the Write Actor exists to make impossible.
3236 ///
3237 /// # Latency
3238 ///
3239 /// One small transaction, but it queues like any other write: see §5.1.8.
3240 pub async fn register_model(&self, model: &ModelName, dim: usize) -> Result<()> {
3241 let model = model.clone();
3242 self.high(|responder| HighPriCommand::RegisterModel {
3243 model,
3244 dim,
3245 responder,
3246 })
3247 .await
3248 }
3249
3250 /// Store or replace vectors for `model`, chunked (§5.9, D-011, D-048).
3251 ///
3252 /// The write path for embeddings. Before 0.5.4 there was none:
3253 /// [`crate::vector::upsert_embedding`] takes a raw connection, `read_conn`
3254 /// is `query_only`, and the write connection lives inside the actor — so an
3255 /// application could search vectors it had no way to store.
3256 ///
3257 /// Low priority and chunked at [`chunk_rows::EMBEDDINGS`], because embedding
3258 /// is bulk derived work: a 50,000-vector backfill must yield to an
3259 /// interactive assertion at every chunk boundary. That constant is the
3260 /// smallest of the four by a wide margin — DiskANN index maintenance makes an
3261 /// embedding the most expensive row in the system (D-058). Atomic per chunk, not overall, which
3262 /// is the same trade [`Database::bulk_import`] makes and is safer here than
3263 /// there — an embedding is derived (Doctrine VII), so a partially written
3264 /// batch is recoverable by re-embedding.
3265 ///
3266 /// Fails with [`DbError::ModelNotRegistered`] if `model` has no table, and
3267 /// [`DbError::DimMismatch`] if a vector's length is not the declared
3268 /// dimension. The dimension is read from the schema once per chunk (D-037):
3269 /// the crate keeps no registry of its own to fall out of date.
3270 ///
3271 /// Chunked, so it returns [`BulkInterrupted`] and its `written` count on
3272 /// failure (0.13.8, W7.6). A 50,000-vector backfill is the longest-running
3273 /// write the crate has, which makes it the one most likely to be cancelled
3274 /// — [`Self::upsert_embeddings_with`] is how.
3275 pub async fn upsert_embeddings(
3276 &self,
3277 model: &ModelName,
3278 rows: Vec<(String, Vec<f32>)>,
3279 ) -> BulkResult<usize> {
3280 self.upsert_embeddings_with(model, rows, BulkControl::new())
3281 .await
3282 }
3283
3284 /// [`Self::upsert_embeddings`] with cancellation and progress (0.13.8,
3285 /// W7.6).
3286 pub async fn upsert_embeddings_with(
3287 &self,
3288 model: &ModelName,
3289 rows: Vec<(String, Vec<f32>)>,
3290 control: BulkControl,
3291 ) -> BulkResult<usize> {
3292 self.low_chunked(rows, chunk_rows::EMBEDDINGS, control, |chunk, responder| {
3293 LowPriCommand::UpsertEmbeddingChunk {
3294 model: model.clone(),
3295 chunk,
3296 responder,
3297 }
3298 })
3299 .await
3300 }
3301
3302 /// Load embeddings **without the DiskANN index in the way**, then rebuild
3303 /// it in one pass (D-276, plan §9.1).
3304 ///
3305 /// Three actor turns in sequence: drop the index (`DROP INDEX IF EXISTS`,
3306 /// µs-scale), load every row through the same chunked path
3307 /// [`Self::upsert_embeddings`] uses, rebuild the index in one statement.
3308 /// Measured on the reference box, medians of three, 2,000 vectors:
3309 ///
3310 /// | dim | indexed (`upsert_embeddings`) | `bulk_embeddings` | one-pass build alone |
3311 /// |---|---|---|---|
3312 /// | 64 | 3.78 s | 2.62 s | 2.61 s |
3313 /// | 256 | 31.0 s | 19.7 s | 19.7 s |
3314 /// | 512 | 56.0 s | 39.0 s | 39.0 s |
3315 ///
3316 /// At 5,000 × 256: 89.2 s indexed, 48.6 s here — **1.8×**. The build, not
3317 /// the blob writes, is the cost: inserts without the index are flat at
3318 /// ~5 µs/row at every dimension measured. The rest of the gap against an
3319 /// HNSW writer is DiskANN-vs-HNSW build economics inside libSQL, engine-
3320 /// side, and out of scope for a crate that does not fork its engine.
3321 ///
3322 /// # The trade, stated
3323 ///
3324 /// Between the drop and the rebuild, the model's vectors are not
3325 /// searchable (`vector_top_k` reports the missing index) and **not
3326 /// dimension-checked at the storage layer** — the index is that check
3327 /// (D-037, `ddl::create_embeddings_index`). The crate-side check
3328 /// ([`crate::vector::EmbeddingCodec::encode`]) still applies to everything
3329 /// this method loads, and the rebuild restores the storage check at the
3330 /// end; what is given up is the check on rows a *different* client inserts
3331 /// during the window. That is the measured decision, opt-in by signature:
3332 /// the plain path keeps the storage backstop at every instant.
3333 ///
3334 /// **A failed or cancelled load still rebuilds.** The load's own failure —
3335 /// [`BulkInterrupted`] with its `written` count — propagates only after
3336 /// the rebuild has run, so a bulk load can never leave the file in the
3337 /// disarmed state. An empty `rows` is a no-op: the index is not touched
3338 /// for a load of nothing.
3339 ///
3340 /// The rebuild turn is budget-exempt (atomic by necessity, same criterion
3341 /// as `shadow_swap` — [`crate::CHUNK_BUDGET`]'s table), and its hold grows
3342 /// with the corpus: ~10 ms/vector at dim 256, ~20 ms at 512. For a very
3343 /// large backfill that hold is the price of the recipe; the alternative —
3344 /// keep the index and pay the same total spread across rows — is what
3345 /// `upsert_embeddings` already is.
3346 pub async fn bulk_embeddings(
3347 &self,
3348 model: &ModelName,
3349 rows: Vec<(String, Vec<f32>)>,
3350 ) -> BulkResult<usize> {
3351 self.bulk_embeddings_with(model, rows, BulkControl::new())
3352 .await
3353 }
3354
3355 /// [`Self::bulk_embeddings`] with cancellation and progress, on the load
3356 /// half. The drop and the rebuild are single statements; `progress` and
3357 /// `cancel` bound the chunked load between them, exactly as they do for
3358 /// [`Self::upsert_embeddings_with`].
3359 pub async fn bulk_embeddings_with(
3360 &self,
3361 model: &ModelName,
3362 rows: Vec<(String, Vec<f32>)>,
3363 control: BulkControl,
3364 ) -> BulkResult<usize> {
3365 // The table must exist before anything drops its index, and the first
3366 // row's width is checked *before* the drop so a wholly wrong batch
3367 // fails without ever disarming the storage check.
3368 let dim = match crate::vector::declared_dimension(self.read_conn(), model).await {
3369 Ok(dim) => dim,
3370 Err(cause) => return Err(BulkInterrupted { written: 0, cause }),
3371 };
3372 if let Some((_, first)) = rows.first() {
3373 if first.len() != dim {
3374 return Err(BulkInterrupted {
3375 written: 0,
3376 cause: DbError::DimMismatch {
3377 got: first.len(),
3378 expected: dim,
3379 model: model.to_string(),
3380 },
3381 });
3382 }
3383 } else {
3384 return Ok(0);
3385 }
3386
3387 if let Err(cause) = self
3388 .low(move |responder| LowPriCommand::DropEmbeddingIndex {
3389 model: model.clone(),
3390 responder,
3391 })
3392 .await
3393 {
3394 // A drop that failed has left the index in place — the armed,
3395 // searchable state — so stopping here is safe: the load never
3396 // ran, and the file is exactly what it was.
3397 return Err(BulkInterrupted { written: 0, cause });
3398 }
3399
3400 let loaded = self
3401 .low_chunked(rows, chunk_rows::EMBEDDINGS, control, |chunk, responder| {
3402 LowPriCommand::UpsertEmbeddingChunk {
3403 model: model.clone(),
3404 chunk,
3405 responder,
3406 }
3407 })
3408 .await;
3409
3410 // Whether the load committed or was interrupted, the index goes back:
3411 // the one state this method must never leave the file in is the state
3412 // `drop_embedding_index` documents as unsearchable and unchecked.
3413 let rebuilt = self
3414 .low(move |responder| LowPriCommand::RebuildEmbeddingIndex {
3415 model: model.clone(),
3416 responder,
3417 })
3418 .await;
3419
3420 // The load's failure is the caller's answer; the rebuild's failure is
3421 // worse and takes precedence, because a load that reports success
3422 // beside a missing index is the silent half of the same defect. In the
3423 // double-failure case the rebuild becomes the cause and the load's
3424 // `written` count survives as the count.
3425 match (loaded, rebuilt) {
3426 (Ok(count), Ok(())) => Ok(count),
3427 (Err(e), Ok(())) => Err(e),
3428 (Ok(count), Err(e)) => Err(BulkInterrupted {
3429 written: count,
3430 cause: e,
3431 }),
3432 (Err(load), Err(rebuild)) => Err(BulkInterrupted {
3433 written: load.written,
3434 cause: rebuild,
3435 }),
3436 }
3437 }
3438
3439 /// Reconstruct the concept-text search index from the ledger (§5.9, D-036).
3440 ///
3441 /// The FTS index is derivative: D-036 promises every derivative table can be
3442 /// rebuilt from the ledger tables, and this is that promise made callable
3443 /// for `concepts_fts`. Needed after a restore that skipped the shadow
3444 /// tables, or if the index is ever suspected of drifting from the text —
3445 /// and, as a matter of policy, cheaper to run than to reason about.
3446 ///
3447 /// The work is `INSERT INTO concepts_fts(concepts_fts) VALUES('rebuild')`,
3448 /// which is FTS5's own operation over the content table, so this is not a
3449 /// second implementation of the sync triggers that could disagree with them.
3450 pub async fn rebuild_fts(&self) -> Result<()> {
3451 self.low(|responder| LowPriCommand::RebuildFts { responder })
3452 .await
3453 }
3454
3455 /// Refresh the query planner's statistics (0.12.4, [D-149]).
3456 ///
3457 /// Runs `ANALYZE`, which writes `sqlite_stat1`. **Before 0.12.4 nothing in
3458 /// this crate ever did**, so the planner costed every query against SQLite's
3459 /// built-in defaults — assume ~1M rows, assume each bound equality column
3460 /// divides by ten. That estimate is structural: it depends on how many
3461 /// columns a query binds, not on what the table contains.
3462 ///
3463 /// Which is this schema's own worst defect restated. D-042, D-059 and D-064
3464 /// are three occasions where *a covering index captured a query because it
3465 /// contained the columns, not because it discriminated*, and two of the four
3466 /// declared indices lead on the same column. Statistics are what let the
3467 /// planner tell them apart by measurement instead of by shape.
3468 ///
3469 /// # Cost, and why it is bounded
3470 ///
3471 /// This is a write and it takes the write lock. `PRAGMA analysis_limit`
3472 /// (set per connection, see [`ddl::ANALYSIS_LIMIT`]) caps the rows examined
3473 /// per index. It is scheduled as low-priority work and will not preempt an
3474 /// interactive assertion.
3475 ///
3476 /// **The bound is a constant factor, not an independence** (0.12.23,
3477 /// D-166). This rustdoc said the hold "scales with the number of indices —
3478 /// four — and not with the size of `links_current`", which is measurably
3479 /// wrong: the pragma is worth 3–4× and what remains still grows with the
3480 /// table. Measured, `examples/analyze_hold.rs`: **5.26 ms at 10,000 edges,
3481 /// 19.1 ms at 40,000**, against a 3 ms [`crate::CHUNK_BUDGET`].
3482 ///
3483 /// So this call **misses the budget by ~6× on a moderately sized ledger**,
3484 /// and [`crate::metrics::CommandKind::Analyze`] is deliberately not among
3485 /// the budget-exempt kinds — `metrics().budget_violations()` names it. That
3486 /// is the honest position: the work is low priority and preemptible between
3487 /// commands, but it is one indivisible statement and cannot be chunked, so
3488 /// the hold is what it is. Prefer [`optimize`], which does nothing when
3489 /// nothing has moved.
3490 ///
3491 /// **Since 0.13.24 the counter is this call and not also [`optimize`]**
3492 /// (W10.5, [D-197]). The two shared `CommandKind::Analyze` until then, which
3493 /// is why an `analyze` row in `budget_violations()` used to be unreadable:
3494 /// it could have been an explicit call or a handle close.
3495 ///
3496 /// # When to call it
3497 ///
3498 /// After a bulk import, and after anything that changes a table's shape by
3499 /// an order of magnitude. Prefer [`optimize`] for routine upkeep: it does
3500 /// nothing when nothing has moved, and this does the work unconditionally.
3501 ///
3502 /// Statistics are derived state in the sense Doctrine VI means it — deleting
3503 /// `sqlite_stat1` costs plan quality and no information, and this call
3504 /// rebuilds it.
3505 ///
3506 /// [D-149]: ../docs/architecture/s13-decision-register.md#d-149
3507 /// [`ddl::ANALYSIS_LIMIT`]: crate::schema::ddl::ANALYSIS_LIMIT
3508 /// [`optimize`]: Database::optimize
3509 pub async fn analyze(&self) -> Result<()> {
3510 self.low(|responder| LowPriCommand::Analyze {
3511 incremental: false,
3512 responder,
3513 })
3514 .await
3515 }
3516
3517 /// Re-analyse only what has gone stale (0.12.4, [D-149]).
3518 ///
3519 /// `PRAGMA optimize`. SQLite tracks how far each table has drifted since its
3520 /// last analysis and re-analyses only where it believes the statistics no
3521 /// longer hold — so this is a no-op on an idle database and the full cost of
3522 /// [`analyze`] on one that has changed completely.
3523 ///
3524 /// That property is the whole point: it is safe to call on a schedule, where
3525 /// [`analyze`] is not. `close()` runs it, so a process that opens, works and
3526 /// closes keeps its statistics current without anybody arranging it.
3527 ///
3528 /// # What it costs, measured, and the threshold it applies rather than takes
3529 /// (0.13.24, W10.5, [D-197])
3530 ///
3531 /// `examples/optimize_hold.rs`, on a 40,000-edge ledger: **10.7 ms the
3532 /// first time on a database that has never been analysed** — there is
3533 /// nothing incremental about the first call — and **90–220 µs every time
3534 /// after**, well inside [`crate::CHUNK_BUDGET`].
3535 ///
3536 /// **The staleness test is SQLite's and it is a ratio, not a row count.**
3537 /// Measured by reading `sqlite_stat1` across the call rather than by timing
3538 /// it: growth of 2× and 5× both left the statistics **untouched**, and
3539 /// only at 25× did it re-analyse — for a 460 ms hold. So this is not a
3540 /// cheaper `analyze()` and calling it after a bulk load is not a way to
3541 /// refresh statistics the load invalidated: below the ratio it declines,
3542 /// and above it it costs what [`analyze`] costs. It reports as
3543 /// [`crate::metrics::CommandKind::Optimize`] since 0.13.24, which is what
3544 /// makes those two outcomes distinguishable in the metrics at all.
3545 ///
3546 /// [D-197]: ../docs/architecture/s13-decision-register.md#d-197
3547 ///
3548 /// [D-149]: ../docs/architecture/s13-decision-register.md#d-149
3549 /// [`analyze`]: Database::analyze
3550 pub async fn optimize(&self) -> Result<()> {
3551 self.low(|responder| LowPriCommand::Analyze {
3552 incremental: true,
3553 responder,
3554 })
3555 .await
3556 }
3557
3558 // **There is deliberately no `verify_fts()` (§5.9, D-071).**
3559 //
3560 // `rebuild_fts` is the repair with no way to ask whether it is needed, and
3561 // Wave 5 set out to add the missing half. FTS5 offers `'integrity-check'`,
3562 // which looked like exactly the engine-provided answer this crate prefers.
3563 // It is not: on libSQL 0.9.30 it verifies the index's *internal* consistency
3564 // and not its agreement with the content table. Measured — after
3565 // `'delete-all'` the index matches nothing where it matched ten rows, and
3566 // both `'integrity-check'` and `'integrity-check', 0` still report success.
3567 //
3568 // A `verify_fts()` on that footing would answer "healthy" for an empty
3569 // index, which is worse than having no method at all: it is the shape of
3570 // defect AC, a function that looks like it checks something and does not.
3571 // `an_emptied_fts_index_still_passes_integrity_check` pins the limitation so
3572 // that if a later libSQL fixes it, the test fails and says so.
3573
3574 /// Write derived analytics results on the background channel, chunked
3575 /// (§5.4, D-041).
3576 ///
3577 /// Rows go to `analytics_annotations`, which has no log trigger, so nothing
3578 /// written here reaches `transaction_log` and nothing here versions a
3579 /// concept. Rerunning an algorithm replaces the previous pass rather than
3580 /// recording that the world changed.
3581 ///
3582 /// Low priority and chunked at up to [`chunk_rows::ANNOTATIONS`] — the
3583 /// largest ceiling of the four, because this is the only bulk table carrying
3584 /// no triggers at all
3585 /// and its rows are correspondingly cheap (D-058) — so a 50,000-label Louvain
3586 /// save yields to interactive writes at every chunk boundary and carries the
3587 /// per-chunk fidelity boundary of §5.1.6 — a partially written pass is
3588 /// recoverable by rerunning, which is the property that makes derived state
3589 /// safe to write this way and assertions not.
3590 ///
3591 /// Chunked, so it returns [`BulkInterrupted`] and its `written` count on
3592 /// failure (0.13.8, W7.6); [`Self::write_analytics_annotations_with`] adds
3593 /// cancellation and progress.
3594 pub async fn write_analytics_annotations(
3595 &self,
3596 annotations: Vec<Annotation>,
3597 ) -> BulkResult<usize> {
3598 self.write_analytics_annotations_with(annotations, BulkControl::new())
3599 .await
3600 }
3601
3602 /// [`Self::write_analytics_annotations`] with cancellation and progress
3603 /// (0.13.8, W7.6).
3604 pub async fn write_analytics_annotations_with(
3605 &self,
3606 annotations: Vec<Annotation>,
3607 control: BulkControl,
3608 ) -> BulkResult<usize> {
3609 self.low_chunked(
3610 annotations,
3611 chunk_rows::ANNOTATIONS,
3612 control,
3613 |chunk, responder| LowPriCommand::WriteAnalyticsChunk { chunk, responder },
3614 )
3615 .await
3616 }
3617
3618 /// Move closed intervals and superseded log rows older than `cutoff` to the
3619 /// cold database (§5.7, D-012).
3620 pub async fn archive(&self, cutoff: &str) -> Result<ArchiveReport> {
3621 let cutoff = timestamp::normalize(cutoff)?;
3622 let archive_path = self.archive_path.clone();
3623 self.low(|responder| LowPriCommand::Archive {
3624 cutoff,
3625 archive_path,
3626 responder,
3627 })
3628 .await
3629 }
3630
3631 /// Forget one lineage: move its whole ledger to the cold database and
3632 /// remove the lineage record (0.14.13, §15.4, D-230).
3633 ///
3634 /// The abandonment arm. A conversation tree discards most of what it grows,
3635 /// and [`Self::archive`] cannot reclaim it: that arm is indexed by *time*,
3636 /// so archiving an abandoned branch's recent history means archiving the
3637 /// trunk's recent history with it.
3638 ///
3639 /// **Everything the lineage holds moves in one transaction** — its `links`,
3640 /// its `concepts`, its `transaction_log` entries and its `branches` row —
3641 /// and afterwards the name is unknown: every read and write naming it
3642 /// raises [`DbError::UnknownBranch`]. That is the design's whole shape, and
3643 /// `temporal::archive::archive_branch` records why it has no smaller
3644 /// version.
3645 ///
3646 /// # It refuses more than it accepts, on purpose
3647 ///
3648 /// - The trunk, and a name that is not registered
3649 /// ([`DbError::UnknownBranch`]).
3650 /// - A branch with **descendants**: they read through it, so archiving it
3651 /// would delete rows they still believe.
3652 /// - A branch whose **concepts another lineage's hot link names**. The road
3653 /// map assumed an abandoned branch's rows were "a contiguous archivable
3654 /// set by construction"; a concept is keyed by identity across the whole
3655 /// ledger (D-214), so they are not, and this refusal is what makes them
3656 /// contiguous in the cases it accepts.
3657 ///
3658 /// All but the first return [`DbError::BranchNotArchivable`] with a reason.
3659 ///
3660 /// The lineage record lands in `cold.branches` with an `archived_at`, so a
3661 /// cold row's `branch_id` still resolves to something — in the cold file,
3662 /// which is now the only place it does.
3663 pub async fn archive_branch(&self, branch: crate::branch::BranchId) -> Result<ArchiveReport> {
3664 let branch = branch.as_str().to_string();
3665 let archive_path = self.archive_path.clone();
3666 self.low(|responder| LowPriCommand::ArchiveBranch {
3667 branch,
3668 archive_path,
3669 responder,
3670 })
3671 .await
3672 }
3673
3674 /// Move the named concepts back from the cold database into the hot tables
3675 /// (§2.3, C3).
3676 ///
3677 /// Rehydration is a **physical move back, not a write**: it mints no
3678 /// transaction-time facts and is invisible to both clocks. An id that is not
3679 /// in the cold file is skipped rather than being an error — the caller
3680 /// generally has a list from a cold-side query, and a partially-stale list is
3681 /// the normal case rather than a mistake. The report says how many actually
3682 /// moved.
3683 ///
3684 /// See [`RehydrateReport::rowids_reassigned`] for the one way a rehydrated
3685 /// row can differ from the row that was archived.
3686 pub async fn rehydrate(&self, ids: &[&str]) -> Result<RehydrateReport> {
3687 let ids: Vec<String> = ids.iter().map(|s| (*s).to_string()).collect();
3688 let archive_path = self.archive_path.clone();
3689 self.low(|responder| LowPriCommand::Rehydrate {
3690 ids,
3691 archive_path,
3692 responder,
3693 })
3694 .await
3695 }
3696
3697 /// Archive up to `cutoff` as a sequence of sessions, each covering at most
3698 /// `window` of **transaction** time (T1.1, D-080).
3699 ///
3700 /// `archive(cutoff)` is one transaction whose size is set by how long it has
3701 /// been since the last one, which makes it the least bounded of the three
3702 /// operations exempt from [`CHUNK_BUDGET`] — its hold is a function of
3703 /// operational history rather than of anything a caller chose. This runs the
3704 /// same work as *N* complete sessions, each with its own marker, horizon row
3705 /// and rebuild, and returns one [`ArchiveReport`] per session in order.
3706 ///
3707 /// # D-012 is satisfied per session, and that is what it requires
3708 ///
3709 /// The atomicity D-012 demands is that copy-then-delete never be split — a
3710 /// crash between the phases duplicates or loses rows. *N* small sessions
3711 /// satisfy that exactly as one large one does. The obligation windowing adds
3712 /// is that a partial run leave a coherent intermediate state, which it does:
3713 /// each session commits a valid horizon, so a failure at window *k* leaves a
3714 /// database archived up to boundary *k−1* and nothing in between. **The
3715 /// sequence is not atomic and does not claim to be** — on error, the reports
3716 /// for the sessions that did commit are lost with it, but their effect is
3717 /// not, and re-running with the same `cutoff` completes the job.
3718 ///
3719 /// # Each session is its own actor turn, and that is the entire point
3720 ///
3721 /// This loop lives here, on the handle, rather than inside the actor's
3722 /// `Archive` arm. Putting it there would have produced *N* small
3723 /// transactions inside **one** hold, which shrinks the transaction and
3724 /// changes the latency not at all: the actor is single-threaded, so nothing
3725 /// else writes until its turn returns regardless of how many `COMMIT`s the
3726 /// turn contains. Sending *N* commands returns the actor to its `select!`
3727 /// between sessions, which is where an interactive assertion gets to jump
3728 /// the queue — and it is high-priority, so it does.
3729 ///
3730 /// The same reasoning is why [`Self::bulk_import`] chunks here and not
3731 /// there, and it is the trap T1.2 names for `CREATE TABLE … AS SELECT`.
3732 ///
3733 /// # Choosing a window
3734 ///
3735 /// The bound is on *transaction* time, so the session count is set by how
3736 /// far back the hot file goes, not by how much it holds. A window is
3737 /// rejected rather than clamped if it would need more than
3738 /// [`MAX_ARCHIVE_SESSIONS`] sessions — see [`DbError::ArchiveWindow`].
3739 ///
3740 /// Windows containing nothing archivable are cheap but not free: each still
3741 /// opens a transaction and writes a horizon row. What they no longer do is
3742 /// re-project `links_current`, which `archive_session` now skips when its
3743 /// `DELETE` removed no rows — without that, windowing costs *more* in total
3744 /// than not windowing, because the repair term scales with the surviving
3745 /// table and not with the batch (D-077).
3746 pub async fn archive_windowed(
3747 &self,
3748 cutoff: &str,
3749 window: std::time::Duration,
3750 ) -> Result<Vec<ArchiveReport>> {
3751 let cutoff = timestamp::normalize(cutoff)?;
3752 let boundaries = self.archive_boundaries(&cutoff, window).await?;
3753
3754 let mut reports = Vec::with_capacity(boundaries.len());
3755 for boundary in boundaries {
3756 let archive_path = self.archive_path.clone();
3757 reports.push(
3758 self.low(|responder| LowPriCommand::Archive {
3759 cutoff: boundary,
3760 archive_path,
3761 responder,
3762 })
3763 .await?,
3764 );
3765 }
3766 Ok(reports)
3767 }
3768
3769 /// The cutoffs [`Self::archive_windowed`] will run, ascending, ending at
3770 /// `cutoff` exactly.
3771 ///
3772 /// Read on `read_conn`, not on the actor: this is two `MIN`s and the actor
3773 /// has no reason to hold its lock for them.
3774 ///
3775 /// The lower end comes from the data rather than from the clock. Stepping
3776 /// from some fixed epoch would make the session count a function of the
3777 /// calendar — a database opened yesterday would still be asked to archive
3778 /// 1970 — whereas the oldest `recorded_at` actually present is the earliest
3779 /// boundary that can contain anything.
3780 async fn archive_boundaries(
3781 &self,
3782 cutoff: &str,
3783 window: std::time::Duration,
3784 ) -> Result<Vec<String>> {
3785 // A single session at `cutoff` is exactly `archive(cutoff)`, and it is
3786 // the right answer for an empty hot file: it still writes the horizon
3787 // row, so windowed and unwindowed runs leave the same observable state.
3788 let Some(oldest) = self.oldest_hot_stamp(cutoff).await? else {
3789 return Ok(vec![cutoff.to_string()]);
3790 };
3791
3792 let start = timestamp::parse(&oldest)?;
3793 let end = timestamp::parse(cutoff)?;
3794 let Ok(span) = end.duration_since(start) else {
3795 // Everything in the hot file is at or after the cutoff, so there is
3796 // nothing in range to divide.
3797 return Ok(vec![cutoff.to_string()]);
3798 };
3799
3800 if window.is_zero() {
3801 return Err(DbError::ArchiveWindow {
3802 window,
3803 reason: "a zero-length window never advances past the first boundary".into(),
3804 });
3805 }
3806
3807 // `div_ceil` on nanos: a span of 90 minutes in 60-minute windows is two
3808 // sessions, not one. `as_nanos` is u128, so neither the division nor the
3809 // span can overflow for any timestamp this crate can store.
3810 let sessions = span.as_nanos().div_ceil(window.as_nanos());
3811 if sessions > MAX_ARCHIVE_SESSIONS as u128 {
3812 return Err(DbError::ArchiveWindow {
3813 window,
3814 reason: format!(
3815 "a span of {span:?} would need {sessions} sessions (limit \
3816 {MAX_ARCHIVE_SESSIONS}); widen the window"
3817 ),
3818 });
3819 }
3820
3821 let mut boundaries = Vec::with_capacity(sessions as usize);
3822 for k in 1..sessions {
3823 boundaries.push(timestamp::format(start + window * k as u32));
3824 }
3825 // The last boundary is `cutoff` itself and not `start + n*window`, which
3826 // would overshoot and archive rows the caller excluded.
3827 boundaries.push(cutoff.to_string());
3828 Ok(boundaries)
3829 }
3830
3831 /// Oldest `recorded_at` below `cutoff` in either hot table, or `None`.
3832 async fn oldest_hot_stamp(&self, cutoff: &str) -> Result<Option<String>> {
3833 let mut oldest: Option<String> = None;
3834 for table in ["links", "transaction_log"] {
3835 let found: Option<String> = self
3836 .read_conn
3837 .query(
3838 &format!("SELECT MIN(recorded_at) FROM {table} WHERE recorded_at < ?1"),
3839 libsql::params![cutoff],
3840 )
3841 .await?
3842 .next()
3843 .await?
3844 .and_then(|row| row.get(0).ok());
3845 if let Some(found) = found {
3846 if oldest.as_ref().is_none_or(|o| found < *o) {
3847 oldest = Some(found);
3848 }
3849 }
3850 }
3851 Ok(oldest)
3852 }
3853
3854 /// Send a high-priority command and wait for its answer.
3855 ///
3856 /// The two error mappings here are the whole reason this helper exists.
3857 /// `send` failing means the actor is gone — `WriterUnavailable`. The
3858 /// responder being dropped without an answer means the actor took the
3859 /// command and never replied — `WriterDroppedResponder`, which is a bug in
3860 /// the actor rather than a condition the caller can retry. Both variants
3861 /// existed in `error.rs` from 0.4.5 and neither was ever constructed, so a
3862 /// dead actor and a hung one were both just a caller waiting forever.
3863 async fn high<T>(
3864 &self,
3865 make: impl FnOnce(oneshot::Sender<Result<T>>) -> HighPriCommand,
3866 ) -> Result<T> {
3867 let (tx, rx) = oneshot::channel();
3868 self.highpri_tx
3869 .send(make(tx))
3870 .await
3871 .map_err(|_| DbError::WriterUnavailable)?;
3872 rx.await.map_err(|_| DbError::WriterDroppedResponder)?
3873 }
3874
3875 /// Send each chunk in turn and sum the counts — the shape all four bulk
3876 /// paths share (T3.4, D-086).
3877 ///
3878 /// # This is sequential on purpose, and the purpose is a measurement
3879 ///
3880 /// T3.4 proposed pipelining: send *k* chunks ahead so the actor never finds
3881 /// an empty queue. The reasoning is that awaiting each chunk before building
3882 /// the next leaves the actor idle for a channel round trip every time, which
3883 /// on a 1M-edge import is ~11,000 idle gaps.
3884 ///
3885 /// Both halves of that are true and the conclusion does not follow. The gaps
3886 /// are real; they are also **four orders of magnitude smaller than the work
3887 /// they interrupt**. A tokio mpsc hop is sub-microsecond and a chunk takes
3888 /// 13–21 ms. Implemented and swept at depths 1, 2, 4, 8 and 16 over 20K and
3889 /// 100K edges: every cell landed within 1% of sequential, in both directions
3890 /// — see `examples/pipeline_diag.rs`, which is kept precisely so this is not
3891 /// re-proposed from the same reasoning.
3892 ///
3893 /// So the pipelining was removed and the deduplication kept. It was not free
3894 /// to hold: with chunks in flight, a failure at chunk `i` no longer leaves a
3895 /// **prefix** committed, because `i+1 ..= i+k-1` were already sent and commit
3896 /// anyway. D-011 promises "earlier chunks committed", and paying for that
3897 /// with a weaker recovery story in exchange for nothing measurable is the
3898 /// wrong trade.
3899 ///
3900 /// Sending stops at the first error, so what commits is exactly the prefix
3901 /// before the failure.
3902 /// # The size is now measured, not assumed (0.12.0, W3)
3903 ///
3904 /// Until 0.11.0 the caller pre-split into `chunks(chunk_rows::WHATEVER)` and
3905 /// this loop sent what it was given. That made the constant *the* size, and
3906 /// D-143 is the record of a constant fitted at one population being wrong at
3907 /// another: all four D-088 shapes agreed the largest in-budget edge chunk was
3908 /// **20** against a shipped 90, and 20 would itself have been wrong at 80,000
3909 /// edges, because per-row cost on that path grows with `links_current`.
3910 ///
3911 /// No row count can bound a duration on such a path, so the loop stopped
3912 /// trying to pick one ahead of time. `ceiling` — still the path's
3913 /// [`chunk_rows`] constant, with its derivation intact — is now the largest
3914 /// size this will ever ask for, and each chunk's measured hold chooses the
3915 /// next through `next_chunk_size`.
3916 ///
3917 /// **Feedback, not preemption.** The chunk in flight always commits in full;
3918 /// the SQLite write lock is not preemptible, so nothing here can shorten a
3919 /// transaction already running. A batch of one chunk gets no protection at
3920 /// all, and convergence costs one or two chunks — which is the price of the
3921 /// bound being a duration rather than a promise.
3922 ///
3923 /// The last chunk's outcome is discarded, there being no next chunk to size.
3924 /// The chunk loop behind all four bulk paths.
3925 ///
3926 /// **Every exit carries `written`** (0.13.8, W7.6, D-181). It used to
3927 /// carry it only out of the success arm: the three error paths were `?` on
3928 /// a [`DbError`], which discards the local, so a caller whose 20,000-row
3929 /// import failed in the last chunk learned that it failed and not that
3930 /// 19,000 rows were already in the database. The count was never expensive
3931 /// to keep — it is right there, and the loop needs it anyway to size the
3932 /// next chunk.
3933 async fn low_chunked<T>(
3934 &self,
3935 items: Vec<T>,
3936 ceiling: usize,
3937 control: BulkControl,
3938 make: impl Fn(Vec<T>, oneshot::Sender<Result<ChunkOutcome>>) -> LowPriCommand,
3939 ) -> BulkResult<usize> {
3940 let total = items.len();
3941 let mut items = items.into_iter();
3942 let mut size = ceiling.max(1);
3943 let mut written = 0usize;
3944 loop {
3945 let chunk: Vec<T> = items.by_ref().take(size).collect();
3946 if chunk.is_empty() {
3947 // Emptiness is checked before cancellation on purpose: a token
3948 // raised after the last chunk committed is asking to stop work
3949 // that is already done, and reporting that as a failure would
3950 // make a race between the caller's two threads decide whether a
3951 // complete import counts as one.
3952 return Ok(written);
3953 }
3954 // Between chunks, never inside one. Nothing is rolled back and no
3955 // transaction is interrupted -- the loop simply stops sending, and
3956 // the prefix that committed is the same kind of prefix a failure
3957 // would have left.
3958 if control.is_cancelled() {
3959 return Err(BulkInterrupted {
3960 written,
3961 cause: DbError::BulkCancelled,
3962 });
3963 }
3964 let stop = |cause: DbError| BulkInterrupted { written, cause };
3965 let (tx, rx) = oneshot::channel();
3966 self.lowpri_tx
3967 .send(make(chunk, tx))
3968 .await
3969 .map_err(|_| stop(DbError::WriterUnavailable))?;
3970 let outcome = match rx.await {
3971 Err(_) => return Err(stop(DbError::WriterDroppedResponder)),
3972 Ok(Err(e)) => return Err(stop(e)),
3973 Ok(Ok(outcome)) => outcome,
3974 };
3975 written += outcome.rows;
3976 control.report(BulkProgress {
3977 written,
3978 total,
3979 rows: outcome.rows,
3980 held: outcome.held,
3981 });
3982 size = next_chunk_size(size, outcome.held, CHUNK_BUDGET, CHUNK_FLOOR, ceiling);
3983 }
3984 }
3985
3986 async fn low<T>(
3987 &self,
3988 make: impl FnOnce(oneshot::Sender<Result<T>>) -> LowPriCommand,
3989 ) -> Result<T> {
3990 let (tx, rx) = oneshot::channel();
3991 self.lowpri_tx
3992 .send(make(tx))
3993 .await
3994 .map_err(|_| DbError::WriterUnavailable)?;
3995 rx.await.map_err(|_| DbError::WriterDroppedResponder)?
3996 }
3997
3998 /// Clean shutdown: stop the Write Actor, then write the final snapshot (§5.1.7).
3999 ///
4000 /// Order matters. The snapshot is taken *after* the actor has stopped and
4001 /// been joined, so no write can land between the fold and the file — the
4002 /// anchor it records is the last thing that happened, not the last thing
4003 /// that happened to be visible.
4004 ///
4005 /// A failed snapshot is reported rather than swallowed. It is not a
4006 /// durability loss — the ledger is in the WAL and the log replays without
4007 /// it — but it means the next open starts from an older anchor, and a caller
4008 /// that never hears about it cannot know why startup got slower.
4009 ///
4010 /// **The cadence stops first (§5.5, D-053).** Both it and `write_final` end
4011 /// by running retention over the snapshot directory, and retention deletes
4012 /// files. Letting them overlap would mean one pass enumerating the directory
4013 /// while the other removes from it — not a correctness problem for the
4014 /// ledger, which is why the ordering is stated rather than locked, but a
4015 /// source of spurious warnings and of a final anchor that could be deleted
4016 /// by a cleanup that started before it existed. Stopping the cadence, then
4017 /// the actor, then taking the snapshot leaves exactly one writer at each
4018 /// step.
4019 pub async fn close(mut self) -> Result<()> {
4020 if let Some(stop) = self.cadence_stop.take() {
4021 let _ = stop.send(true);
4022 }
4023 if let Some(handle) = self.cadence.take() {
4024 let _ = handle.await;
4025 }
4026
4027 // Top up the planner's statistics while the actor is still alive to do
4028 // it (0.12.4, D-149). `PRAGMA optimize` re-analyses only what SQLite
4029 // believes has gone stale, so on a database that did nothing this costs
4030 // nothing, and on one that was just bulk-loaded it is the difference
4031 // between the next process planning on measurements and planning on
4032 // built-in guesses.
4033 //
4034 // **Deliberately not fatal.** A failure here costs plan quality on the
4035 // next open and nothing else — no ledger state depends on it — and
4036 // `close()` is where a caller learns whether their *writes* survived.
4037 // Turning a stale-statistics problem into a failed close would bury that
4038 // answer under a much less important one.
4039 if let Err(e) = self.optimize().await {
4040 tracing::warn!(
4041 "PRAGMA optimize failed during close(): {e}. Statistics may be \
4042 stale for the next process; call analyze() to rebuild them. \
4043 Nothing else is affected."
4044 );
4045 }
4046
4047 let (tx, rx) = oneshot::channel();
4048 let _ = self
4049 .highpri_tx
4050 .send(HighPriCommand::Shutdown { responder: tx })
4051 .await;
4052 let _ = rx.await;
4053
4054 // **The writer's exit status is propagated, not discarded (Wave 4.2).**
4055 // It used to be `let _ = handle.await`, so an actor that had died closed
4056 // "successfully" and the caller's last chance to learn that the write
4057 // path was gone was spent silently.
4058 //
4059 // Through 0.13.3 this awaited a `JoinHandle<Result<()>>` and did
4060 // `Ok(res) => res?`, which looked like two failure paths and was one:
4061 // the actor's `Result` could not be `Err` (W7.3, D-177). What remains is
4062 // the branch that can fire — the actor panicked or was aborted — mapped
4063 // by `writer_exit`, which is tested against a real `JoinError`.
4064 //
4065 // Ordered before the final snapshot on purpose: a snapshot written after
4066 // a dead writer records a state the caller has no reason to trust, and
4067 // returning the error while also having written that file is worse than
4068 // not writing it.
4069 if let Some(handle) = self.writer.take() {
4070 writer_exit(handle.await)?;
4071 }
4072
4073 let ts = self.clock.now();
4074 let archive = crate::temporal::archive::archive_present(&self.archive_path)
4075 .then_some(self.archive_path.as_path());
4076 snapshot::write_final(&self.read_conn, &self.snapshots_dir, &ts, archive).await?;
4077
4078 // Marks the handle closed so `Drop` knows not to complain.
4079 self.closed = true;
4080 Ok(())
4081 }
4082}
4083
4084/// Notes a missed `close()` at `warn!`, and deliberately does **not** assert.
4085///
4086/// **§7.3 offered option B — document `close()` as mandatory and `debug_assert`
4087/// in `Drop` — and Wave 4.2 implemented it, measured the consequence, and
4088/// reduced it to a warning.** The assert fired on roughly thirty tests on its
4089/// first run. That is the signal it was built to produce, and the right reading
4090/// of it was not "thirty tests are wrong".
4091///
4092/// What dropping actually costs is one final snapshot. Nothing else: every
4093/// public write method awaits its responder, so by the time a caller *can* drop
4094/// the handle, every write it issued has already committed; and the cadence stops
4095/// on its own, because `cadence_stop` is a `watch::Sender` whose drop signals the
4096/// task. A snapshot is derivative state under Doctrine VI — disposable,
4097/// reconstructible, and never the only copy of anything. Losing one makes the
4098/// next `reconstruct` fold from an older anchor, which is **slower, not wrong**.
4099///
4100/// A `debug_assert` aborts a test run. Spending that on a performance loss, in a
4101/// project whose own notes say a suite that fails for reasons unrelated to the
4102/// code under test trains people to ignore red, is the wrong trade — and paying
4103/// it in thirty places would have made `close()` look mandatory by ceremony
4104/// rather than by consequence. `close()` remains the right thing to call, and
4105/// the two reasons to call it are now stated where they can be acted on: the
4106/// snapshot, and the writer's `Result`, which only `close()` can return.
4107///
4108/// Option A ("abort the actor and log") stays rejected, for the reason it was
4109/// rejected twice before: `Drop` cannot await, so it cannot drain, and cleanup
4110/// that cannot clean up is worse than none — it looks like cleanup.
4111impl Drop for Database {
4112 fn drop(&mut self) {
4113 if !self.closed {
4114 tracing::warn!(
4115 "Database dropped without close(): the final snapshot was not written, \
4116 so the next reconstruct folds from an older anchor, and the write \
4117 actor's exit status was not checked. Prefer close().await."
4118 );
4119 }
4120 }
4121}
4122
4123/// A failure before the first chunk was sent, which committed nothing.
4124///
4125/// Normalisation runs over the whole batch up front, so its errors are the one
4126/// class the chunk loop never sees — and they are still [`BulkInterrupted`],
4127/// because a caller matching on one error type should not have to match on two
4128/// to find out that nothing landed (0.13.8, W7.6).
4129fn before_any_chunk(cause: DbError) -> BulkInterrupted {
4130 BulkInterrupted { written: 0, cause }
4131}
4132
4133fn normalize_all(edges: Vec<EdgeAssertion>) -> Result<Vec<EdgeAssertion>> {
4134 edges.into_iter().map(EdgeAssertion::normalized).collect()
4135}
4136
4137/// One `FULL` checkpoint for the numbers, then a `TRUNCATE` for the file.
4138///
4139/// # Why `TRUNCATE` and not a mode parameter
4140///
4141/// The four SQLite modes are not four things a caller of *this* crate wants.
4142/// `PASSIVE` is what the automatic checkpointer already runs on its own, so an
4143/// explicit `PASSIVE` asks for something that was going to happen anyway;
4144/// `RESTART` and `FULL` differ from `TRUNCATE` only in whether the WAL file is
4145/// left at its high-water size. The reason W5.2 exists is
4146/// [`Tuning::wal_autocheckpoint`] — a bulk importer turns the automatic
4147/// checkpointer off and calls this once at the end — and what that caller wants
4148/// is the WAL *gone*, not smaller than it was. So the mode is fixed and decided
4149/// here rather than pushed to the caller as a choice they would have to read
4150/// SQLite's documentation to make. If a mode ever needs selecting, that is an
4151/// additive method, not a change to this one.
4152///
4153/// # Why it is two pragmas, which is not the obvious implementation
4154///
4155/// **A successful `TRUNCATE` reports `busy=0, log=0, checkpointed=0`** — the
4156/// counts describe the WAL *after* the operation, and after a truncation there
4157/// is no WAL to describe. Measured, not inferred: on a 387-frame WAL, `PASSIVE`
4158/// returns `0, 387, 387` and `TRUNCATE` on the same file returns `0, 0, 0`. So
4159/// the single-pragma implementation returns a [`CheckpointReport`] whose two
4160/// counts are structurally zero on success, which makes the whole struct a
4161/// less useful `bool`.
4162///
4163/// `FULL` copies every frame back and reports what it moved; the `TRUNCATE`
4164/// that follows finds nothing left to copy and resets the file. The second pass
4165/// is close to free for exactly that reason — it is a file operation, not a
4166/// second copy. `busy` is the **union**: a checkpoint that was blocked in
4167/// either phase did not fully happen, and a caller about to copy the database
4168/// file elsewhere needs the pessimistic answer.
4169///
4170/// # They return rows, so they go through `query()`
4171///
4172/// The same libsql constraint the pragmas in `configure` document: `execute()`
4173/// rejects any statement that yields rows, and these yield the row that is the
4174/// entire point.
4175async fn run_checkpoint(conn: &libsql::Connection) -> Result<CheckpointReport> {
4176 // The columns are `busy, log, checkpointed`. SQLite reports -1 for the two
4177 // counts when the checkpoint could not run; clamped to 0 rather than
4178 // surfaced as a signed count, because `busy` already carries "this did not
4179 // happen" and a negative frame count is not a quantity anyone can use.
4180 //
4181 // A database not in WAL mode returns no row at all. `configure` puts every
4182 // connection this crate opens into WAL, so that is unreachable here — but a
4183 // zeroed report is a better failure than a panic if it stops being.
4184 async fn one(conn: &libsql::Connection, sql: &str) -> Result<(bool, u64, u64)> {
4185 let mut rows = conn.query(sql, ()).await?;
4186 let Some(row) = rows.next().await? else {
4187 return Ok((false, 0, 0));
4188 };
4189 let field = |i: i32| -> u64 { row.get::<i64>(i).unwrap_or(0).max(0) as u64 };
4190 Ok((row.get::<i64>(0).unwrap_or(0) != 0, field(1), field(2)))
4191 }
4192
4193 let (full_busy, _, moved) = one(conn, "PRAGMA wal_checkpoint(FULL)").await?;
4194 let (trunc_busy, log_frames, _) = one(conn, "PRAGMA wal_checkpoint(TRUNCATE)").await?;
4195
4196 Ok(CheckpointReport {
4197 busy: full_busy || trunc_busy,
4198 log_frames,
4199 checkpointed_frames: moved,
4200 })
4201}
4202
4203/// Pragmas that mean something on **any** connection, including one opened
4204/// `SQLITE_OPEN_READ_ONLY` (0.12.16, W5.5, D-159).
4205///
4206/// Both of these are per-connection state that a reader is subject to just as a
4207/// writer is. `busy_timeout` is the one that made this a finding:
4208/// [`Database::diagnostic_conn`] ran with SQLite's default of **0** — return
4209/// `SQLITE_BUSY` immediately — while every other connection in the process
4210/// waited 5 s, so the one surface whose job is to answer questions when the
4211/// typed path is already suspect was also the one most likely to fail with
4212/// "database is locked" under exactly the contention that prompted the
4213/// question.
4214/// Empty the slot unless what is in it is fit to hand to the next caller
4215/// (0.15.15, W15.5, [D-257]).
4216///
4217/// Shared by [`Database::diagnostic_conn`], which runs it on the way in, and
4218/// [`Database::scrub_diagnostic_conn`], which the Python binding runs on the way
4219/// out. One implementation because the two must agree: a connection the entry
4220/// path would have discarded is one the exit path must not leave sitting there.
4221async fn scrub(slot: &mut Option<libsql::Connection>) {
4222 let Some(conn) = slot.as_ref() else {
4223 return;
4224 };
4225 // A leaked `BEGIN` is the one that leaves this surface: it pins a WAL read
4226 // snapshot, so later diagnostic reads answer from it and `checkpoint()`
4227 // cannot truncate past it. Rolled back rather than reported, because the
4228 // caller who would read the report is the one who did not do it.
4229 let recovered = conn.is_autocommit() || conn.execute("ROLLBACK", ()).await.is_ok();
4230 // A connection that will not roll back is not one to hand on, and neither
4231 // is one whose own state cannot be read: either way the answer is a fresh
4232 // connection, which costs 56.5 us on the call that dirtied it and nothing
4233 // on any other.
4234 if !recovered || diagnostic_is_dirty(conn).await.unwrap_or(true) {
4235 *slot = None;
4236 }
4237}
4238
4239/// Is this cached diagnostic connection carrying state from an earlier caller
4240/// (0.15.15, W15.5, [D-257])?
4241///
4242/// Two questions — are there temp objects, is anything attached beyond `main`
4243/// and `temp` — asked as pragmas rather than as a query over
4244/// `temp.sqlite_master` and `pragma_database_list`. Same answers, and
4245/// `examples/diagnostic_hygiene_probe.rs` measures the pragma form at
4246/// **2.4 µs** against the query form's **7.8 µs**, on a call whose whole warm
4247/// cost is the `stat` in front of it. Detection is measured rather than
4248/// assumed: `temp.schema_version` goes 0 → 1 on `CREATE TEMP TABLE`, and
4249/// `database_list` 2 → 3 on `ATTACH`.
4250///
4251/// **What it cannot see is a `PRAGMA`**, and neither can any other cheap check:
4252/// SQLite does not enumerate connection-scoped pragma state. The crate restates
4253/// the two it sets instead — see the call to `configure_common` in
4254/// [`Database::diagnostic_conn`].
4255///
4256/// An `Err` here is not propagated by the caller: a connection whose own state
4257/// cannot be read is replaced rather than reported on.
4258///
4259/// [D-257]: ../../docs/architecture/s13-decision-register.md#d-257
4260async fn diagnostic_is_dirty(conn: &libsql::Connection) -> Result<bool> {
4261 let mut rows = conn.query("PRAGMA temp.schema_version", ()).await?;
4262 let temp_schema: i64 = match rows.next().await? {
4263 Some(row) => row.get(0)?,
4264 // No row at all is not a clean connection, it is an answer this
4265 // function did not understand.
4266 None => return Ok(true),
4267 };
4268 if temp_schema != 0 {
4269 return Ok(true);
4270 }
4271 let mut rows = conn.query("PRAGMA database_list", ()).await?;
4272 let mut databases = 0_usize;
4273 while rows.next().await?.is_some() {
4274 databases += 1;
4275 }
4276 // `main` and `temp`, always both, on a connection nobody has attached to.
4277 Ok(databases > 2)
4278}
4279
4280async fn configure_common(conn: &libsql::Connection, cache_size: Option<i32>) -> Result<()> {
4281 // NOTE: `busy_timeout` returns its resulting value as a row, and libsql's
4282 // `execute()` rejects any statement that yields rows ("Execute returned
4283 // rows"). It must be issued through `query()`.
4284 let _ = conn.query("PRAGMA busy_timeout = 5000", ()).await?;
4285 // Per-connection, and split writer from reader since 0.12.15 (W5.4,
4286 // D-158). `None` runs no pragma at all rather than restating SQLite's
4287 // default, so the default remains SQLite's to change.
4288 if let Some(pages) = cache_size {
4289 conn.execute(&format!("PRAGMA cache_size = {pages}"), ())
4290 .await?;
4291 }
4292 Ok(())
4293}
4294
4295/// Pragmas that only mean anything where writes can happen (0.12.16, W5.5).
4296///
4297/// Not run on [`Database::diagnostic_conn`], and the reason is not tidiness:
4298/// `journal_mode = WAL` is a change to the *database file*, which a connection
4299/// opened `SQLITE_OPEN_READ_ONLY` cannot make. The rest —
4300/// `synchronous`, `foreign_keys`, `recursive_triggers`, and the `ANALYZE`
4301/// bound — govern how writes behave, and a connection that cannot write is not
4302/// governed by them.
4303///
4304/// The write connection and the two internal readers all still get these. The
4305/// internal readers are opened from the same read-write `libsql::Database`, so
4306/// the pragmas apply; leaving them out would be a behaviour change made for
4307/// symmetry, which is not a reason.
4308async fn configure_writable(conn: &libsql::Connection) -> Result<()> {
4309 // Returns its resulting value as a row — see the note in `configure_common`.
4310 let _ = conn.query("PRAGMA journal_mode = WAL", ()).await?;
4311 conn.execute("PRAGMA synchronous = NORMAL", ()).await?;
4312 conn.execute("PRAGMA foreign_keys = ON", ()).await?;
4313 conn.execute("PRAGMA recursive_triggers = OFF", ()).await?;
4314 // Bounds every `ANALYZE` this connection will ever run, explicit or
4315 // triggered by `PRAGMA optimize` (D-149). Set here rather than around the
4316 // call sites so the scheduled path is bounded too — that is the half that
4317 // runs with nobody watching. Returns the previous limit as a row, so it goes
4318 // through `query()` for the reason the note above gives.
4319 let _ = conn.query(crate::schema::ddl::ANALYSIS_LIMIT, ()).await?;
4320 Ok(())
4321}
4322
4323/// Full pragma configuration, for a connection that can write.
4324async fn configure(
4325 conn: libsql::Connection,
4326 cache_size: Option<i32>,
4327) -> Result<libsql::Connection> {
4328 configure_writable(&conn).await?;
4329 configure_common(&conn, cache_size).await?;
4330 Ok(conn)
4331}
4332
4333/// Helper to derive the snapshot directory by convention: foo.db -> foo_snapshots/
4334fn derive_snapshots_dir(path: &Path) -> PathBuf {
4335 let mut dir = path.to_path_buf();
4336 let stem = path
4337 .file_stem()
4338 .and_then(|s| s.to_str())
4339 .unwrap_or("macrame");
4340 dir.set_file_name(format!("{stem}_snapshots"));
4341 dir
4342}
4343
4344/// Helper to derive archive database path by convention: foo.db -> foo_archive.db
4345fn derive_archive_path(path: &Path) -> PathBuf {
4346 let mut archive = path.to_path_buf();
4347 if let Some(stem) = path.file_stem().and_then(|s| s.to_str()) {
4348 let ext = path.extension().and_then(|e| e.to_str()).unwrap_or("db");
4349 archive.set_file_name(format!("{stem}_archive.{ext}"));
4350 } else {
4351 archive.set_extension("archive.db");
4352 }
4353 archive
4354}
4355
4356/// Dedicated Write Actor event loop prioritizing high-priority UI requests over low-priority background work.
4357///
4358/// # The turn is the unit, not the statement (T1.4)
4359///
4360/// One iteration of this loop is one *hold*: the actor is single-threaded and
4361/// the SQLite write lock is not preemptible, so from the moment a command starts
4362/// executing until it returns, nothing else writes. That is the quantity
4363/// [`CHUNK_BUDGET`] bounds, and so it is the quantity
4364/// [`crate::metrics::ActorMetrics`] measures — deliberately around the whole
4365/// `execute` call rather than inside it. Timing the SQL alone would have
4366/// reported a bound that held while callers waited.
4367///
4368/// Queue depth is sampled *before* the `select!`, so it is the backlog the turn
4369/// found on arrival rather than the one it left behind.
4370///
4371/// # `biased` has no floor, and since 0.12.10 that is measured (W4.4, D-153)
4372///
4373/// `biased` makes the arms poll in declaration order, so high-priority work is
4374/// taken whenever any is ready. Nothing bounds how long that can continue:
4375/// sustained interactive traffic can hold the low tier off indefinitely, and
4376/// through 0.12.9 nothing in the crate could say whether it ever did.
4377/// `record_priority_choice` counts the turns where the choice went against
4378/// queued low-priority work, and the longest unbroken run of them, which is the
4379/// half that distinguishes "prioritised" from "starved".
4380///
4381/// **No forced yield is added here.** Whether one is needed is the question the
4382/// counter answers, and adding a policy now would be fixing a bound nobody has
4383/// observed being hit — the same mistake D-124 was retracted for.
4384///
4385/// # It returns nothing, and used to return a `Result` it could not fail
4386/// (0.13.4, W7.3, §3.5, [D-177])
4387///
4388/// The two exits are `LoopCtl::Break` from [`HighPriCommand::Shutdown`] and the
4389/// `else` arm when both channels are closed. Neither can fail, and neither
4390/// could before: every command's error goes back on that command's own
4391/// responder, where the caller who issued it can act on it. There was no third
4392/// thing for an actor-level `Err` to carry, and none was ever constructed.
4393///
4394/// A `Result` that is structurally always `Ok` is not free. It reads as a
4395/// failure path under review, so `close()`'s `res?` looked like it was doing
4396/// something, and the branch that actually fires — a **panicked** actor,
4397/// reported as a `JoinError` — sat beside it untested. That is the swap this
4398/// change makes: the unfireable branch is gone and the real one is pinned, in
4399/// [`writer_exit`].
4400async fn run_writer_actor(
4401 conn: libsql::Connection,
4402 clock: Arc<dyn Clock>,
4403 mut highpri_rx: mpsc::Receiver<HighPriCommand>,
4404 mut lowpri_rx: mpsc::Receiver<LowPriCommand>,
4405 shared: Arc<ActorShared>,
4406) {
4407 // Owned by the loop and lent to each command, which is the whole of A-3:
4408 // the actor had a connection and no memory, so every turn re-established
4409 // what the turn before it had just established (0.15.6, W14.3, D-248).
4410 let mut state = ActorState::new();
4411 loop {
4412 // Read once and reused by both the depth sample and the starvation
4413 // counter, so the two cannot disagree about what was queued when this
4414 // turn went looking (W4.4, D-153).
4415 let low_queued = lowpri_rx.len();
4416 shared.metrics.record_turn(highpri_rx.len(), low_queued);
4417
4418 let ctl = tokio::select! {
4419 biased;
4420 Some(cmd) = highpri_rx.recv() => {
4421 shared.metrics.record_priority_choice(true, low_queued);
4422 let turn = Turn::start(cmd.kind(), &shared);
4423 cmd.execute(&conn, &*clock, &turn, &mut state).await
4424 }
4425 Some(cmd) = lowpri_rx.recv() => {
4426 shared.metrics.record_priority_choice(false, low_queued);
4427 let turn = Turn::start(cmd.kind(), &shared);
4428 cmd.execute(&conn, &*clock, &turn, &mut state).await
4429 }
4430 else => LoopCtl::Break,
4431 };
4432 if matches!(ctl, LoopCtl::Break) {
4433 break;
4434 }
4435 }
4436}
4437
4438/// Turn the write actor's join status into the error `close()` reports.
4439///
4440/// One line of mapping, given a name so it can be tested against a real
4441/// [`tokio::task::JoinError`]. Before 0.13.4 this was inline beside a `res?` on
4442/// an actor `Result` that could only ever be `Ok`, and the arrangement had the
4443/// coverage exactly backwards: the branch that cannot fire was plumbed through
4444/// two signatures, and the branch that does fire — the actor panicked, and the
4445/// caller's writes are going nowhere — had no test at all (W7.3, D-177).
4446///
4447/// Cancellation is folded in with panics deliberately. `JoinError` distinguishes
4448/// them, and nothing in the crate ever aborts this task, so a cancelled writer
4449/// means something outside the crate reached in and stopped it. That is not a
4450/// gentler condition than a panic and must not read as one.
4451fn writer_exit(joined: std::result::Result<(), tokio::task::JoinError>) -> Result<()> {
4452 joined.map_err(|e| DbError::WriterStopped(format!("the write actor did not exit cleanly: {e}")))
4453}
4454
4455/// One command's hold: the timer, its label, and the counters it reports to.
4456///
4457/// # The hold is recorded *before* the caller is answered, and it has to be
4458///
4459/// The obvious placement — time the whole `execute` call from the loop — is
4460/// wrong in a way that only shows up under test. Every arm of `execute` ends by
4461/// sending on a `oneshot`, which wakes the waiting caller; the actor then
4462/// returns to the loop and records. Those are two tasks, so a caller that awaits
4463/// its own write and immediately reads [`Database::metrics`] can be scheduled
4464/// first and see a turn count that does not include the write it just did.
4465///
4466/// Not a correctness bug in the ledger, and it would never have been noticed in
4467/// production — a dashboard sampling every few seconds cannot see the window.
4468/// It makes every test and diagnostic of the counters flaky, which is worse: the
4469/// instrumentation would have been *believed* while being wrong exactly when
4470/// someone tried to check it. `examples/bulk_atomic_diag.rs` was the thing that
4471/// caught it, reporting a 20,000-row batch as a 0 ms hold.
4472///
4473/// So `answer` records and then sends, in that order, and the ordering is the
4474/// method's whole reason to exist. What it costs is that the `oneshot::send`
4475/// itself falls outside the measurement, which is a few nanoseconds against a
4476/// turn measured in microseconds at best.
4477struct Turn<'a> {
4478 kind: crate::metrics::CommandKind,
4479 timer: crate::metrics::HoldTimer,
4480 shared: &'a ActorShared,
4481}
4482
4483/// State the actor owns and a `Turn` needs to reach.
4484///
4485/// `archive_epoch` is here rather than in [`crate::metrics::ActorMetrics`]
4486/// because it is **not** a metric: T1.2's shadow rebuild reads it to decide
4487/// whether its work is still valid, so it has to be present in every build, not
4488/// only under the `metrics` feature. Counting archives happens to be what both
4489/// want; only one of them is allowed to be compiled out.
4490///
4491/// `turns` is here for the same reason and serves the snapshot cadence — see
4492/// its own note.
4493#[derive(Default)]
4494struct ActorShared {
4495 metrics: crate::metrics::ActorMetrics,
4496 archive_epoch: std::sync::atomic::AtomicU64,
4497 /// Commands this actor has answered `Ok` to (0.15.19, review C-19).
4498 ///
4499 /// # What it is for, and why it is not a `seq_id`
4500 ///
4501 /// `snapshot::run_cadence` used to run `SELECT MAX(seq_id), MAX(recorded_at)`
4502 /// on **every tick**, five seconds apart by default, whether or not
4503 /// anything had been written. Two aggregates on an idle database, for a
4504 /// fact the actor already had: *nothing has happened*.
4505 ///
4506 /// The review asked for a `watch<u64>` of the last committed `seq_id`. This
4507 /// is the same idea one step cheaper, and the difference matters. The actor
4508 /// does not currently know the `seq_id` its writes produced — the log rows
4509 /// are written by triggers — so publishing one would mean adding a query to
4510 /// **every write** in order to remove a query from an idle timer, which is
4511 /// the wrong direction. A turn count needs no query at all: one relaxed
4512 /// `fetch_add` on a path already doing a database round trip.
4513 ///
4514 /// # Why this cannot change when a snapshot is written
4515 ///
4516 /// The cadence skips its tick when the count has not moved since the last
4517 /// one. That is sound because the implication runs the right way: if
4518 /// `MAX(seq_id)` grew, some command committed, so some turn answered `Ok`,
4519 /// so the count moved. The converse is not claimed and does not need to be
4520 /// — a turn that answered `Ok` without writing a log row makes the cadence
4521 /// do exactly the query it used to do every time. It over-counts, never
4522 /// under-counts, and the tick it protects had nothing to do anyway.
4523 ///
4524 /// `Relaxed` because nothing is ordered against it. The cadence reads a
4525 /// number to compare with a number it read before; a value one tick stale
4526 /// costs one deferred tick and no correctness, and the same is true of the
4527 /// `archive_epoch` beside it.
4528 turns: std::sync::atomic::AtomicU64,
4529}
4530
4531impl crate::temporal::snapshot::CommittedTurns for ActorShared {
4532 fn committed_turns(&self) -> u64 {
4533 self.turns.load(std::sync::atomic::Ordering::Relaxed)
4534 }
4535}
4536
4537impl<'a> Turn<'a> {
4538 fn start(kind: crate::metrics::CommandKind, shared: &'a ActorShared) -> Self {
4539 Self {
4540 kind,
4541 timer: crate::metrics::HoldTimer::start(),
4542 shared,
4543 }
4544 }
4545
4546 fn epoch(&self) -> u64 {
4547 self.shared
4548 .archive_epoch
4549 .load(std::sync::atomic::Ordering::Relaxed)
4550 }
4551
4552 /// Record that an archive session committed.
4553 ///
4554 /// Bumped on **success only**: a failed archive rolls back, so it deletes
4555 /// nothing and invalidates no shadow build.
4556 fn archive_committed(&self) {
4557 self.shared
4558 .archive_epoch
4559 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
4560 }
4561
4562 /// Close the hold and hand the result back. Never the other way round.
4563 ///
4564 /// The `let _ =` on the send is deliberate and predates this: a caller that
4565 /// dropped its receiver — `tokio::time::timeout` around a write, which
4566 /// [`Database`]'s write surface explicitly documents — is not an actor
4567 /// error, and the command committed regardless.
4568 fn answer<T>(&self, responder: oneshot::Sender<Result<T>>, res: Result<T>) {
4569 self.shared
4570 .metrics
4571 .record_hold(self.kind, self.timer.elapsed());
4572 if res.is_ok() {
4573 self.turn_committed();
4574 }
4575 let _ = responder.send(res);
4576 }
4577
4578 /// [`answer`](Self::answer) for a chunk: the same reading, handed back to the
4579 /// caller as well as recorded (0.12.0, W1).
4580 ///
4581 /// One `elapsed()` serves both, so the duration the chunk loop sizes against
4582 /// is *the same number* the histogram shows — a controller and a dashboard
4583 /// disagreeing about what a chunk cost would be a bad way to spend a
4584 /// debugging session.
4585 ///
4586 /// The record-then-send ordering documented on [`Turn`] is preserved, and
4587 /// matters here for the same reason: the send wakes the caller, which may be
4588 /// scheduled before this method returns.
4589 fn answer_chunk(&self, responder: oneshot::Sender<Result<ChunkOutcome>>, res: Result<usize>) {
4590 let held = self.timer.elapsed();
4591 self.shared.metrics.record_hold(self.kind, held);
4592 if res.is_ok() {
4593 self.turn_committed();
4594 }
4595 let _ = responder.send(res.map(|rows| ChunkOutcome { rows, held }));
4596 }
4597
4598 /// Record that a turn answered `Ok`, for [`ActorShared::turns`].
4599 ///
4600 /// Called from both answer paths rather than from the loop, because those
4601 /// are the two places that know the result. A command that returns an error
4602 /// does not bump it: a failed write rolls back and the log is where it was.
4603 fn turn_committed(&self) {
4604 self.shared
4605 .turns
4606 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
4607 }
4608}
4609
4610const INSERT_LINK: &str = "INSERT INTO links \
4611 (source_id, target_id, edge_type, valid_from, valid_to, weight, properties, \
4612 recorded_at, branch_id) \
4613 VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)";
4614
4615/// The parameter row for [`INSERT_LINK`], in one place since 0.14.8.
4616///
4617/// The single-edge path and the chunk path spelled these out separately, which
4618/// was survivable at eight and is not at nine: `branch_id` is the one parameter
4619/// whose omission is *silent* — the column defaults to `'main'`, so a path that
4620/// forgot it would write to the trunk and pass every test that did not fork.
4621/// [`concept_params`] has existed for this reason since D-056.
4622fn edge_params<'a>(edge: &'a EdgeAssertion, stamp: &'a str) -> [libsql::Value; 9] {
4623 [
4624 edge.source.as_str().into(),
4625 edge.target.as_str().into(),
4626 edge.edge_type.as_str().into(),
4627 edge.valid_from.as_str().into(),
4628 edge.valid_to.as_str().into(),
4629 edge.weight.into(),
4630 edge.properties.as_str().into(),
4631 stamp.into(),
4632 edge.branch_name().into(),
4633 ]
4634}
4635
4636/// Shared by the single-concept write and the chunked one, so the two paths
4637/// cannot drift into upserting different column sets — and so the chunk has a
4638/// statement text it can prepare once (D-056).
4639const UPSERT_CONCEPT: &str = "INSERT INTO concepts \
4640 (id, title, content, embedding_model, valid_from, valid_to, recorded_at, retired, \
4641 branch_id) \
4642 VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9) \
4643 ON CONFLICT(id) DO UPDATE SET \
4644 title = excluded.title, \
4645 content = excluded.content, \
4646 embedding_model = excluded.embedding_model, \
4647 valid_from = excluded.valid_from, \
4648 valid_to = excluded.valid_to, \
4649 recorded_at = excluded.recorded_at, \
4650 retired = excluded.retired";
4651// `branch_id` is deliberately **not** in that `DO UPDATE` list. The column is
4652// provenance and minting happened once (D-214), and
4653// `trg_concepts_branch_immutable` would abort an update that moved it — so
4654// listing it would turn every re-upsert of an inherited concept into a guard
4655// abort instead of the no-op it is. The insert arm carries it; the update arm
4656// leaves the row where it was minted.
4657
4658/// The parameter row for [`UPSERT_CONCEPT`], in one place for the same reason.
4659fn concept_params<'a>(concept: &'a ConceptUpsert, stamp: &'a str) -> [libsql::Value; 9] {
4660 [
4661 concept.id.as_str().into(),
4662 concept.title.as_str().into(),
4663 concept.content.as_str().into(),
4664 concept
4665 .embedding_model
4666 .as_deref()
4667 .map_or(libsql::Value::Null, Into::into),
4668 concept.valid_from.as_str().into(),
4669 concept.valid_to.as_str().into(),
4670 stamp.into(),
4671 (concept.retired as i64).into(),
4672 concept.branch_name().into(),
4673 ]
4674}
4675
4676/// Check every lineage a write names, and decide which shape its guard takes.
4677///
4678/// **One function, two answers, one query per distinct lineage** — and it is
4679/// [`Lineages::shape`](crate::graph::lineage::Lineages::shape), the same function
4680/// the read path calls, for the same reason it calls it. A write naming a
4681/// branch that is not in `branches` has asked about something that does not
4682/// exist, and answering it by writing to the trunk is [D-069]'s failure in its
4683/// most expensive form: not a right-looking answer to a question that was not
4684/// asked, but a *durable* one.
4685///
4686/// Relying on the foreign key instead would refuse the write — `branch_id`
4687/// `REFERENCES branches(branch_id)` and the key is enforced — but it would
4688/// refuse it as an unqualified "FOREIGN KEY constraint failed" from inside a
4689/// rolled-back transaction, naming neither the column nor the branch. The same
4690/// argument [`classify`](crate::error::classify) makes for annotations and
4691/// edges, one table further along.
4692///
4693/// # Why a trunk write pays for it too
4694///
4695/// `None` resolves to `'main'` here rather than skipping the check, and that is
4696/// not tidiness. Once a second lineage can write, the *trunk's* overlap guard
4697/// is wrong in the other direction — it would be refused for overlapping a
4698/// branch's belief it cannot see — so the shape decision is one every write
4699/// needs, not one that branched writes need. On a database that has never
4700/// forked the answer is [`LineageShape::Trunk`] and the guard is the statement
4701/// it has always been.
4702///
4703/// # This was a query per name until 0.15.6
4704///
4705/// It ran [`crate::graph::lineage::Lineages::shape`] once per name and kept the
4706/// last answer — one round trip per write, for a table only this task writes.
4707/// [`ActorState`] holds `branches` instead, and [`Lineages::shape_of`] carries
4708/// what is left of this function's reasoning, including the part about the last
4709/// answer that stopped being true at 0.15.2.
4710///
4711/// [D-069]: ../../docs/architecture/s13-decision-register.md
4712async fn check_lineages(
4713 state: &mut ActorState,
4714 conn: &libsql::Connection,
4715 names: &[&str],
4716) -> Result<LineageShape> {
4717 state.shape_of(conn, names).await
4718}
4719
4720/// The shape and the rows, for a caller that also has to resolve an ancestry.
4721///
4722/// [`check_lineages`] with the table it read handed back rather than dropped
4723/// (0.15.17). Both callers need it: the guard compiles a statement per lineage,
4724/// and the retirement binds one.
4725async fn check_lineages_with<'a>(
4726 state: &'a mut ActorState,
4727 conn: &libsql::Connection,
4728 names: &[&str],
4729) -> Result<(LineageShape, &'a Lineages)> {
4730 let lineages = state.lineages(conn).await?;
4731 Ok((lineages.shape_of(names)?, lineages))
4732}
4733
4734/// The distinct lineages a batch names, in first-seen order.
4735///
4736/// A `Vec` and a linear scan rather than a set: batches name one lineage in
4737/// every case this crate has, the bound is the number of *branches* and not the
4738/// number of rows, and a `BTreeSet` would allocate per batch to deduplicate a
4739/// list of length one.
4740fn distinct_branches(edges: &[EdgeAssertion]) -> Vec<&str> {
4741 let mut out: Vec<&str> = Vec::with_capacity(1);
4742 for edge in edges {
4743 let name = edge.branch_name();
4744 if !out.contains(&name) {
4745 out.push(name);
4746 }
4747 }
4748 if out.is_empty() {
4749 out.push(crate::schema::ddl::MAIN_BRANCH);
4750 }
4751 out
4752}
4753
4754/// The overlap guard's prepared statement, and which question it asks.
4755///
4756/// The two statements take different parameter counts and mean different things
4757/// by the rows they return, so pairing them with the shape here is what stops
4758/// [`check_prepared`] from having to be told twice.
4759struct OverlapGuard {
4760 stmt: libsql::Statement,
4761 shape: LineageShape,
4762 /// The lineage this statement was compiled for. See [`Self::prepare`].
4763 branch: String,
4764 /// That lineage's ancestry, bound after the statement's own parameters.
4765 /// Empty under both trunk shapes, which emit no `lineage` relation.
4766 ancestry: Vec<Ancestor>,
4767}
4768
4769impl OverlapGuard {
4770 /// Prepare once per turn or per chunk, never per row (D-056, §8.8).
4771 ///
4772 /// # One statement per *lineage* since 0.15.17 ([D-259])
4773 ///
4774 /// The statement used to be a function of the shape alone: the recursive
4775 /// `lineage` CTE derived the ancestry from the branch bound at `?5`, so one
4776 /// compiled form answered for every lineage that shared a shape. A bound
4777 /// ancestry is not derived from anything — it *is* the answer for one
4778 /// reader — so the guard now carries the lineage it was compiled for and
4779 /// the values that lineage binds.
4780 ///
4781 /// The cost is bounded by [`distinct_branches`], which is a `Vec` because
4782 /// every batch this crate has seen names one lineage. A chunk that names
4783 /// two prepares two, which is the price of the resolution being correct for
4784 /// both; the shape it would otherwise share is `Resolved`, since the two
4785 /// trunk shapes each describe a database with exactly one lineage to name.
4786 ///
4787 /// [D-259]: ../../docs/architecture/s13-decision-register.md#d-259
4788 async fn prepare(
4789 conn: &libsql::Connection,
4790 shape: LineageShape,
4791 lineages: &Lineages,
4792 branch: &str,
4793 ) -> Result<Self> {
4794 // Three shapes, three statements, one spelling (0.15.8, W13.3,
4795 // D-250). Until that release the trunk had a hand-written constant and
4796 // the other two shared the resolved form, which was exact for a root
4797 // only because a root's ancestry is itself — so `Lineages::shape_of`
4798 // could return either of them and nothing observable changed. It
4799 // cannot now: the root gets a two-predicate lookup on the projection
4800 // and a branch gets the four-CTE resolution, and D-248's C-24 repair
4801 // is what decides which.
4802 let ancestry = match shape {
4803 LineageShape::Resolved => lineages.ancestry(branch),
4804 _ => Vec::new(),
4805 };
4806 let sql = crate::graph::lineage::overlap_candidates_resolved(shape, &ancestry);
4807 Ok(Self {
4808 stmt: conn.prepare(&sql).await?,
4809 shape,
4810 branch: branch.to_string(),
4811 ancestry,
4812 })
4813 }
4814
4815 /// Whether this guard answers for `branch` under `shape`.
4816 fn answers_for(&self, shape: LineageShape, branch: &str) -> bool {
4817 self.shape == shape && self.branch == branch
4818 }
4819}
4820
4821impl HighPriCommand {
4822 /// The metrics label for this variant (T1.4).
4823 ///
4824 /// Exhaustive for the same reason `execute` is: a new variant that silently
4825 /// borrowed another's label would attribute its holds to the wrong command,
4826 /// and the one question the counters exist to answer is *which* command
4827 /// broke the budget.
4828 fn kind(&self) -> crate::metrics::CommandKind {
4829 use crate::metrics::CommandKind as K;
4830 match self {
4831 HighPriCommand::AssertEdge { .. } => K::AssertEdge,
4832 HighPriCommand::RetireEdge { .. } => K::RetireEdge,
4833 HighPriCommand::UpsertConcept { .. } => K::UpsertConcept,
4834 HighPriCommand::WriteBulkAtomic { .. } => K::WriteBulkAtomic,
4835 HighPriCommand::RebuildCurrent { .. } => K::RebuildCurrent,
4836 HighPriCommand::RegisterModel { .. } => K::RegisterModel,
4837 HighPriCommand::Fork { .. } => K::Fork,
4838 HighPriCommand::Checkpoint { .. } => K::Checkpoint,
4839 HighPriCommand::Shutdown { .. } => K::Shutdown,
4840 }
4841 }
4842
4843 /// Run one command and answer its caller.
4844 ///
4845 /// Deliberately exhaustive — there is no `_` arm. The 0.4.5–0.5.4 actor
4846 /// matched `Shutdown` and `AssertEdge` and sent everything else to
4847 /// `_ => LoopCtl::Continue`, which **dropped the responder**: the caller's
4848 /// `rx.await` resolved to a `RecvError` that no code mapped, so four of six
4849 /// commands were indistinguishable from a hung database. An exhaustive match
4850 /// makes that failure a compile error instead of a runtime silence, which is
4851 /// why adding a variant should break this function.
4852 async fn execute(
4853 self,
4854 conn: &libsql::Connection,
4855 clock: &dyn Clock,
4856 turn: &Turn<'_>,
4857 state: &mut ActorState,
4858 ) -> LoopCtl {
4859 match self {
4860 HighPriCommand::Shutdown { responder } => {
4861 turn.answer(responder, Ok(()));
4862 return LoopCtl::Break;
4863 }
4864 HighPriCommand::Checkpoint { responder } => {
4865 let res = run_checkpoint(conn).await;
4866 turn.answer(responder, res);
4867 }
4868 HighPriCommand::AssertEdge { edge, responder } => {
4869 let stamp = clock.now();
4870 // Before the guard, because a write naming an unregistered
4871 // lineage should be refused by name rather than by whatever the
4872 // guard happens to find when it looks in the wrong place.
4873 let shape = match check_lineages(state, conn, &[edge.branch_name()]).await {
4874 Ok(shape) => shape,
4875 Err(e) => {
4876 turn.answer(responder, Err(e));
4877 return LoopCtl::Continue;
4878 }
4879 };
4880 if let Err(e) = reject_overlapping_interval(state, conn, &edge, shape).await {
4881 turn.answer(responder, Err(e));
4882 return LoopCtl::Continue;
4883 }
4884 // The statement is held across turns, so it is reset before it
4885 // is bound rather than after it was stepped — `check_prepared`
4886 // makes the same argument at more length.
4887 let res = match state.insert_link(conn).await {
4888 Err(e) => Err(e),
4889 Ok(stmt) => {
4890 stmt.reset();
4891 match stmt.execute(edge_params(&edge, &stamp)).await {
4892 Ok(_) => Ok(()),
4893 Err(e) => Err(classify(
4894 conn,
4895 e,
4896 WriteOp::Edge {
4897 source_id: &edge.source,
4898 target_id: &edge.target,
4899 edge_type: &edge.edge_type,
4900 },
4901 )
4902 .await),
4903 }
4904 }
4905 };
4906 turn.answer(responder, res);
4907 }
4908 HighPriCommand::RetireEdge {
4909 source,
4910 target,
4911 edge_type,
4912 valid_from,
4913 valid_to,
4914 branch,
4915 responder,
4916 } => {
4917 let stamp = clock.now();
4918 let name = branch
4919 .as_ref()
4920 .map_or(crate::schema::ddl::MAIN_BRANCH, |b| b.as_str());
4921 let resolved = check_lineages_with(state, conn, &[name])
4922 .await
4923 .map(|(shape, l)| (shape, l.ancestry(name)));
4924 let res = match resolved {
4925 Ok((shape, ancestry)) => {
4926 retire_edge(
4927 conn,
4928 &source,
4929 &target,
4930 &edge_type,
4931 &valid_from,
4932 &valid_to,
4933 &stamp,
4934 name,
4935 shape,
4936 &ancestry,
4937 )
4938 .await
4939 }
4940 Err(e) => Err(e),
4941 };
4942 turn.answer(responder, res);
4943 }
4944 HighPriCommand::UpsertConcept { concept, responder } => {
4945 let stamp = clock.now();
4946 let res = match check_lineages(state, conn, &[concept.branch_name()]).await {
4947 Ok(_) => upsert_concept(conn, &concept, &stamp).await,
4948 Err(e) => Err(e),
4949 };
4950 turn.answer(responder, res);
4951 }
4952 HighPriCommand::WriteBulkAtomic { edges, responder } => {
4953 // One stamp for the whole batch (D-014): the rows were asserted
4954 // by one act, and giving them different transaction times would
4955 // invent an ordering the caller never expressed.
4956 let stamp = clock.now();
4957 let res = write_edges_atomic(state, conn, &edges, &stamp).await;
4958 turn.answer(responder, res);
4959 }
4960 HighPriCommand::RebuildCurrent { responder } => {
4961 turn.answer(responder, rebuild_current(conn).await);
4962 }
4963 HighPriCommand::RegisterModel {
4964 model,
4965 dim,
4966 responder,
4967 } => {
4968 turn.answer(
4969 responder,
4970 crate::vector::register_model(conn, &model, dim).await,
4971 );
4972 }
4973 HighPriCommand::Fork {
4974 name,
4975 parent,
4976 responder,
4977 } => {
4978 // The same clock as every other write, and the same instant in
4979 // both columns: `forked_at` is a transaction-time point in the
4980 // parent's history, and the point this release can fork from is
4981 // now. See `branch::Branch::created_at` for why they are two
4982 // columns anyway.
4983 let stamp = clock.now();
4984 let res = crate::branch::fork(conn, &name, &parent, &stamp).await;
4985 // `branches` has a row it did not have. Unconditional rather
4986 // than `if res.is_ok()`: a fork that failed leaves the table
4987 // as it was, so forgetting costs one query and asserting that
4988 // it failed cleanly costs an argument (0.15.6, D-248).
4989 state.forget_lineages();
4990 turn.answer(responder, res);
4991 }
4992 }
4993 LoopCtl::Continue
4994 }
4995}
4996
4997impl LowPriCommand {
4998 /// The metrics label for this variant (T1.4). See [`HighPriCommand::kind`].
4999 fn kind(&self) -> crate::metrics::CommandKind {
5000 use crate::metrics::CommandKind as K;
5001 match self {
5002 LowPriCommand::WriteConceptsChunk { .. } => K::WriteConceptsChunk,
5003 LowPriCommand::WriteAnalyticsChunk { .. } => K::WriteAnalyticsChunk,
5004 LowPriCommand::UpsertEmbeddingChunk { .. } => K::UpsertEmbeddingChunk,
5005 LowPriCommand::DropEmbeddingIndex { .. } => K::DropEmbeddingIndex,
5006 LowPriCommand::RebuildEmbeddingIndex { .. } => K::RebuildEmbeddingIndex,
5007 LowPriCommand::BulkImportChunk { .. } => K::BulkImportChunk,
5008 LowPriCommand::LinksCurrentMirror { .. } => K::LinksCurrentMirror,
5009 LowPriCommand::Archive { .. } => K::Archive,
5010 // Its own counter since 0.12.9 (W4.3, D-152). It reported as
5011 // `K::Archive` from 0.9.0 to 0.12.8 — the budget really is shared,
5012 // but attribution is not budget, and an operator reading a long
5013 // `archive` hold could not tell whether anything had been archived.
5014 // What kept it folded was that a `CommandKind` variant was a
5015 // breaking addition; `#[non_exhaustive]` (W4.2) removed that.
5016 LowPriCommand::Rehydrate { .. } => K::Rehydrate,
5017 // Its own counter from the day it shipped, which is the whole point
5018 // of the paragraph above: `Rehydrate` spent four releases folded
5019 // into `Archive` for a reason that was never good, and the cost of
5020 // unfolding it was a rung's worth of care about declaration order.
5021 LowPriCommand::ArchiveBranch { .. } => K::ArchiveBranch,
5022 LowPriCommand::RebuildFts { .. } => K::RebuildFts,
5023 // Two kinds out of one variant since 0.13.24 (W10.5, D-197). The
5024 // command carries the flag; the counter has to carry it too, or the
5025 // budget exemption for either half is decided about both (D-168).
5026 LowPriCommand::Analyze { incremental, .. } => {
5027 if *incremental {
5028 K::Optimize
5029 } else {
5030 K::Analyze
5031 }
5032 }
5033 // Two kinds out of one variant since 0.14.16 (W12.16, D-233),
5034 // and for D-197's reason one line up: the command carries the step,
5035 // so the counter has to carry it too, or the budget exemption for
5036 // either half is decided about both. Here that is not hypothetical
5037 // — the halves want opposite answers. The swap is over budget by
5038 // construction and the fill chunks are meant to fit, so a merged
5039 // kind's `over_budget` read `N(rebuilds) + regressions` and could
5040 // not be decomposed.
5041 LowPriCommand::ShadowRebuild { step, .. } => match step {
5042 crate::integrity::ShadowStep::Swap { .. } => K::ShadowSwap,
5043 crate::integrity::ShadowStep::Begin | crate::integrity::ShadowStep::Fill { .. } => {
5044 K::ShadowRebuild
5045 }
5046 },
5047 }
5048 }
5049
5050 /// Run one background command and answer its caller.
5051 ///
5052 /// Also exhaustive. The pre-0.5.4 version was a single `LoopCtl::Continue`
5053 /// for *every* variant — every background write silently discarded, its
5054 /// caller waiting forever.
5055 async fn execute(
5056 self,
5057 conn: &libsql::Connection,
5058 clock: &dyn Clock,
5059 turn: &Turn<'_>,
5060 state: &mut ActorState,
5061 ) -> LoopCtl {
5062 match self {
5063 LowPriCommand::BulkImportChunk { chunk, responder } => {
5064 // A stamp per chunk, not per batch: the chunks commit
5065 // separately, so a shared stamp would claim a simultaneity the
5066 // storage does not have.
5067 let stamp = clock.now();
5068 turn.answer_chunk(
5069 responder,
5070 write_edges_atomic(state, conn, &chunk, &stamp).await,
5071 );
5072 }
5073 LowPriCommand::LinksCurrentMirror { present, responder } => {
5074 // Schema work on the actor's own connection, like
5075 // RegisterModel and the embedding-index pair: the
5076 // single-writer invariant is what keeps the mirror's window
5077 // on one timeline with the chunks it wraps.
5078 let sql = if present {
5079 crate::schema::ddl::CREATE_LINKS_CURRENT_SYNC.to_string()
5080 } else {
5081 "DROP TRIGGER IF EXISTS trg_links_current_sync".to_string()
5082 };
5083 turn.answer(
5084 responder,
5085 conn.execute(&sql, ()).await.map(|_| ()).map_err(Into::into),
5086 );
5087 }
5088 LowPriCommand::WriteConceptsChunk { chunk, responder } => {
5089 let stamp = clock.now();
5090 turn.answer_chunk(
5091 responder,
5092 write_concepts_atomic(state, conn, &chunk, &stamp).await,
5093 );
5094 }
5095 LowPriCommand::WriteAnalyticsChunk { chunk, responder } => {
5096 let stamp = clock.now();
5097 turn.answer_chunk(
5098 responder,
5099 write_annotations_atomic(conn, &chunk, &stamp).await,
5100 );
5101 }
5102 LowPriCommand::UpsertEmbeddingChunk {
5103 model,
5104 chunk,
5105 responder,
5106 } => {
5107 // No clock reading: an embedding carries no timestamp on either
5108 // axis. It is a derived artifact of a model applied to content
5109 // (Doctrine VII), and the ledger already records when the
5110 // content changed.
5111 turn.answer_chunk(
5112 responder,
5113 crate::vector::search::upsert_embedding_chunk(conn, &model, &chunk).await,
5114 );
5115 }
5116 LowPriCommand::DropEmbeddingIndex { model, responder } => {
5117 // Schema work on the actor's own connection, like
5118 // RegisterModel: the single-writer invariant is what keeps the
5119 // drop and the load on one timeline.
5120 turn.answer(
5121 responder,
5122 crate::vector::registry::drop_embedding_index(conn, &model).await,
5123 );
5124 }
5125 LowPriCommand::RebuildEmbeddingIndex { model, responder } => {
5126 // One statement, one turn, no smaller unit. Its hold is the
5127 // one-pass DiskANN build's and it is budget-exempt by the same
5128 // criterion as `ShadowSwap` — see the CommandKind doc.
5129 turn.answer(
5130 responder,
5131 crate::vector::registry::rebuild_embedding_index(conn, &model).await,
5132 );
5133 }
5134 LowPriCommand::Archive {
5135 cutoff,
5136 archive_path,
5137 responder,
5138 } => {
5139 // The archive *time*, not the cutoff. `archive_horizon` records
5140 // both and they are different facts — see `archive()` (Wave 4.5).
5141 let archived_at = clock.now();
5142 let res = archive(conn, &cutoff, &archived_at, &archive_path).await;
5143 // The session attached and detached a second database. The
5144 // statements are dropped rather than trusted to recompile —
5145 // see [`ActorState::forget_statements`].
5146 state.forget_statements();
5147 // Before the answer, so a shadow rebuild that reads the epoch on
5148 // its next turn cannot miss an archive that has already deleted
5149 // rows out from under it (T1.2).
5150 if res.is_ok() {
5151 turn.archive_committed();
5152 }
5153 turn.answer(responder, res);
5154 }
5155 LowPriCommand::ArchiveBranch {
5156 branch,
5157 archive_path,
5158 responder,
5159 } => {
5160 // The wall clock, recorded in `cold.branches.archived_at`: when
5161 // the ledger stopped knowing about the lineage. Not a ledger
5162 // fact and not on either of Doctrine II's timelines — nothing
5163 // was asserted or retired here.
5164 let archived_at = clock.now();
5165 let res = crate::temporal::archive::archive_branch(
5166 conn,
5167 &branch,
5168 &archived_at,
5169 &archive_path,
5170 )
5171 .await;
5172 // Both: this session attaches a second database *and* deletes
5173 // the lineage's row from `branches`.
5174 state.forget_everything();
5175 // `Archive`'s reason exactly: a shadow rebuild reading the epoch
5176 // on its next turn must not miss a session that has already
5177 // deleted rows out from under it (T1.2).
5178 if res.is_ok() {
5179 turn.archive_committed();
5180 }
5181 turn.answer(responder, res);
5182 }
5183 LowPriCommand::Rehydrate {
5184 ids,
5185 archive_path,
5186 responder,
5187 } => {
5188 let refs: Vec<&str> = ids.iter().map(String::as_str).collect();
5189 let res = rehydrate(conn, &refs, &archive_path).await;
5190 // Attaches, like the two archive sessions. It restores concepts
5191 // and log rows, never a lineage — `cold.branches` is read by
5192 // `archive_hint` and not written back — so the lineages stand.
5193 state.forget_statements();
5194 // Same reason as `Archive`: rehydration moves rows into `links`'
5195 // parent table, so a shadow rebuild in flight must see the epoch
5196 // move before the caller is answered (T1.2).
5197 if res.is_ok() {
5198 turn.archive_committed();
5199 }
5200 turn.answer(responder, res);
5201 }
5202 LowPriCommand::ShadowRebuild { step, responder } => {
5203 use crate::integrity::{shadow, ShadowOutcome, ShadowStep};
5204 let res = match step {
5205 ShadowStep::Begin => {
5206 shadow::begin(conn)
5207 .await
5208 .map(|build_start| ShadowOutcome::Started {
5209 build_start,
5210 epoch: turn.epoch(),
5211 })
5212 }
5213 ShadowStep::Fill { after } => shadow::fill_chunk(conn, after.as_deref())
5214 .await
5215 .map(|last| ShadowOutcome::Filled { last }),
5216 ShadowStep::Swap { build_start, epoch } => {
5217 shadow::swap(conn, &build_start, epoch, turn.epoch())
5218 .await
5219 .map(|rows| ShadowOutcome::Swapped { rows })
5220 }
5221 };
5222 turn.answer(responder, res);
5223 }
5224 LowPriCommand::RebuildFts { responder } => {
5225 let res = conn
5226 .execute(crate::schema::ddl::REBUILD_CONCEPTS_FTS, ())
5227 .await
5228 .map(|_| ())
5229 .map_err(Into::into);
5230 turn.answer(responder, res);
5231 }
5232 LowPriCommand::Analyze {
5233 incremental,
5234 responder,
5235 } => {
5236 // Both go through `query()`, not `execute()`. `PRAGMA optimize`
5237 // yields rows, and libsql's `execute()` rejects any statement
5238 // that does ("Execute returned rows") — the same trap
5239 // `configure` documents. `ANALYZE` does not yield rows, but is
5240 // issued the same way so the two arms cannot drift into needing
5241 // different call shapes for no visible reason.
5242 let sql = if incremental {
5243 crate::schema::ddl::OPTIMIZE
5244 } else {
5245 crate::schema::ddl::ANALYZE
5246 };
5247 let res = conn.query(sql, ()).await.map(|_| ()).map_err(Into::into);
5248 turn.answer(responder, res);
5249 }
5250 }
5251 LoopCtl::Continue
5252 }
5253}
5254
5255/// Close an open interval by asserting its successor (Doctrine III).
5256///
5257/// Never an `UPDATE`. The replacement row copies weight and properties from
5258/// current belief and differs only in `valid_to` and `recorded_at`, so the
5259/// original assertion survives intact and `reconstruct` at an earlier instant
5260/// still sees the interval open — which is the entire point of a bitemporal
5261/// ledger.
5262// The first of these in the crate proper (0.14.8). All nine are the edge key,
5263// two stamps and the lineage — a struct to carry them would exist for one call
5264// site and would put a name between the caller and parameters it already spells
5265// out positionally at the only place it calls this.
5266#[allow(clippy::too_many_arguments)]
5267async fn retire_edge(
5268 conn: &libsql::Connection,
5269 source: &str,
5270 target: &str,
5271 edge_type: &str,
5272 valid_from: &str,
5273 valid_to: &str,
5274 stamp: &str,
5275 branch: &str,
5276 shape: LineageShape,
5277 ancestry: &[Ancestor],
5278) -> Result<()> {
5279 // Shadow retirement: the row being closed may belong to an ancestor, and
5280 // the row written carries *this* lineage's id. See
5281 // `lineage::retire_from_resolved`.
5282 //
5283 // One statement for all three shapes since 0.15.8 (W13.3, D-250). The
5284 // trunk had its own until then, kept apart on [`LineageShape`]'s ground —
5285 // the resolved form was opaque to the planner and cost 3.0x where there
5286 // was nothing to resolve (D-220). That ground is gone rather than
5287 // overruled: a keyed `Trunk` resolution lowers to no CTEs at all, so the
5288 // statement the lowering emits *is* the one this arm used to hold, with
5289 // the lineage stamped rather than defaulted.
5290 // The ancestry follows the seven, at `RETIRE_ANCESTRY_SLOT`.
5291 let mut params: Vec<libsql::Value> = vec![
5292 source.into(),
5293 target.into(),
5294 edge_type.into(),
5295 valid_from.into(),
5296 branch.into(),
5297 valid_to.into(),
5298 stamp.into(),
5299 ];
5300 params.extend(crate::graph::lineage::ancestry_params(ancestry));
5301 let affected = conn
5302 .execute(
5303 &crate::graph::lineage::retire_from_resolved(shape, ancestry),
5304 params,
5305 )
5306 .await
5307 .map_err(DbError::Engine)?;
5308
5309 if affected == 0 {
5310 return Err(DbError::NotFound(format!(
5311 "{source} -> {target} ({edge_type}) at {valid_from}"
5312 )));
5313 }
5314 Ok(())
5315}
5316
5317async fn upsert_concept(
5318 conn: &libsql::Connection,
5319 concept: &ConceptUpsert,
5320 stamp: &str,
5321) -> Result<()> {
5322 let res = conn
5323 .execute(UPSERT_CONCEPT, concept_params(concept, stamp))
5324 .await;
5325
5326 match res {
5327 Ok(_) => Ok(()),
5328 Err(e) => Err(classify(
5329 conn,
5330 e,
5331 WriteOp::Concept {
5332 id: &concept.id,
5333 recorded_at: stamp,
5334 branch: concept.branch_name(),
5335 },
5336 )
5337 .await),
5338 }
5339}
5340
5341/// Whether this pair is the storage layer's case rather than this guard's.
5342///
5343/// Two **open** intervals overlap — they share every instant from the later
5344/// start onwards — so a naive overlap check reports them, and reporting them
5345/// here would leave `DbError::SingleOpenViolation` constructible by nothing.
5346/// That variant is the more specific error, it is enforced by
5347/// `trg_links_single_open` rather than by this function, and its field names
5348/// were ratified in §1.2. Shadowing it with a general one would be defect Q's
5349/// shape reintroduced by a fix: a typed error that no code path can produce.
5350///
5351/// So the two guards partition the space rather than overlapping it. Both open
5352/// belongs to the trigger. Everything else — open against closed, closed against
5353/// closed — is unguarded at the storage layer and belongs here. That the split
5354/// is exactly the trigger's `WHEN` clause is not a coincidence; it is the
5355/// definition of what was missing.
5356fn defer_to_single_open(proposed: &Interval, existing: &Interval) -> bool {
5357 proposed.is_open() && existing.is_open()
5358}
5359
5360/// Refuse an assertion whose valid-time interval overlaps one already recorded
5361/// for the same `(source, target, edge_type)` — **defect AA, D-060**.
5362///
5363/// `trg_links_single_open` fires only `WHEN NEW.valid_to = '9999-…'`, so it
5364/// guards the open sentinel and nothing else. Two *closed* intervals that
5365/// overlap were accepted without complaint, and `query_as_of_edges` at an
5366/// instant inside both returned one relationship as two edges.
5367///
5368/// **This runs in the write actor, which is what makes it sound.** The obvious
5369/// place is `EdgeAssertion::normalized`, and it cannot go there — `normalized`
5370/// is a pure function with no connection, and doing the read at the API boundary
5371/// instead would leave a check-then-write race between the read and the actor's
5372/// insert. Inside the actor there is one writer by construction (D-014), and for
5373/// the batch paths this runs inside the same transaction as the insert, so the
5374/// window does not exist rather than being small.
5375///
5376/// **What it does not cover, and §4.2 now says so:** raw SQL against the same
5377/// file. The storage layer permits what this API refuses, which is the honest
5378/// cost of not putting the check in a trigger. The alternative was a second
5379/// index probe inside `trg_links_single_open` on every insert — on the path
5380/// D-059 has just finished making fast — for a guarantee that only holds against
5381/// callers who were going through the actor anyway.
5382///
5383/// `valid_from <> ?4` excludes the row being re-asserted. Re-assertion at the
5384/// same `valid_from` is Doctrine III's ordinary case — a new belief about the
5385/// same interval — and is settled by the primary key and the single-open
5386/// trigger, not here.
5387/// The single-assertion path holds one statement across turns (0.15.6, D-248);
5388/// the batch path prepares one inside its own transaction and calls
5389/// [`check_prepared`] per row.
5390async fn reject_overlapping_interval(
5391 state: &mut ActorState,
5392 conn: &libsql::Connection,
5393 edge: &EdgeAssertion,
5394 shape: LineageShape,
5395) -> Result<()> {
5396 let branch = edge.branch_name().to_string();
5397 check_prepared(state.guard(conn, shape, &branch).await?, edge).await
5398}
5399
5400/// The guard's body, against a statement the caller has already prepared.
5401///
5402/// **Split out because preparing per row was worth 10.4 ms on a 90-edge chunk**
5403/// (§8.8) — the same defect D-056 and D-057 diagnosed and fixed for
5404/// `INSERT_LINK`, reintroduced by the Wave 2 guard that was written beside it.
5405/// Measured with and without the guard, on a 2,000-edge hub: 8.65 ms → 19.25 ms,
5406/// and *identical* with and without `idx_lc_open_interval`, which is what
5407/// identified preparation rather than a scan as the cost. A guard that reads an
5408/// index correctly and prepares its statement 90 times is indistinguishable, at
5409/// the call site, from one that scans.
5410///
5411/// `reset()` between rows is not optional: libsql binds and steps without
5412/// resetting, so a reused statement must be returned to its initial state.
5413///
5414/// # And once more on the way out (0.15.6, W14.3)
5415///
5416/// The reset used to be enough at the top, because the guard was compiled per
5417/// call or per chunk and dropped where it was made — the drop finalized it, and
5418/// SQLite's objection to a live statement never came up. [`ActorState`] holds
5419/// this one across turns, and the loop below can leave a cursor open on it: the
5420/// overlap arm returns from inside the `while`. A statement left mid-scan is
5421/// what makes SQLite refuse to end a transaction, so the next `Archive` — not
5422/// the next assertion — would be the thing that failed, a command and a
5423/// diagnosis apart from the code that caused it. So the scan is a function of
5424/// its own, and the statement is reset on both ways out of it.
5425async fn check_prepared(guard: &OverlapGuard, edge: &EdgeAssertion) -> Result<()> {
5426 let proposed = Interval::new(edge.valid_from.clone(), edge.valid_to.clone());
5427
5428 guard.stmt.reset();
5429 // The resolved form takes a fifth parameter, the writing lineage, and
5430 // returns what that lineage can see; the trunk form takes four and returns
5431 // the table. Binding five to the trunk statement would be an error from
5432 // libsql rather than a wrong answer, which is the failure mode to prefer.
5433 let mut rows = match guard.shape {
5434 LineageShape::Trunk => {
5435 guard
5436 .stmt
5437 .query(libsql::params![
5438 edge.source.as_str(),
5439 edge.target.as_str(),
5440 edge.edge_type.as_str(),
5441 edge.valid_from.as_str()
5442 ])
5443 .await?
5444 }
5445 LineageShape::Resolved | LineageShape::TrunkOnForked => {
5446 // The ancestry follows the five, at `GUARD_ANCESTRY_SLOT`, and is
5447 // empty for `TrunkOnForked` — a root emits no `lineage` relation.
5448 let mut params: Vec<libsql::Value> = vec![
5449 edge.source.as_str().into(),
5450 edge.target.as_str().into(),
5451 edge.edge_type.as_str().into(),
5452 edge.valid_from.as_str().into(),
5453 edge.branch_name().into(),
5454 ];
5455 params.extend(crate::graph::lineage::ancestry_params(&guard.ancestry));
5456 guard.stmt.query(params).await?
5457 }
5458 };
5459
5460 let verdict = scan_candidates(&mut rows, &proposed, edge).await;
5461 drop(rows);
5462 guard.stmt.reset();
5463 verdict
5464}
5465
5466/// The guard's loop, over candidates the statement has already produced.
5467///
5468/// Split from [`check_prepared`] so that the statement is reset on both exits
5469/// from it, including the one that returns an overlap.
5470async fn scan_candidates(
5471 rows: &mut libsql::Rows,
5472 proposed: &Interval,
5473 edge: &EdgeAssertion,
5474) -> Result<()> {
5475 while let Some(row) = rows.next().await? {
5476 let existing = Interval::new(row.get::<String>(0)?, row.get::<String>(1)?);
5477 if defer_to_single_open(proposed, &existing) {
5478 continue;
5479 }
5480 if proposed.overlaps(&existing) {
5481 return Err(DbError::OverlappingInterval {
5482 overlap: Box::new(crate::error::Overlap {
5483 source_id: edge.source.clone(),
5484 target_id: edge.target.clone(),
5485 edge_type: edge.edge_type.clone(),
5486 valid_from: edge.valid_from.clone(),
5487 valid_to: edge.valid_to.clone(),
5488 existing_from: existing.valid_from,
5489 existing_to: existing.valid_to,
5490 // This guard reads committed rows, so the interval it names
5491 // is one the caller can go and look at (D-180).
5492 within_batch: false,
5493 }),
5494 });
5495 }
5496 }
5497
5498 Ok(())
5499}
5500
5501/// The same guard applied *within* a batch, before any of it is written.
5502///
5503/// The database check cannot see rows that are not in the database yet, so a
5504/// batch carrying two overlapping intervals for one relationship would pass
5505/// every per-row check and commit the overlap in one transaction.
5506///
5507/// # Sorted and swept rather than compared pairwise (0.13.6, W7.5, D-179)
5508///
5509/// This used to compare every pair. At [`chunk_rows::EDGES`] = 90 that is
5510/// nothing, and the chunked paths are the only ones where 90 is the bound —
5511/// [`Database::write_bulk_atomic`] is exempt from [`CHUNK_BUDGET`] by contract,
5512/// so its batch is whatever the caller passed, and the quadratic term is what
5513/// made 20,000 corrections to one relationship's history cost seconds rather
5514/// than milliseconds. Sorting by `(source, target, edge_type, valid_from)` and
5515/// sweeping costs `n log n` and changes nothing a caller can observe except the
5516/// wait.
5517///
5518/// **Adjacent pairs are not sufficient, and that is the whole difficulty.** For
5519/// plain intervals they would be: sort by start, and if any two overlap then
5520/// some neighbouring two overlap. That proof needs every pair to be *eligible*,
5521/// and here two are not — identical `valid_from` is re-assertion rather than
5522/// overlap, and two open intervals belong to `trg_links_single_open`. Skip an
5523/// adjacent pair for either reason and a real overlap can hide behind it:
5524/// `[5,20)`, `[5,6)`, `[7,8)` has the first pair skipped for equal `valid_from`
5525/// and the second not overlapping, while `[5,20)` and `[7,8)` overlap plainly.
5526/// So the sweep carries the widest `valid_to` reached so far instead of looking
5527/// only backwards one step, and carries a second one restricted to closed
5528/// intervals — because an open predecessor is excluded for an open candidate
5529/// and eligible for a closed one, which are different questions with different
5530/// answers.
5531///
5532/// Equal `valid_from` is handled by advancing in runs: everything with the same
5533/// start is checked against the maxima, and only then folded into them, so the
5534/// members of a run never see each other.
5535///
5536/// The report names the *earlier* interval as the existing one, which is the
5537/// pairwise version's input order only by accident. Within a batch neither is
5538/// older in transaction time — they arrive under one stamp — so valid-time order
5539/// is the only ordering that means anything, and it is the one a reader will
5540/// assume the words carry.
5541fn reject_overlaps_within(edges: &[EdgeAssertion]) -> Result<()> {
5542 // Indices, not the edges. The batch is borrowed and its order is the order
5543 // the rows are written in; sorting it would either clone it or reorder the
5544 // caller's data, which is `estimated_bulk_hold`'s reason for grouping too.
5545 let mut order: Vec<u32> = (0..edges.len() as u32).collect();
5546 order.sort_unstable_by(|&i, &j| {
5547 let a = &edges[i as usize];
5548 let b = &edges[j as usize];
5549 (
5550 &a.source,
5551 &a.target,
5552 &a.edge_type,
5553 a.branch_name(),
5554 &a.valid_from,
5555 )
5556 .cmp(&(
5557 &b.source,
5558 &b.target,
5559 &b.edge_type,
5560 b.branch_name(),
5561 &b.valid_from,
5562 ))
5563 });
5564
5565 fn key(e: &EdgeAssertion) -> (&str, &str, &str, &str) {
5566 (
5567 e.source.as_str(),
5568 e.target.as_str(),
5569 e.edge_type.as_str(),
5570 e.branch_name(),
5571 )
5572 }
5573 let at = |k: usize| &edges[order[k] as usize];
5574
5575 let mut group = 0;
5576 while group < order.len() {
5577 let mut group_end = group + 1;
5578 while group_end < order.len() && key(at(group_end)) == key(at(group)) {
5579 group_end += 1;
5580 }
5581
5582 // The furthest `valid_to` reached by anything already swept in this key
5583 // group, and the edge it came from so the error can name it. The second
5584 // one ignores open intervals: an open candidate may not be compared
5585 // against an open predecessor, and the sentinel would otherwise win the
5586 // maximum every time and make every such pair look like an overlap.
5587 let mut widest: Option<&EdgeAssertion> = None;
5588 let mut widest_closed: Option<&EdgeAssertion> = None;
5589
5590 let mut run = group;
5591 while run < group_end {
5592 let mut run_end = run + 1;
5593 while run_end < group_end && at(run_end).valid_from == at(run).valid_from {
5594 run_end += 1;
5595 }
5596
5597 for k in run..run_end {
5598 let e = at(k);
5599 let existing = if e.valid_to == timestamp::OPEN_SENTINEL {
5600 widest_closed
5601 } else {
5602 widest
5603 };
5604 let Some(p) = existing else { continue };
5605 // `Interval::overlaps` is `max(from) < min(to)`, and the sort
5606 // has already settled the max: `p.valid_from <= e.valid_from`.
5607 // What is left is the same predicate with the maximum resolved,
5608 // and it is written out rather than allocating two `Interval`s
5609 // per row to ask the same question.
5610 if e.valid_from < p.valid_to && e.valid_from < e.valid_to {
5611 return Err(DbError::OverlappingInterval {
5612 overlap: Box::new(crate::error::Overlap {
5613 source_id: e.source.clone(),
5614 target_id: e.target.clone(),
5615 edge_type: e.edge_type.clone(),
5616 valid_from: e.valid_from.clone(),
5617 valid_to: e.valid_to.clone(),
5618 existing_from: p.valid_from.clone(),
5619 existing_to: p.valid_to.clone(),
5620 // Nothing here is in the database, and the batch is
5621 // refused whole, so nothing here ever will be. The
5622 // message has to say so (D-180).
5623 within_batch: true,
5624 }),
5625 });
5626 }
5627 }
5628
5629 for k in run..run_end {
5630 let e = at(k);
5631 if widest.is_none_or(|w| e.valid_to > w.valid_to) {
5632 widest = Some(e);
5633 }
5634 if e.valid_to != timestamp::OPEN_SENTINEL
5635 && widest_closed.is_none_or(|w| e.valid_to > w.valid_to)
5636 {
5637 widest_closed = Some(e);
5638 }
5639 }
5640
5641 run = run_end;
5642 }
5643
5644 group = group_end;
5645 }
5646
5647 Ok(())
5648}
5649
5650/// Write every edge or none, under a single stamp.
5651///
5652/// **The statement is prepared once for the whole chunk (§9, D-056).** It used to
5653/// be `tx.execute(INSERT_LINK, …)` per row, which re-prepares on every call — and
5654/// `links` carries two triggers, so each preparation compiles their bodies along
5655/// with the insert.
5656///
5657/// Measured at 500 rows: **≈62 ms → ≈37 ms, a 41% saving.** Preparation was a
5658/// large cost and *not* the dominant one, which the first guess had it as. The
5659/// residual is the triggers themselves: the same 500 rows with
5660/// `trg_links_log_insert` and `trg_links_current_sync` dropped commit in **2.96
5661/// ms**, so trigger amplification is ~92% of what remains. There is no further
5662/// win available here without changing what the ledger records, and Doctrine IV
5663/// is what says it must be recorded. See D-056 for what that implies about §9's
5664/// ≤ 3 ms budget — briefly, 2.96 ms *is* the un-amplified figure, so the budget
5665/// appears to have been set without the amplification its own preamble says is
5666/// included.
5667///
5668/// `reset()` between rows is not optional: libsql's `execute` binds and steps
5669/// without resetting, so a reused statement must be returned to its initial state
5670/// or the second row steps a completed statement.
5671async fn write_edges_atomic(
5672 state: &mut ActorState,
5673 conn: &libsql::Connection,
5674 edges: &[EdgeAssertion],
5675 stamp: &str,
5676) -> Result<usize> {
5677 if edges.is_empty() {
5678 return Ok(0);
5679 }
5680
5681 // Before the transaction opens: a batch that contradicts itself is refused
5682 // without taking the write lock at all (D-060), and a batch naming a
5683 // lineage that does not exist is refused before it can take the lock at all
5684 // (0.14.8).
5685 reject_overlaps_within(edges)?;
5686 let branches = distinct_branches(edges);
5687 let (shape, lineages) = check_lineages_with(state, conn, &branches).await?;
5688 let lineages = lineages.clone();
5689
5690 let tx = conn
5691 .transaction_with_behavior(libsql::TransactionBehavior::Immediate)
5692 .await?;
5693
5694 // Inside the transaction, so the rows this checks against cannot change
5695 // between the check and the insert.
5696 // One preparation for the whole chunk, not one per row — see
5697 // `check_prepared`, and D-056 for the same lesson learned on `INSERT_LINK`.
5698 // One per lineage the chunk names, not one per row — see
5699 // `OverlapGuard::prepare` for why the shape alone stopped being enough, and
5700 // `distinct_branches` for why this is a `Vec` of length one in every batch
5701 // this crate has seen.
5702 let mut guards = Vec::with_capacity(branches.len());
5703 for name in &branches {
5704 guards.push(OverlapGuard::prepare(&tx, shape, &lineages, name).await?);
5705 }
5706 for edge in edges {
5707 let guard = guards
5708 .iter()
5709 .find(|g| g.answers_for(shape, edge.branch_name()))
5710 .expect("a guard per distinct branch, and the row names one of them");
5711 if let Err(e) = check_prepared(guard, edge).await {
5712 // Released before the rollback: a live statement on the connection
5713 // is what makes SQLite refuse to end a transaction.
5714 drop(guards);
5715 let _ = tx.rollback().await;
5716 return Err(e);
5717 }
5718 }
5719 drop(guards);
5720
5721 let stmt = tx.prepare(INSERT_LINK).await?;
5722
5723 for edge in edges {
5724 stmt.reset();
5725 let res = stmt.execute(edge_params(edge, stamp)).await;
5726
5727 if let Err(e) = res {
5728 let typed = classify(
5729 &tx,
5730 e,
5731 WriteOp::Edge {
5732 source_id: &edge.source,
5733 target_id: &edge.target,
5734 edge_type: &edge.edge_type,
5735 },
5736 )
5737 .await;
5738 // Released before the rollback: a live statement on the connection
5739 // is exactly what makes SQLite refuse to end a transaction.
5740 drop(stmt);
5741 let _ = tx.rollback().await;
5742 return Err(typed);
5743 }
5744 }
5745
5746 drop(stmt);
5747 tx.commit().await?;
5748 Ok(edges.len())
5749}
5750
5751/// Write every concept or none, under a single stamp.
5752/// Upsert one chunk of derived annotations in a single transaction (D-041).
5753///
5754/// `stamp` is the actor's clock reading, exactly as for every other chunk — but
5755/// it lands in `computed_at`, not in a `recorded_at`, and the difference is not
5756/// cosmetic. `recorded_at` is the transaction-time axis and is subject to
5757/// Doctrine II and the monotonicity guard; `computed_at` is a note about when a
5758/// derivation last ran, on a table the ledger does not see. Rerunning an
5759/// algorithm therefore replaces the row and advances the note, rather than
5760/// versioning a concept the world did not change.
5761///
5762/// # Failures name the concept (0.13.3, W7.2, D-176)
5763///
5764/// This was the one write path in the crate that returned
5765/// [`DbError::Engine`] raw, and the omission looked harmless: the table
5766/// carries no triggers, so none of [`crate::error::AbortKind`]'s guards can
5767/// fire on it and [`classify`] would have returned the same raw error it was
5768/// given. What that reasoning missed is the foreign key onto `concepts`, which
5769/// the engine enforces itself. Annotating a concept that does not exist is the
5770/// one failure a caller can cause here, and it reported as
5771/// `FOREIGN KEY constraint failed` with no row named — out of a chunk of up to
5772/// [`chunk_rows::ANNOTATIONS`].
5773///
5774/// It now goes through [`classify`] with [`WriteOp::Annotation`] like every
5775/// other write, and a missing concept returns [`DbError::NotFound`] carrying
5776/// its id.
5777async fn write_annotations_atomic(
5778 conn: &libsql::Connection,
5779 annotations: &[Annotation],
5780 stamp: &str,
5781) -> Result<usize> {
5782 if annotations.is_empty() {
5783 return Ok(0);
5784 }
5785
5786 let tx = conn
5787 .transaction_with_behavior(libsql::TransactionBehavior::Immediate)
5788 .await?;
5789
5790 let stmt = tx
5791 .prepare(
5792 "INSERT INTO analytics_annotations (concept_id, label, value, computed_at) \
5793 VALUES (?1, ?2, ?3, ?4) \
5794 ON CONFLICT(concept_id, label) DO UPDATE SET \
5795 value = excluded.value, computed_at = excluded.computed_at",
5796 )
5797 .await?;
5798
5799 for a in annotations {
5800 stmt.reset();
5801 let res = stmt
5802 .execute(libsql::params![
5803 a.concept_id.as_str(),
5804 a.label.as_str(),
5805 a.value.as_str(),
5806 stamp
5807 ])
5808 .await;
5809 if let Err(e) = res {
5810 let typed = classify(
5811 &tx,
5812 e,
5813 WriteOp::Annotation {
5814 concept_id: &a.concept_id,
5815 },
5816 )
5817 .await;
5818 drop(stmt);
5819 let _ = tx.rollback().await;
5820 return Err(typed);
5821 }
5822 }
5823
5824 drop(stmt);
5825 tx.commit().await?;
5826 Ok(annotations.len())
5827}
5828
5829async fn write_concepts_atomic(
5830 state: &mut ActorState,
5831 conn: &libsql::Connection,
5832 concepts: &[ConceptUpsert],
5833 stamp: &str,
5834) -> Result<usize> {
5835 if concepts.is_empty() {
5836 return Ok(0);
5837 }
5838
5839 // Named lineages, before the write lock — `check_lineages`' reason, and the
5840 // shape it also returns is unused here because `concepts` is keyed by
5841 // identity and has no resolution to do (see `ConceptUpsert::branch`).
5842 let mut named: Vec<&str> = Vec::with_capacity(1);
5843 for concept in concepts {
5844 let name = concept.branch_name();
5845 if !named.contains(&name) {
5846 named.push(name);
5847 }
5848 }
5849 check_lineages(state, conn, &named).await?;
5850
5851 let tx = conn
5852 .transaction_with_behavior(libsql::TransactionBehavior::Immediate)
5853 .await?;
5854
5855 // Prepared once, like the edge chunk (D-056). This no longer routes through
5856 // [`upsert_concept`] — that function prepares per call by construction — but
5857 // it shares that function's statement text and parameter row, so the two
5858 // cannot upsert different columns.
5859 let stmt = tx.prepare(UPSERT_CONCEPT).await?;
5860
5861 for concept in concepts {
5862 stmt.reset();
5863 let res = stmt.execute(concept_params(concept, stamp)).await;
5864
5865 if let Err(e) = res {
5866 let typed = classify(
5867 &tx,
5868 e,
5869 WriteOp::Concept {
5870 id: &concept.id,
5871 recorded_at: stamp,
5872 branch: concept.branch_name(),
5873 },
5874 )
5875 .await;
5876 drop(stmt);
5877 let _ = tx.rollback().await;
5878 return Err(typed);
5879 }
5880 }
5881
5882 drop(stmt);
5883 tx.commit().await?;
5884 Ok(concepts.len())
5885}
5886
5887#[cfg(test)]
5888mod lineage_cache {
5889 //! [`Lineages::shape_of`] against every shape combination (0.15.6, W14.3).
5890 //!
5891 //! Unit tests rather than a write through the actor, because the case this
5892 //! function exists for **cannot be observed from outside**: where a batch
5893 //! names lineages of different shapes, both of them currently compile the
5894 //! same overlap statement, so a wrong choice between them is invisible
5895 //! until W13.3 gives the guard a third lowering. A behavioural test would
5896 //! pass on the code this replaces and on the code that replaces it, and
5897 //! would go on passing through the release that made it matter.
5898
5899 use super::*;
5900
5901 /// `(name, is_root)`, kept as the fixture's spelling because that is what
5902 /// these tests are about — the shape, not the ancestry. A root is a row
5903 /// with no parent and no fork point, which is the pairing the `branches`
5904 /// CHECK enforces; a non-root is given both, since a row with one and not
5905 /// the other is not a state the schema permits.
5906 fn lineages(rows: &[(&str, bool)]) -> Lineages {
5907 Lineages {
5908 rows: rows
5909 .iter()
5910 .map(|(id, root)| crate::graph::lineage::BranchRow {
5911 id: (*id).into(),
5912 parent: (!root).then(|| "main".to_string()),
5913 forked_at: (!root).then(|| "2026-01-01T00:00:00.000000Z".to_string()),
5914 })
5915 .collect(),
5916 }
5917 }
5918
5919 fn trunk_only() -> Lineages {
5920 lineages(&[("main", true)])
5921 }
5922
5923 fn forked() -> Lineages {
5924 lineages(&[("main", true), ("alt", false), ("other", false)])
5925 }
5926
5927 /// One lineage is the trunk, whoever asks.
5928 #[test]
5929 fn one_lineage_is_the_trunk() {
5930 assert_eq!(
5931 trunk_only().shape_of(&["main"]).unwrap(),
5932 LineageShape::Trunk
5933 );
5934 }
5935
5936 /// A root on a forked database is `TrunkOnForked`; anything else resolves.
5937 #[test]
5938 fn the_shape_reads_the_name_and_not_only_the_count() {
5939 let l = forked();
5940 assert_eq!(
5941 l.shape_of(&["main"]).unwrap(),
5942 LineageShape::TrunkOnForked,
5943 "a root has no ancestors and D-244 emits that reduction directly"
5944 );
5945 assert_eq!(l.shape_of(&["alt"]).unwrap(), LineageShape::Resolved);
5946 }
5947
5948 /// The case review C-24 is about: names of two different shapes.
5949 ///
5950 /// Pinned in **both orders**, which is the whole content of the finding.
5951 /// The loop this replaces returned whichever came last, so a batch naming
5952 /// `[main, alt]` and one naming `[alt, main]` disagreed about their own
5953 /// shape while describing the same set of lineages.
5954 #[test]
5955 fn a_batch_of_mixed_shapes_resolves_rather_than_taking_the_last_name() {
5956 let l = forked();
5957 assert_eq!(
5958 l.shape_of(&["main", "alt"]).unwrap(),
5959 LineageShape::Resolved,
5960 "the resolved form is exact for a root as well, which is the \
5961 argument OverlapGuard::prepare already makes"
5962 );
5963 assert_eq!(
5964 l.shape_of(&["alt", "main"]).unwrap(),
5965 LineageShape::Resolved,
5966 "and it must not depend on the order the batch happened to name them"
5967 );
5968 }
5969
5970 /// Agreement is kept rather than widened: two forks are still `Resolved`,
5971 /// and two mentions of one root are still `TrunkOnForked`.
5972 #[test]
5973 fn names_that_agree_keep_their_shape() {
5974 let l = forked();
5975 assert_eq!(
5976 l.shape_of(&["alt", "other"]).unwrap(),
5977 LineageShape::Resolved
5978 );
5979 assert_eq!(
5980 l.shape_of(&["main", "main"]).unwrap(),
5981 LineageShape::TrunkOnForked
5982 );
5983 }
5984
5985 /// Existence is checked for **every** name, not only the one that decides.
5986 ///
5987 /// The check is why the loop existed at all, and the shape was the thing it
5988 /// returned on the way past. A version that stopped at the first name would
5989 /// let a batch name a lineage that does not exist and be refused later by
5990 /// the foreign key, from inside a rolled-back transaction, naming neither
5991 /// the column nor the branch — `check_lineages`' opening paragraph.
5992 #[test]
5993 fn an_unknown_name_is_refused_wherever_it_sits() {
5994 let l = forked();
5995 for names in [
5996 ["ghost", "main"].as_slice(),
5997 ["main", "ghost"].as_slice(),
5998 ["main", "ghost", "alt"].as_slice(),
5999 ] {
6000 match l.shape_of(names) {
6001 Err(DbError::UnknownBranch(name)) => assert_eq!(name, "ghost"),
6002 other => panic!("expected UnknownBranch for {names:?}, got {other:?}"),
6003 }
6004 }
6005 }
6006}
6007
6008#[cfg(test)]
6009mod tests {
6010 use super::*;
6011
6012 fn edge(target: &str, micros: usize) -> EdgeAssertion {
6013 EdgeAssertion::new("src", target, "LINKS")
6014 .valid_from(format!("2026-01-01T00:00:00.{micros:06}Z"))
6015 .valid_to(format!("2026-01-01T00:00:00.{:06}Z", micros + 1))
6016 }
6017
6018 /// The estimate must **no longer** depend on the batch's shape (0.13.6).
6019 ///
6020 /// Its dependence on shape was correct for as long as the guard was
6021 /// quadratic and the constant differed 16× between the two paths through
6022 /// its inner loop. W7.5 removed that term, and measurement agrees: 1.94 s
6023 /// and 2.22 s for the two 20,000-edge batches that used to differ by 7×.
6024 /// A model that kept predicting a 7× spread would now be wrong in the
6025 /// *expensive* direction — warning loudly about a batch that is fine.
6026 #[test]
6027 fn two_batches_of_one_size_are_predicted_alike() {
6028 const N: usize = 20_000;
6029 let fanout: Vec<_> = (0..N).map(|i| edge(&format!("t{i:07}"), i)).collect();
6030 let history: Vec<_> = (0..N).map(|i| edge("t0", i)).collect();
6031
6032 assert_eq!(
6033 estimated_bulk_hold(&fanout),
6034 estimated_bulk_hold(&history),
6035 "the guard no longer reads the batch's shape, so neither may this"
6036 );
6037 }
6038
6039 /// Measured on libSQL 0.9.30 after W7.5: 1.94 s and 2.22 s for those two
6040 /// batches, against 2.6 s and 18.1 s before it. This pins that the model
6041 /// still tracks them — a coefficient edited without re-measuring fails here.
6042 #[test]
6043 fn the_estimate_matches_what_was_measured() {
6044 const N: usize = 20_000;
6045 let fanout: Vec<_> = (0..N).map(|i| edge(&format!("t{i:07}"), i)).collect();
6046 let history: Vec<_> = (0..N).map(|i| edge("t0", i)).collect();
6047
6048 for (batch, measured_ms, label) in
6049 [(fanout, 1_936u128, "fanout"), (history, 2_220, "history")]
6050 {
6051 let predicted = estimated_bulk_hold(&batch).as_millis();
6052 let ratio = predicted as f64 / measured_ms as f64;
6053 assert!(
6054 (0.8..1.25).contains(&ratio),
6055 "{label}: predicted {predicted} ms against a measured \
6056 {measured_ms} ms ({ratio:.2}x). Re-run \
6057 examples/bulk_atomic_diag.rs before changing the coefficients."
6058 );
6059 }
6060 }
6061
6062 /// `ilog2` panics on zero, and an empty batch is the caller asking whether
6063 /// a batch they have not built yet would be slow.
6064 #[test]
6065 fn an_empty_batch_estimates_nothing_rather_than_panicking() {
6066 assert_eq!(estimated_bulk_hold(&[]), std::time::Duration::ZERO);
6067 let one = [edge("t0", 0)];
6068 assert_eq!(
6069 estimated_bulk_hold(&one),
6070 std::time::Duration::from_nanos(7_400)
6071 );
6072 }
6073
6074 /// The model is used as a threshold test, so it must not go backwards.
6075 #[test]
6076 fn a_bigger_batch_never_predicts_a_shorter_hold() {
6077 let mut last = std::time::Duration::ZERO;
6078 for n in [1usize, 2, 3, 7, 8, 100, 511, 512, 513, 5_000, 20_000] {
6079 let batch: Vec<_> = (0..n).map(|i| edge(&format!("t{i:07}"), i)).collect();
6080 let now = estimated_bulk_hold(&batch);
6081 assert!(now >= last, "{n} rows predicts {now:?} after {last:?}");
6082 last = now;
6083 }
6084 }
6085
6086 /// The warning threshold sits well above the bound this path is exempt from.
6087 ///
6088 /// Warning at `CHUNK_BUDGET` would fire on batches working exactly as
6089 /// designed — the exemption is a contract (D-014), not a failure — and a
6090 /// warning that fires on correct behaviour gets filtered out, taking the
6091 /// 18-second case with it.
6092 #[test]
6093 fn the_warning_threshold_is_not_the_chunk_budget() {
6094 assert!(BULK_ATOMIC_WARN_HOLD > CHUNK_BUDGET * 10);
6095 }
6096
6097 // -----------------------------------------------------------------------
6098 // reject_overlaps_within — sorted and swept (0.13.6, W7.5, D-179)
6099 //
6100 // The pairwise version was obviously correct and too slow; this one is
6101 // neither, so what follows pins the cases where the obvious fix is wrong
6102 // rather than only the cases the guard already caught.
6103 // -----------------------------------------------------------------------
6104
6105 /// An edge over an explicit interval, all four key columns spelled out.
6106 fn span(target: &str, edge_type: &str, from: usize, to: Option<usize>) -> EdgeAssertion {
6107 let stamp = |n: usize| format!("2026-01-01T00:00:00.{n:06}Z");
6108 EdgeAssertion::new("src", target, edge_type)
6109 .valid_from(stamp(from))
6110 .valid_to(to.map_or_else(|| timestamp::OPEN_SENTINEL.to_string(), stamp))
6111 }
6112
6113 fn closed(from: usize, to: usize) -> EdgeAssertion {
6114 span("t0", "LINKS", from, Some(to))
6115 }
6116
6117 fn open_at(from: usize) -> EdgeAssertion {
6118 span("t0", "LINKS", from, None)
6119 }
6120
6121 /// The case that makes adjacent pairs insufficient.
6122 ///
6123 /// Sort by start and any overlap shows up between neighbours — but only if
6124 /// every neighbouring pair is eligible to be checked. `[5,20)` and `[5,6)`
6125 /// are not: identical `valid_from` is re-assertion. Skip them, and `[5,6)`
6126 /// against `[7,8)` is a clean gap, and the plain overlap between `[5,20)`
6127 /// and `[7,8)` never gets looked at.
6128 #[test]
6129 fn an_overlap_hidden_behind_an_equal_valid_from_is_still_found() {
6130 let batch = vec![closed(5, 20), closed(5, 6), closed(7, 8)];
6131 assert!(matches!(
6132 reject_overlaps_within(&batch),
6133 Err(DbError::OverlappingInterval { .. })
6134 ));
6135 }
6136
6137 /// The same trap in the other direction: skipped for being open.
6138 ///
6139 /// Two open intervals are `trg_links_single_open`'s case and are passed
6140 /// over here. A running maximum that counted them would take the sentinel
6141 /// as the widest reach and report every later open interval as overlapping
6142 /// it — inventing an error rather than missing one, which is why the sweep
6143 /// carries a second maximum restricted to closed intervals.
6144 #[test]
6145 fn two_open_intervals_are_left_to_the_trigger() {
6146 let batch = vec![closed(1, 5), open_at(10), open_at(20)];
6147 assert!(reject_overlaps_within(&batch).is_ok());
6148 }
6149
6150 /// An open interval still overlaps a closed one that reaches past its start.
6151 #[test]
6152 fn an_open_interval_over_a_closed_one_is_an_overlap() {
6153 let batch = vec![closed(1, 50), open_at(10)];
6154 assert!(matches!(
6155 reject_overlaps_within(&batch),
6156 Err(DbError::OverlappingInterval { .. })
6157 ));
6158 }
6159
6160 /// Same `valid_from`, different `valid_to`: a batch correcting itself.
6161 ///
6162 /// Last writer wins by `seq_id`, exactly as it does across batches. The
6163 /// guard has no opinion.
6164 #[test]
6165 fn equal_valid_from_is_re_assertion_not_overlap() {
6166 let batch = vec![closed(5, 20), closed(5, 6), closed(5, 900)];
6167 assert!(reject_overlaps_within(&batch).is_ok());
6168 }
6169
6170 /// Grouping is what makes the sweep sound, so it is pinned rather than read.
6171 #[test]
6172 fn edges_with_different_keys_do_not_see_each_other() {
6173 let batch = vec![
6174 span("t0", "LINKS", 1, Some(50)),
6175 span("t1", "LINKS", 10, Some(60)),
6176 span("t0", "CITES", 10, Some(60)),
6177 span("t0", "LINKS", 50, Some(60)),
6178 ];
6179 assert!(reject_overlaps_within(&batch).is_ok());
6180 }
6181
6182 /// Which of the two the report calls *existing* (0.13.6).
6183 ///
6184 /// Neither is older in transaction time — a batch lands under one stamp —
6185 /// so the pairwise version's answer was its input order, which means
6186 /// nothing. Valid-time order is the only ordering the two intervals have.
6187 #[test]
6188 fn the_report_names_the_earlier_interval_as_the_existing_one() {
6189 let batch = vec![closed(7, 8), closed(5, 20)];
6190 let Err(DbError::OverlappingInterval { overlap }) = reject_overlaps_within(&batch) else {
6191 panic!("the batch overlaps itself");
6192 };
6193 assert!(overlap.valid_from.ends_with(".000007Z"), "{overlap:?}");
6194 assert!(overlap.existing_from.ends_with(".000005Z"), "{overlap:?}");
6195 }
6196
6197 /// A guard whose answer depended on the caller's ordering would be a worse
6198 /// guard than the one it replaced, and sorting is exactly the change that
6199 /// could introduce that.
6200 #[test]
6201 fn the_answer_does_not_depend_on_the_order_the_caller_passed() {
6202 let mut batch = vec![closed(5, 20), closed(5, 6), closed(7, 8)];
6203 batch.reverse();
6204 assert!(reject_overlaps_within(&batch).is_err());
6205
6206 let mut clean = vec![closed(1, 5), closed(5, 6), closed(7, 8), open_at(8)];
6207 clean.reverse();
6208 assert!(reject_overlaps_within(&clean).is_ok());
6209 }
6210
6211 /// §2.6, and the reason the rewrite happened rather than the doc alone.
6212 ///
6213 /// Every edge shares a key, so the old loop reached `Interval::overlaps`
6214 /// on all n(n−1)/2 pairs — 50 million of them here, which is seconds even
6215 /// in release and considerably worse in the debug profile this runs under.
6216 /// The bound is loose on purpose: it is an order of magnitude, not a
6217 /// benchmark, and the only thing it can fail on is the quadratic term
6218 /// coming back.
6219 #[test]
6220 fn one_relationships_whole_history_is_no_longer_quadratic() {
6221 const N: usize = 10_000;
6222 let batch: Vec<_> = (0..N).map(|i| closed(i * 2, i * 2 + 1)).collect();
6223
6224 let started = std::time::Instant::now();
6225 assert!(reject_overlaps_within(&batch).is_ok());
6226 let took = started.elapsed();
6227
6228 assert!(
6229 took < std::time::Duration::from_secs(2),
6230 "{N} same-key edges took {took:?} in the guard"
6231 );
6232 }
6233
6234 // -----------------------------------------------------------------------
6235 // next_chunk_size — the control law (0.12.0, W2)
6236 //
6237 // All of these run without a database, a clock or an actor, which is why
6238 // W2 comes before W3: the loop that will use this function can only be
6239 // tested against a real write, and the properties below cannot be observed
6240 // there without also observing the machine.
6241 // -----------------------------------------------------------------------
6242
6243 use std::time::Duration;
6244
6245 /// `next_chunk_size` with the shipped budget and floor.
6246 fn step_to(current: usize, held_ms: f64, ceiling: usize) -> usize {
6247 next_chunk_size(
6248 current,
6249 Duration::from_nanos((held_ms * 1_000_000.0) as u64),
6250 CHUNK_BUDGET,
6251 CHUNK_FLOOR,
6252 ceiling,
6253 )
6254 }
6255
6256 /// The edge path, which is every test here that does not say otherwise.
6257 fn step(current: usize, held_ms: f64) -> usize {
6258 step_to(current, held_ms, chunk_rows::EDGES)
6259 }
6260
6261 /// Iterate the law against a machine that costs `per_row_us` per row plus a
6262 /// fixed `overhead_ms` per transaction — the two-term model D-142 measured.
6263 fn converge(
6264 start: usize,
6265 per_row_us: f64,
6266 overhead_ms: f64,
6267 ceiling: usize,
6268 steps: usize,
6269 ) -> Vec<usize> {
6270 let mut size = start;
6271 (0..steps)
6272 .map(|_| {
6273 let held = overhead_ms + per_row_us * size as f64 / 1000.0;
6274 size = step_to(size, held, ceiling);
6275 size
6276 })
6277 .collect()
6278 }
6279
6280 /// The reason the shrink is proportional rather than a halving: at 4× over
6281 /// budget, halving needs three steps and every one of them is a latency
6282 /// miss a caller can feel.
6283 ///
6284 /// Run on the annotations path, because it is the only one whose ceiling
6285 /// leaves room to start far above a size that is reachable — on the edge
6286 /// path a 4× miss lands under [`CHUNK_FLOOR`], which is a different test.
6287 #[test]
6288 fn a_chunk_far_over_budget_converges_from_above_in_at_most_two_steps() {
6289 const CEILING: usize = chunk_rows::ANNOTATIONS;
6290 let (per_row_us, overhead_ms) = (20.0, 0.05);
6291 let held = |n: usize| overhead_ms + per_row_us * n as f64 / 1000.0;
6292 assert!(
6293 held(CEILING) > 4.0 * 3.0,
6294 "the start is not far over budget"
6295 );
6296
6297 let trace = converge(CEILING, per_row_us, overhead_ms, CEILING, 4);
6298 let first_in_budget = trace
6299 .iter()
6300 .position(|&n| held(n) <= 3.0)
6301 .expect("never reached the budget");
6302 assert!(
6303 first_in_budget <= 1,
6304 "took {} steps to get under budget: {trace:?}",
6305 first_in_budget + 1
6306 );
6307 }
6308
6309 /// Growth is additive, so a size that is merely comfortable cannot leap the
6310 /// ceiling — and cannot overshoot the budget by more than a quarter.
6311 #[test]
6312 fn growth_is_slow_and_shrinking_is_fast() {
6313 let grown = step(40, 1.0);
6314 assert!(
6315 (41..=50).contains(&grown),
6316 "40 rows at 1 ms should grow by about a quarter, got {grown}"
6317 );
6318 let shrunk = step(90, 9.0);
6319 assert!(
6320 shrunk <= 40,
6321 "90 rows at 3x the budget should shrink proportionally, got {shrunk}"
6322 );
6323 }
6324
6325 /// The dead band. Between `budget / 2` and `budget` the size is right and
6326 /// moving it only costs a re-measurement; without this the law oscillates
6327 /// across the bound forever.
6328 #[test]
6329 fn a_chunk_inside_the_band_is_left_alone() {
6330 for held_ms in [1.6, 2.0, 2.5, 2.9, 3.0] {
6331 assert_eq!(step(60, held_ms), 60, "moved at {held_ms} ms");
6332 }
6333 assert_ne!(step(60, 1.4), 60, "did not grow at well under half budget");
6334 }
6335
6336 /// Both clamps, and the floor's violation stated as a test rather than only
6337 /// as a comment: a populated table drives this to `CHUNK_FLOOR` and holds it
6338 /// there **over budget**, which is [`CHUNK_FLOOR`]'s documented trade.
6339 #[test]
6340 fn the_floor_and_the_ceiling_both_hold() {
6341 // 118 µs/row + 0.03 ms fixed — the populated arm, where 35 rows is
6342 // ~4.1 ms and no size in range meets the bound.
6343 let trace = converge(chunk_rows::EDGES, 118.0, 0.03, chunk_rows::EDGES, 8);
6344 assert!(
6345 trace.iter().all(|&n| n >= CHUNK_FLOOR),
6346 "fell through the floor: {trace:?}"
6347 );
6348 assert_eq!(*trace.last().unwrap(), CHUNK_FLOOR, "settled off the floor");
6349
6350 // A free machine cannot grow past the path's constant.
6351 let fast = converge(CHUNK_FLOOR, 1.0, 0.01, chunk_rows::EDGES, 40);
6352 assert_eq!(*fast.last().unwrap(), chunk_rows::EDGES);
6353 assert!(fast.iter().all(|&n| n <= chunk_rows::EDGES));
6354 }
6355
6356 /// Zero is the one answer that cannot be recovered from: a loop asked for
6357 /// chunks of no rows makes no progress and never finishes. Degenerate
6358 /// inputs included, since `held` is a measurement and measurements arrive
6359 /// from a machine under load.
6360 #[test]
6361 fn the_law_never_returns_zero() {
6362 let cases = [
6363 (0usize, Duration::ZERO),
6364 (0, Duration::from_secs(60)),
6365 (1, Duration::from_secs(60)),
6366 (90, Duration::from_secs(3600)),
6367 (usize::MAX, Duration::from_nanos(1)),
6368 (1, Duration::ZERO),
6369 ];
6370 for (current, held) in cases {
6371 for (floor, ceiling) in [(35, 90), (1, 1), (0, 0), (90, 35)] {
6372 let n = next_chunk_size(current, held, CHUNK_BUDGET, floor, ceiling);
6373 assert!(
6374 n > 0,
6375 "returned 0 for current={current}, held={held:?}, \
6376 floor={floor}, ceiling={ceiling}"
6377 );
6378 }
6379 }
6380 }
6381
6382 /// A zero budget is not a configuration anyone should reach, but it is one
6383 /// division away from a panic, so it is pinned.
6384 #[test]
6385 fn a_zero_budget_shrinks_to_the_floor_rather_than_dividing_by_it() {
6386 assert_eq!(
6387 next_chunk_size(90, Duration::from_millis(1), Duration::ZERO, 35, 90),
6388 35
6389 );
6390 }
6391
6392 /// A panicked write actor is reported, and the report says so (W7.3, D-177).
6393 ///
6394 /// This is the branch `close()` actually has. Through 0.13.3 it sat beside
6395 /// `Ok(res) => res?` on an actor `Result` that could never be `Err`, and the
6396 /// pair looked like two failure paths under review — so the one that cannot
6397 /// fire was carried through two signatures and the one that can had no test.
6398 ///
6399 /// The `JoinError` is real rather than mocked: `JoinError` has no public
6400 /// constructor, and one built by hand would pin the mapping against a value
6401 /// tokio does not produce.
6402 #[tokio::test]
6403 async fn a_writer_that_panicked_is_reported_by_close() {
6404 // Swallow the panic's own output. The task is *meant* to panic, and a
6405 // backtrace in a green suite trains people to skim it.
6406 let prev = std::panic::take_hook();
6407 std::panic::set_hook(Box::new(|_| {}));
6408 let handle = tokio::spawn(async { panic!("the write connection is gone") });
6409 let joined = handle.await;
6410 std::panic::set_hook(prev);
6411
6412 assert!(
6413 joined.is_err(),
6414 "the task must have panicked for this to test anything"
6415 );
6416
6417 match writer_exit(joined) {
6418 Err(DbError::WriterStopped(reason)) => {
6419 assert!(
6420 reason.contains("did not exit cleanly"),
6421 "the message must say what happened: {reason}"
6422 );
6423 }
6424 other => panic!("a panicked actor must be WriterStopped, got {other:?}"),
6425 }
6426 }
6427
6428 /// An actor that ran to completion closes clean.
6429 ///
6430 /// The other half, and the one that must not acquire a failure mode by
6431 /// accident: `run_writer_actor` returns `()`, so the only way this can start
6432 /// reporting an error is if someone gives the actor a `Result` again.
6433 #[tokio::test]
6434 async fn a_writer_that_finished_normally_closes_clean() {
6435 let handle = tokio::spawn(async {});
6436 assert!(writer_exit(handle.await).is_ok());
6437 }
6438
6439 /// A token is a handle to one flag, not a value that is copied (0.13.8,
6440 /// W7.6). The clone the caller keeps and the clone the import holds have to
6441 /// be the same flag, or `cancel()` reaches nothing.
6442 #[test]
6443 fn a_cloned_token_cancels_the_original() {
6444 let token = CancelToken::new();
6445 let held_by_the_import = token.clone();
6446 assert!(!held_by_the_import.is_cancelled());
6447 token.cancel();
6448 assert!(held_by_the_import.is_cancelled());
6449 // And it stays cancelled: there is no un-cancel, deliberately, because
6450 // a token that could be reset would let a second import inherit a
6451 // decision made about the first.
6452 token.cancel();
6453 assert!(held_by_the_import.is_cancelled());
6454 }
6455
6456 /// The default control is the one the plain bulk methods pass, and it must
6457 /// never stop a write.
6458 #[test]
6459 fn the_default_control_neither_cancels_nor_reports() {
6460 let control = BulkControl::new();
6461 assert!(!control.is_cancelled());
6462 // No callback, so this is a no-op rather than a panic on an `unwrap`.
6463 control.report(BulkProgress {
6464 written: 1,
6465 total: 1,
6466 rows: 1,
6467 held: std::time::Duration::ZERO,
6468 });
6469 }
6470
6471 /// The callback receives what it was promised, once per call to `report`.
6472 #[test]
6473 fn progress_reaches_the_callback_unchanged() {
6474 let seen = Arc::new(std::sync::Mutex::new(Vec::new()));
6475 let control = BulkControl::new().on_progress({
6476 let seen = Arc::clone(&seen);
6477 move |p| seen.lock().unwrap().push(p)
6478 });
6479 let sample = BulkProgress {
6480 written: 180,
6481 total: 900,
6482 rows: 90,
6483 held: std::time::Duration::from_millis(12),
6484 };
6485 control.report(sample);
6486 assert_eq!(*seen.lock().unwrap(), vec![sample]);
6487 }
6488}