Skip to main content

day_persistence/
lib.rs

1// Copyright © The Daybrite Project
2// SPDX-License-Identifier: MPL-2.0
3
4//! SQLite persistence for the observable model (docs/persistence.md).
5//!
6//! An app depends on this crate directly, beside the `day` facade with its `model` feature on,
7//! and starts a model module with `use day_persistence::prelude::*`. The crate is built on the
8//! framework's public API and nothing in the framework depends on it; `#[derive(Model)]` comes
9//! from day-persistence-macros and the web engine from day-sqlite-worker.
10//!
11//! The engine owns the data; memory holds a working set. A [`ModelContainer`] opens a
12//! database through a pluggable [`SqliteDriver`], creates or migrates each model's table,
13//! and stops: no rows are read at open, so opening a million-row store costs the same as
14//! opening an empty one. Rows enter memory by faulting: a typed [`ModelContainer::get`], a
15//! batch [`ModelContainer::ensure_resident`], or a list binding materializing the rows it is
16//! about to show. Each model's [`Store`] is that working set (an ordinary day-model store,
17//! so every binding and accessor works on it unchanged), bounded by a per-table cache limit;
18//! clean rows nothing observes leave silently and fault back on the next read.
19//!
20//! The write half is day-model's change log, folded. Any write requires its row resident
21//! (editing is what faulted it in); at the end of any turn that touched a store (autosave,
22//! the default) the accumulated changes fold into the smallest statement list that expresses
23//! them: twenty keystrokes into one field is one `UPDATE`; a row inserted and then filled is
24//! one `INSERT`; a thousand-child cascade is a handful of chunked `DELETE`s.
25//!
26//! Typed live queries ([`ModelContainer::query`]) compile entirely to SQL: predicate to
27//! `WHERE` (relation crossings as correlated `EXISTS`, full-text as an FTS5 subquery, spatial
28//! boxes through the R*Tree shadow), sort to `ORDER BY` with the key as tie-break, window to
29//! `LIMIT`, so the engine's indexes answer them at any table size. Live maintenance is
30//! dependency-gated: a change to a column no query mentions costs nothing; a change a query
31//! does depend on marks it stale, and one requery after the turn's flush re-derives its id
32//! set, diffed against the previous answer so a list animates the difference instead of
33//! reloading. Rows behind the ids stay lazy: the query holds ids, and the list faults the
34//! window it shows.
35//!
36//! The driver is a trait so the engine is the app's choice: the built-in [`Sqlite`] driver
37//! (feature `driver-rusqlite`, on by default) compiles a bundled SQLite, links the system one
38//! (`system`), or builds `SQLCipher` (`cipher`); the [`Recorder`] answers from fixtures and
39//! records every statement, which is what keeps persistence assertable headlessly: a test
40//! can check the SQL a UI action produced without a database on disk.
41//!
42//! Another connection's committed writes arrive through [`ModelContainer::check_external`],
43//! which re-reads only the resident rows and re-runs the live queries: O(working set), never
44//! O(table). Row identity is the model's `#[model(id)]` key; display order is a projection
45//! concern and is not persisted.
46
47use std::any::{Any, TypeId};
48use std::cell::{Cell, RefCell};
49use std::collections::HashMap;
50use std::fmt;
51use std::fmt::Write as _;
52use std::path::{Path, PathBuf};
53use std::rc::Rc;
54
55use day_model::Source;
56use day_model::{Identified, Key, Keyed, ModelId, Op, Store};
57
58#[cfg(not(target_arch = "wasm32"))]
59mod worker;
60#[cfg(not(target_arch = "wasm32"))]
61pub use worker::{
62    DatabaseWorker, WorkerOptions, WorkerRequest, WorkerSnapshot, WorkerSubscription,
63};
64
65mod queries;
66pub use queries::{
67    Col, Delta, Deps, Fetch, FtsRef, GeoRect, GeoRef, Pred, Quant, RelatedDep, RelationCol,
68    ResultSet, RowView, SetChange, Sort, compare_values, encode_column, rank,
69};
70use queries::{
71    CompileErr, FtsSql, RelSql, SqlIndex, compile_count, compile_fallback, compile_fetch,
72};
73
74mod blob;
75pub use blob::{Blob, BlobField, BlobReader};
76mod relations;
77pub use relations::{
78    DeleteRule, Many, One, Registrar, RelationDef, RelationRef, wire_join, wire_to_many,
79};
80
81#[cfg(feature = "driver-rusqlite")]
82mod rusqlite_driver;
83#[cfg(feature = "driver-rusqlite")]
84pub use rusqlite_driver::Sqlite;
85#[cfg(all(
86    feature = "driver-rusqlite",
87    target_family = "wasm",
88    target_os = "unknown"
89))]
90pub use rusqlite_driver::WebStorage;
91
92/// The derive for the [`Model`] trait (the two share a name, so `use day_persistence::Model`
93/// brings both): everything `#[derive(Observable)]` generates plus the schema half.
94/// `#[model(id)]` marks the key; docs/persistence.md lists the field and struct options.
95pub use day_persistence_macros::Model;
96
97// `#[derive(Model)]` expands its observable half to this (day-macros' `observable_impls!`),
98// reached by this crate's name so a model crate needs no day-macros dependency of its own.
99#[doc(hidden)]
100pub use day_macros::observable_impls as __observable_impls;
101
102/// What a model module starts from: `use day_persistence::prelude::*;` beside the app's `use
103/// day::prelude::*;`. The container, the `Model` trait and its derive, the relation markers,
104/// the drivers and `schema!`.
105pub mod prelude {
106    #[cfg(feature = "driver-rusqlite")]
107    pub use crate::Sqlite;
108    pub use crate::{DeleteRule, Many, Model, ModelContainer, One, Recorder, Secret, schema};
109}
110
111// ---------------------------------------------------------------------------
112// Errors
113// ---------------------------------------------------------------------------
114
115/// Why a persistence operation failed. `message` is for the log; `kind` is for the code path
116/// that can do something about it.
117#[derive(Clone, Debug)]
118pub struct DbError {
119    pub kind: DbErrorKind,
120    pub message: String,
121}
122
123#[derive(Clone, Copy, PartialEq, Eq, Debug)]
124pub enum DbErrorKind {
125    /// An asynchronous request was rejected because its worker has closed.
126    Closed,
127    /// The bounded worker queue has no room; the request was not accepted.
128    Busy,
129    /// A queued read was cancelled before it began.
130    Cancelled,
131    /// A worker operation panicked. The worker is closed and must be reopened.
132    Panicked,
133    /// The transaction committed, but refreshing resident models failed. Do not replay the
134    /// write. Recover the cache with `rescan` before editing; workers retry before their next job.
135    CommittedRefreshFailed,
136    /// The engine said no: SQL error, I/O, constraint.
137    Driver,
138    /// The stored schema and the declared one disagree in a way lightweight migration cannot
139    /// close (a type change, a rename); the file is refused, never silently rewritten.
140    Schema,
141    /// The database would not open with the key it was given.
142    BadKey,
143    /// A stored value would not decode as the field's type.
144    Decode,
145    /// The driver cannot do this (encryption on a non-cipher build, …).
146    Unsupported,
147    /// A `DeleteRule::Deny` relation refused the delete: children still reference the row.
148    Deny,
149}
150
151impl DbError {
152    pub fn new(kind: DbErrorKind, message: impl Into<String>) -> Self {
153        DbError {
154            kind,
155            message: message.into(),
156        }
157    }
158    /// Engine-error shorthand for driver implementations (public: external drivers use it too).
159    pub fn driver(message: impl Into<String>) -> Self {
160        Self::new(DbErrorKind::Driver, message)
161    }
162}
163
164impl fmt::Display for DbError {
165    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
166        write!(f, "{:?}: {}", self.kind, self.message)
167    }
168}
169
170impl std::error::Error for DbError {}
171
172// ---------------------------------------------------------------------------
173// Values and rows
174// ---------------------------------------------------------------------------
175
176/// SQLite's five storage classes, and nothing else; no driver type reaches the layer above.
177#[derive(Clone, Debug, PartialEq)]
178pub enum Value {
179    Null,
180    Int(i64),
181    Real(f64),
182    Text(String),
183    Blob(Vec<u8>),
184}
185
186impl Value {
187    /// The integer this value holds.
188    ///
189    /// # Errors
190    /// Returns a decoding error when the value is not an `INTEGER`.
191    pub fn as_int(&self) -> Result<i64, DbError> {
192        match self {
193            Value::Int(i) => Ok(*i),
194            other => Err(DbError::new(
195                DbErrorKind::Decode,
196                format!("expected INTEGER, found {other:?}"),
197            )),
198        }
199    }
200    /// The floating-point number this value holds.
201    ///
202    /// # Errors
203    /// Returns a decoding error when the value is not a `REAL`.
204    pub fn as_real(&self) -> Result<f64, DbError> {
205        match self {
206            Value::Real(r) => Ok(*r),
207            Value::Int(i) => Ok(*i as f64),
208            other => Err(DbError::new(
209                DbErrorKind::Decode,
210                format!("expected REAL, found {other:?}"),
211            )),
212        }
213    }
214    /// The text this value holds.
215    ///
216    /// # Errors
217    /// Returns a decoding error when the value is not `TEXT`.
218    pub fn as_text(&self) -> Result<&str, DbError> {
219        match self {
220            Value::Text(t) => Ok(t),
221            other => Err(DbError::new(
222                DbErrorKind::Decode,
223                format!("expected TEXT, found {other:?}"),
224            )),
225        }
226    }
227    /// The bytes this value holds.
228    ///
229    /// # Errors
230    /// Returns a decoding error when the value is not a `BLOB`.
231    pub fn as_blob(&self) -> Result<&[u8], DbError> {
232        match self {
233            Value::Blob(b) => Ok(b),
234            other => Err(DbError::new(
235                DbErrorKind::Decode,
236                format!("expected BLOB, found {other:?}"),
237            )),
238        }
239    }
240}
241
242/// A column's declared type. Part of the schema fingerprint, so changing one is a migration.
243#[derive(Clone, Copy, PartialEq, Eq, Debug)]
244pub enum SqlType {
245    Integer,
246    Real,
247    Text,
248    Blob,
249}
250
251impl SqlType {
252    pub fn ddl(self) -> &'static str {
253        match self {
254            SqlType::Integer => "INTEGER",
255            SqlType::Real => "REAL",
256            SqlType::Text => "TEXT",
257            SqlType::Blob => "BLOB",
258        }
259    }
260}
261
262/// One result row, by position.
263pub trait Row {
264    fn len(&self) -> usize;
265    fn get(&self, i: usize) -> Value;
266    fn is_empty(&self) -> bool {
267        self.len() == 0
268    }
269}
270
271impl Row for Vec<Value> {
272    fn len(&self) -> usize {
273        Vec::len(self)
274    }
275    fn get(&self, i: usize) -> Value {
276        self.as_slice().get(i).cloned().unwrap_or(Value::Null)
277    }
278}
279
280// ---------------------------------------------------------------------------
281// The driver interface
282// ---------------------------------------------------------------------------
283
284/// Where the database lives.
285#[derive(Clone, Debug)]
286pub enum Location {
287    Memory,
288    File(PathBuf),
289}
290
291/// A key for an encrypted database. Redacted in Debug and zeroed on drop; where it lives
292/// between launches is the app's business, never this crate's.
293#[derive(Clone)]
294pub struct Secret(String);
295
296impl Secret {
297    pub fn new(key: impl Into<String>) -> Self {
298        Secret(key.into())
299    }
300    #[cfg(feature = "cipher")]
301    pub(crate) fn reveal(&self) -> &str {
302        &self.0
303    }
304}
305
306impl fmt::Debug for Secret {
307    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
308        f.write_str("Secret(…)")
309    }
310}
311
312impl Drop for Secret {
313    fn drop(&mut self) {
314        // Best-effort scrub; the allocation may have been moved by earlier clones.
315        unsafe {
316            // SAFETY: we own the String and overwrite only its initialized bytes in place.
317            for b in self.0.as_bytes_mut() {
318                *b = 0;
319            }
320        }
321    }
322}
323
324/// How to open the database. Built by the driver's own constructors ([`Sqlite::at`] etc.);
325/// public so alternative drivers speak the same vocabulary.
326#[derive(Clone, Debug)]
327pub struct OpenOptions {
328    pub location: Location,
329    /// `SQLCipher` key; drivers without encryption reject it loudly.
330    pub key: Option<Secret>,
331    /// Journal in WAL mode (files only; ignored in memory). Default true.
332    pub wal: bool,
333    /// Enforce foreign keys. Default true.
334    pub foreign_keys: bool,
335    pub busy_timeout_ms: u32,
336    /// Upgrade a file written by an older `SQLCipher` generation (PRAGMA `cipher_migrate`).
337    pub cipher_migrate: bool,
338}
339
340impl OpenOptions {
341    pub fn new(location: Location) -> Self {
342        OpenOptions {
343            location,
344            key: None,
345            wal: true,
346            foreign_keys: true,
347            busy_timeout_ms: 5_000,
348            cipher_migrate: false,
349        }
350    }
351}
352
353/// What a driver can and cannot do, so the container degrades visibly instead of silently.
354#[derive(Clone, Copy, Debug, Default)]
355pub struct Capabilities {
356    /// Survives a relaunch (false for memory stores and the Recorder).
357    pub durable: bool,
358    pub encryption: bool,
359    pub wal: bool,
360    pub full_text_search: bool,
361    pub rtree: bool,
362    /// Another connection's committed writes are detectable (SQLite's `PRAGMA data_version`
363    /// counter), so [`ModelContainer::check_external`] can merge them. The built-in driver
364    /// claims it for file databases on native targets; memory databases (no second connection
365    /// can reach one), the Recorder, and the web engine (its OPFS access is exclusive) do not.
366    pub external_changes: bool,
367    /// The connection has `day_fold` registered (Rust's full-Unicode `to_lowercase` as a
368    /// scalar SQL function), so case-insensitive text predicates compile to exact SQL. The
369    /// built-in native driver registers it at open; without it those predicates take the
370    /// fallback path (SQL for everything else, the folding test re-checked in memory).
371    pub unicode_fold: bool,
372}
373
374/// What `PRAGMA wal_checkpoint(TRUNCATE)` answered ([`ModelContainer::checkpoint`]): the
375/// three columns of its status row. The engine reads the counts after the checkpoint, so a
376/// completed one answers `0` and `0` (the log was emptied); a busy one describes the log it
377/// had to leave behind. Both counts are `-1` when the database is not in WAL mode (a memory
378/// database, or `wal: false`).
379#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
380pub struct Checkpoint {
381    /// A reader or writer kept the checkpoint from completing within the busy timeout; the
382    /// WAL was folded as far as it could be and the rest waits for the next call.
383    pub busy: bool,
384    /// Frames in the WAL when the checkpoint finished.
385    pub log_frames: i64,
386    /// Frames of that log already carried into the main file.
387    pub checkpointed_frames: i64,
388}
389
390impl Checkpoint {
391    /// The whole log was folded and truncated.
392    pub fn is_complete(&self) -> bool {
393        !self.busy
394    }
395}
396
397/// One row of `PRAGMA foreign_key_check` ([`ModelContainer::foreign_key_check`]): a child
398/// row whose reference names no parent.
399#[derive(Clone, Debug, PartialEq, Eq)]
400pub struct ForeignKeyViolation {
401    /// The child table.
402    pub table: String,
403    /// The child row's rowid; `None` for a `WITHOUT ROWID` table.
404    pub rowid: Option<i64>,
405    /// The parent table the reference names.
406    pub parent: String,
407    /// The constraint's index in the child's `PRAGMA foreign_key_list`.
408    pub constraint: i64,
409}
410
411impl fmt::Display for ForeignKeyViolation {
412    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
413        match self.rowid {
414            Some(rowid) => write!(
415                f,
416                "foreign key: {} row {rowid} references missing {}",
417                self.table, self.parent
418            ),
419            None => write!(
420                f,
421                "foreign key: {} row references missing {}",
422                self.table, self.parent
423            ),
424        }
425    }
426}
427
428/// The engine trait. Object-safe on the connection side, so the container stores
429/// `Box<dyn SqliteConnection>` and never names an engine type.
430pub trait SqliteDriver {
431    type Connection: SqliteConnection;
432    /// Open the connection.
433    ///
434    /// # Errors
435    /// Returns the engine's error when the database cannot be opened.
436    fn open(self) -> Result<Self::Connection, DbError>;
437    fn capabilities(&self) -> Capabilities;
438}
439
440impl SqliteConnection for Box<dyn SqliteConnection> {
441    fn execute(&mut self, sql: &str, params: &[Value]) -> Result<u64, DbError> {
442        (**self).execute(sql, params)
443    }
444    fn query(
445        &mut self,
446        sql: &str,
447        params: &[Value],
448        row: &mut dyn FnMut(&dyn Row),
449    ) -> Result<(), DbError> {
450        (**self).query(sql, params, row)
451    }
452    fn execute_batch(&mut self, sql: &str) -> Result<(), DbError> {
453        (**self).execute_batch(sql)
454    }
455    fn query_named(
456        &mut self,
457        sql: &str,
458        params: &[Value],
459        row: &mut dyn FnMut(&[String], &dyn Row),
460    ) -> Result<(), DbError> {
461        (**self).query_named(sql, params, row)
462    }
463}
464
465/// One open connection. Everything above speaks plain SQL and [`Value`]s through it.
466pub trait SqliteConnection: 'static {
467    /// Run one statement and return the number of rows it changed.
468    ///
469    /// # Errors
470    /// Returns the engine's error when the statement fails.
471    fn execute(&mut self, sql: &str, params: &[Value]) -> Result<u64, DbError>;
472    /// Run one statement, handing each result row to `row`.
473    ///
474    /// # Errors
475    /// Returns the engine's error when the statement fails.
476    fn query(
477        &mut self,
478        sql: &str,
479        params: &[Value],
480        row: &mut dyn FnMut(&dyn Row),
481    ) -> Result<(), DbError>;
482    /// Begin a transaction.
483    ///
484    /// # Errors
485    /// Returns the engine's error when the statement fails.
486    fn begin(&mut self) -> Result<(), DbError> {
487        self.execute("BEGIN", &[]).map(|_| ())
488    }
489    /// Commit the open transaction.
490    ///
491    /// # Errors
492    /// Returns the engine's error when the statement fails.
493    fn commit(&mut self) -> Result<(), DbError> {
494        self.execute("COMMIT", &[]).map(|_| ())
495    }
496    /// Roll the open transaction back.
497    ///
498    /// # Errors
499    /// Returns the engine's error when the statement fails.
500    fn rollback(&mut self) -> Result<(), DbError> {
501        self.execute("ROLLBACK", &[]).map(|_| ())
502    }
503    /// Several statements in one string, no parameters: a DDL script, a migration step
504    /// (day-lite's storage speaks this). The default runs the string as one statement;
505    /// drivers whose engine executes scripts override it (the built-in native driver does).
506    ///
507    /// # Errors
508    /// Returns the engine's error of the first statement that fails.
509    fn execute_batch(&mut self, sql: &str) -> Result<(), DbError> {
510        self.execute(sql, &[]).map(|_| ())
511    }
512    /// [`SqliteConnection::query`], with the result's column names alongside each row, for
513    /// callers that surface rows as named objects (day-lite's JS bridge). A driver that
514    /// cannot name columns refuses rather than guessing.
515    ///
516    /// # Errors
517    /// Returns the engine's error when the statement fails, or an error from a driver that cannot
518    /// name columns.
519    fn query_named(
520        &mut self,
521        sql: &str,
522        params: &[Value],
523        row: &mut dyn FnMut(&[String], &dyn Row),
524    ) -> Result<(), DbError> {
525        let _ = (sql, params, row);
526        Err(DbError::new(
527            DbErrorKind::Unsupported,
528            "this driver does not report column names",
529        ))
530    }
531}
532
533// ---------------------------------------------------------------------------
534// The Recorder: the headless fake
535// ---------------------------------------------------------------------------
536
537/// A driver that answers from fixtures and records every statement: the piece that keeps
538/// persistence assertable with no database on disk. `let (driver, log) = Recorder::new();`
539///
540/// Fixtures answer by table, not by statement: any `SELECT … FROM <table>` serves every
541/// fixture row registered for it, whatever the `WHERE` clause says. Faulting paths keep only
542/// the keys they asked for, so `get` and `ensure_resident` behave; a query against the
543/// Recorder answers with all fixture keys, so assert the SQL it recorded, and use
544/// `Sqlite::memory()` where predicate results themselves are under test.
545pub struct Recorder {
546    state: Rc<RecorderState>,
547}
548
549struct RecorderState {
550    log: RefCell<Vec<(String, Vec<Value>)>>,
551    /// table name → rows served for a `SELECT … FROM <table>`.
552    fixtures: RefCell<HashMap<String, Vec<Vec<Value>>>>,
553}
554
555/// The recorded statement log, shared with the test that holds it.
556#[derive(Clone)]
557pub struct RecorderLog {
558    state: Rc<RecorderState>,
559}
560
561impl RecorderLog {
562    /// Every statement so far, SQL text only.
563    pub fn sql(&self) -> Vec<String> {
564        self.state
565            .log
566            .borrow()
567            .iter()
568            .map(|(s, _)| s.clone())
569            .collect()
570    }
571    /// Statements with their bound parameters.
572    pub fn entries(&self) -> Vec<(String, Vec<Value>)> {
573        self.state.log.borrow().clone()
574    }
575    pub fn clear(&self) {
576        self.state.log.borrow_mut().clear();
577    }
578}
579
580impl Recorder {
581    pub fn new() -> (Recorder, RecorderLog) {
582        let state = Rc::new(RecorderState {
583            log: RefCell::new(Vec::new()),
584            fixtures: RefCell::new(HashMap::new()),
585        });
586        (
587            Recorder {
588                state: state.clone(),
589            },
590            RecorderLog { state },
591        )
592    }
593
594    /// Rows a table's `SELECT` will answer with.
595    #[must_use]
596    pub fn with_table(self, table: &str, rows: Vec<Vec<Value>>) -> Self {
597        self.state.fixtures.borrow_mut().insert(table.into(), rows);
598        self
599    }
600}
601
602pub struct RecorderConn {
603    state: Rc<RecorderState>,
604}
605
606impl SqliteDriver for Recorder {
607    type Connection = RecorderConn;
608    fn open(self) -> Result<RecorderConn, DbError> {
609        Ok(RecorderConn { state: self.state })
610    }
611    fn capabilities(&self) -> Capabilities {
612        Capabilities {
613            durable: false,
614            wal: false,
615            // The fake answers everything: an FTS/spatial model must open against fixtures
616            // (its reads just come back empty unless a fixture answers them), and folded
617            // predicates compile as if day_fold existed; the SQL is recorded, never run.
618            full_text_search: true,
619            rtree: true,
620            unicode_fold: true,
621            ..Default::default()
622        }
623    }
624}
625
626impl SqliteConnection for RecorderConn {
627    fn execute(&mut self, sql: &str, params: &[Value]) -> Result<u64, DbError> {
628        self.state
629            .log
630            .borrow_mut()
631            .push((sql.to_string(), params.to_vec()));
632        Ok(0)
633    }
634    fn query(
635        &mut self,
636        sql: &str,
637        params: &[Value],
638        row: &mut dyn FnMut(&dyn Row),
639    ) -> Result<(), DbError> {
640        self.state
641            .log
642            .borrow_mut()
643            .push((sql.to_string(), params.to_vec()));
644        // Serve a fixture when the statement reads `FROM` a table one was registered for:
645        // quoted, as the container writes it, or plain, as a raw query might.
646        if let Some(from) = sql.split(" FROM ").nth(1) {
647            let table = quoted_name(from)
648                .unwrap_or_else(|| from.split_whitespace().next().unwrap_or("").to_string());
649            if let Some(rows) = self.state.fixtures.borrow().get(&table) {
650                for r in rows {
651                    row(r);
652                }
653            }
654        }
655        Ok(())
656    }
657}
658
659// ---------------------------------------------------------------------------
660// Column values and codecs (docs/persistence.md)
661// ---------------------------------------------------------------------------
662
663/// A Rust type that knows how to be a column. One impl per type: its canonical form.
664pub trait ColumnValue: Sized + 'static {
665    /// The declared column type. Part of the schema fingerprint, so changing it is a
666    /// migration, never a surprise.
667    const SQL_TYPE: SqlType;
668    fn to_sqlite_value(&self) -> Value;
669    /// Decode the stored value.
670    ///
671    /// # Errors
672    /// Returns a decoding error when `v` is not a value this type stores.
673    fn from_sqlite_value(v: Value) -> Result<Self, DbError>;
674}
675
676/// A named alternative representation for `T`, serde's `#[serde(with = …)]` idiom.
677/// Implement on a unit struct; select it per field with `#[model(with = …)]`.
678pub trait ValueCodec<T>: 'static {
679    const SQL_TYPE: SqlType;
680    fn to_sqlite_value(v: &T) -> Value;
681    /// Decode the stored value.
682    ///
683    /// # Errors
684    /// Returns a decoding error when `v` is not a value this codec stores.
685    fn from_sqlite_value(v: Value) -> Result<T, DbError>;
686}
687
688macro_rules! int_column {
689    ($($t:ty),*) => {$(
690        impl ColumnValue for $t {
691            const SQL_TYPE: SqlType = SqlType::Integer;
692            #[allow(
693                clippy::cast_lossless,
694                reason = "one macro body serves every integer width, and `From` exists for only some of them"
695            )]
696            fn to_sqlite_value(&self) -> Value {
697                Value::Int(*self as i64)
698            }
699            fn from_sqlite_value(v: Value) -> Result<Self, DbError> {
700                Ok(v.as_int()? as $t)
701            }
702        }
703    )*};
704}
705int_column!(i8, i16, i32, i64, u8, u16, u32, u64, usize, isize);
706
707impl ColumnValue for bool {
708    const SQL_TYPE: SqlType = SqlType::Integer;
709    fn to_sqlite_value(&self) -> Value {
710        Value::Int(i64::from(*self))
711    }
712    fn from_sqlite_value(v: Value) -> Result<Self, DbError> {
713        Ok(v.as_int()? != 0)
714    }
715}
716
717impl ColumnValue for f64 {
718    const SQL_TYPE: SqlType = SqlType::Real;
719    fn to_sqlite_value(&self) -> Value {
720        Value::Real(*self)
721    }
722    fn from_sqlite_value(v: Value) -> Result<Self, DbError> {
723        v.as_real()
724    }
725}
726
727impl ColumnValue for f32 {
728    const SQL_TYPE: SqlType = SqlType::Real;
729    fn to_sqlite_value(&self) -> Value {
730        Value::Real(f64::from(*self))
731    }
732    fn from_sqlite_value(v: Value) -> Result<Self, DbError> {
733        Ok(v.as_real()? as f32)
734    }
735}
736
737impl ColumnValue for String {
738    const SQL_TYPE: SqlType = SqlType::Text;
739    fn to_sqlite_value(&self) -> Value {
740        Value::Text(self.clone())
741    }
742    fn from_sqlite_value(v: Value) -> Result<Self, DbError> {
743        Ok(v.as_text()?.to_string())
744    }
745}
746
747impl ColumnValue for Vec<u8> {
748    const SQL_TYPE: SqlType = SqlType::Blob;
749    fn to_sqlite_value(&self) -> Value {
750        Value::Blob(self.clone())
751    }
752    fn from_sqlite_value(v: Value) -> Result<Self, DbError> {
753        Ok(v.as_blob()?.to_vec())
754    }
755}
756
757/// A Uuid is a 16-byte `BLOB`: compact, indexable, and readable by any tool that knows the
758/// convention. (`day_model::Uuid` is `uuid::Uuid`, so no second dependency edge exists.)
759impl ColumnValue for day_model::Uuid {
760    const SQL_TYPE: SqlType = SqlType::Blob;
761    fn to_sqlite_value(&self) -> Value {
762        Value::Blob(self.as_bytes().to_vec())
763    }
764    fn from_sqlite_value(v: Value) -> Result<Self, DbError> {
765        day_model::Uuid::from_slice(v.as_blob()?)
766            .map_err(|e| DbError::new(DbErrorKind::Decode, format!("uuid: {e}")))
767    }
768}
769
770/// A key handle, as the bound parameter its stored form takes: `INTEGER` for integer keys,
771/// a 16-byte `BLOB` for Uuid keys, `TEXT` for string keys. `Pair` keys (join rows) bind
772/// through their own two-column clause, never through this.
773fn key_param(handle: u64) -> Value {
774    match Key::of_handle(handle) {
775        Some(Key::U64(k)) => Value::Int(k as i64),
776        Some(Key::Uuid(u)) => Value::Blob(day_model::Uuid::from_u128(u).as_bytes().to_vec()),
777        Some(Key::Str(s)) => Value::Text(s.to_string()),
778        Some(Key::Pair(..)) | None => Value::Null,
779    }
780}
781
782/// A stored key value, back as its path handle. `None` for shapes no key takes (a negative
783/// integer, a blob that is not 16 bytes); the caller treats those rows as unaddressable.
784fn value_to_handle(v: &Value) -> Option<u64> {
785    match v {
786        Value::Int(i) if *i >= 0 => Some(Key::U64(*i as u64).handle()),
787        Value::Blob(b) if b.len() == 16 => day_model::Uuid::from_slice(b)
788            .ok()
789            .map(|u| Key::Uuid(u.as_u128()).handle()),
790        Value::Text(t) => Some(Key::Str(t.as_str().into()).handle()),
791        _ => None,
792    }
793}
794
795/// `NULL` belongs to the framework: `Option` wraps any column type, encoding is called only on
796/// present values, and a NULL decodes to `None`, so an impl never sees `Null` and cannot
797/// disagree about it.
798impl<T: ColumnValue> ColumnValue for Option<T> {
799    const SQL_TYPE: SqlType = T::SQL_TYPE;
800    fn to_sqlite_value(&self) -> Value {
801        match self {
802            Some(v) => v.to_sqlite_value(),
803            None => Value::Null,
804        }
805    }
806    fn from_sqlite_value(v: Value) -> Result<Self, DbError> {
807        match v {
808            Value::Null => Ok(None),
809            v => Ok(Some(T::from_sqlite_value(v)?)),
810        }
811    }
812}
813
814/// The serde codec: any `Serialize + Deserialize` type as a `TEXT` column SQLite's own
815/// `json_*` functions can reach. `#[model(json)]` is sugar for `#[model(with = Json)]`.
816pub struct Json;
817
818impl<T> ValueCodec<T> for Json
819where
820    T: serde::Serialize + serde::de::DeserializeOwned + 'static,
821{
822    const SQL_TYPE: SqlType = SqlType::Text;
823    fn to_sqlite_value(v: &T) -> Value {
824        match serde_json::to_string(v) {
825            Ok(s) => Value::Text(s),
826            // Serialization of an in-memory value failing is a bug in the type's serde impl;
827            // surface it as a NULL rather than a panic, and the decode side will report it.
828            Err(_) => Value::Null,
829        }
830    }
831    fn from_sqlite_value(v: Value) -> Result<T, DbError> {
832        serde_json::from_str(v.as_text()?)
833            .map_err(|e| DbError::new(DbErrorKind::Decode, format!("json: {e}")))
834    }
835}
836
837// ---------------------------------------------------------------------------
838// The Model trait: what #[derive(Model)] implements
839// ---------------------------------------------------------------------------
840
841/// One column of a model's table, as the derive declares it. `field` is the struct field the
842/// column stores: the change log speaks field names, the SQL speaks column names, and this is
843/// where the two meet (they differ only under `#[model(column = "…")]`).
844#[derive(Clone, Copy, Debug)]
845pub struct ColumnDef {
846    pub name: &'static str,
847    pub field: &'static str,
848    pub sql: SqlType,
849    pub not_null: bool,
850    pub unique: bool,
851    pub indexed: bool,
852    /// A [`Blob`] column: never read by a row fault, written only when it carries pending
853    /// bytes, served on demand through the field's accessor.
854    pub lazy: bool,
855}
856
857/// The column pair a `#[model(spatial(lat = …, lon = …))]` declaration names.
858#[derive(Clone, Copy, Debug, PartialEq, Eq)]
859pub struct SpatialCols {
860    pub lat: &'static str,
861    pub lon: &'static str,
862}
863
864/// One `#[model(link(…))]` as the derive emits it: `owner.local = target.remote` is the join.
865#[derive(Clone, Copy, Debug)]
866pub struct LinkDef {
867    /// The marker field the link was declared on: the name `Owner::field()` answers to.
868    pub field: &'static str,
869    /// The owner's column holding the value.
870    pub local: &'static str,
871    /// The target's (qualified) table.
872    pub target_table: &'static str,
873    /// The target's column the value names.
874    pub remote: &'static str,
875}
876
877/// The marker a `#[model(link(…))]` field carries. It has neither a column nor state; the
878/// type is what names the far model for the derive.
879pub struct Linked<M: ?Sized> {
880    _m: std::marker::PhantomData<fn() -> M>,
881}
882
883impl<M: ?Sized> Clone for Linked<M> {
884    fn clone(&self) -> Self {
885        *self
886    }
887}
888impl<M: ?Sized> Copy for Linked<M> {}
889impl<M: ?Sized> Default for Linked<M> {
890    fn default() -> Self {
891        Linked {
892            _m: std::marker::PhantomData,
893        }
894    }
895}
896impl<M: ?Sized> PartialEq for Linked<M> {
897    fn eq(&self, _: &Self) -> bool {
898        true
899    }
900}
901impl<M: ?Sized> std::fmt::Debug for Linked<M> {
902    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
903        f.write_str("Linked")
904    }
905}
906
907/// A wired link, as the SQL compiler and the query watcher see it.
908#[derive(Clone, Copy, Debug)]
909pub(crate) struct LinkInfo {
910    pub(crate) owner_table: &'static str,
911    pub(crate) field: &'static str,
912    pub(crate) local: &'static str,
913    pub(crate) target_table: &'static str,
914    pub(crate) remote: &'static str,
915}
916
917/// A persistable model: an [`Identified`] observable struct with a schema half. Implemented by
918/// `#[derive(Model)]`; the container consumes it.
919pub trait Model: Identified + day_model::ApplyField + Clone + 'static {
920    const TABLE: &'static str;
921    /// The key column's name (the `#[model(id)]` field). Also the first entry of `COLUMNS`.
922    const KEY: &'static str;
923    const COLUMNS: &'static [ColumnDef];
924    /// Composite indexes from struct-level `#[model(index("a", "b"))]`.
925    const COMPOSITE_INDEXES: &'static [&'static [&'static str]] = &[];
926    /// `#[model(unique("a", "b"))]` on the struct: multi-column UNIQUE constraints, kept as
927    /// unique indexes (`day_uidx_{table}_{a}_{b}`), part of the fingerprint and of the
928    /// lightweight migration.
929    const UNIQUE_INDEXES: &'static [&'static [&'static str]] = &[];
930    /// Full-text-indexed columns from struct-level `#[model(fts("a", "b"))]`: an
931    /// external-content FTS5 shadow table plus sync triggers, generated at open.
932    const FTS_COLUMNS: &'static [&'static str] = &[];
933    /// The FTS5 tokenizer, from `#[model(fts(…, tokenize = "…"))]`, e.g.
934    /// `"unicode61 remove_diacritics 2"` for diacritics-insensitive search. `None` keeps
935    /// FTS5's default. Part of the schema fingerprint, so changing it rebuilds the shadow.
936    const FTS_TOKENIZE: Option<&'static str> = None;
937    /// The R*Tree pair from struct-level `#[model(spatial(lat = "…", lon = "…"))]`.
938    const SPATIAL: Option<SpatialCols> = None;
939    /// The key column's SQL shape: what a `One<Self>` foreign-key column stores.
940    const KEY_SQL: SqlType = SqlType::Integer;
941    /// Relations declared on this model's `Many` fields (`#[model(relation(…))]`).
942    const RELATIONS: &'static [RelationDef] = &[];
943    /// Relations by value (`#[model(link(…))]`): rows of another model whose `remote` column
944    /// equals this row's `local` column, with no foreign key: the shape that reaches across an
945    /// attached database (see [`ModelContainer::attach_database`]).
946    const LINKS: &'static [LinkDef] = &[];
947    /// The fields declared as [`Blob`]: the columns a fault names as `NULL` and the fold
948    /// writes only when they carry pending bytes.
949    const LAZY_FIELDS: &'static [&'static str] = &[];
950    /// After a flush wrote this row: hand over every lazy field's pending bytes and mark
951    /// them stored. The derive generates it for models with [`Blob`] fields.
952    fn settle_lazy(&mut self) -> Vec<(&'static str, std::sync::Arc<Vec<u8>>)> {
953        Vec::new()
954    }
955    /// Declared `external = "alias"`: the table belongs to a database attached under that alias
956    /// and managed elsewhere. The container creates, migrates, and fingerprints nothing for
957    /// it, and treats it as read-only.
958    const EXTERNAL: bool = false;
959    /// The alias an `EXTERNAL` table's database is attached under, the part `TABLE` is
960    /// qualified with (`catalog` in `catalog.stations`). Kept apart so SQL can quote each
961    /// half; `None` for a table in the container's own file.
962    const SCHEMA: Option<&'static str> = None;
963    /// One [`Value`] per column, in `COLUMNS` order.
964    fn to_row(&self) -> Vec<Value>;
965    /// Decode one row, its columns in `COLUMNS` order.
966    ///
967    /// # Errors
968    /// Returns a decoding error when a column holds a value its field cannot take.
969    fn from_row(row: &dyn Row) -> Result<Self, DbError>;
970    /// Each column's Rust-default value: what an added column backfills with.
971    fn default_row() -> Vec<Value>;
972    /// Wire this model's declared relations. Generated; the default declares none.
973    fn wire(reg: &mut Registrar<'_>) {
974        let _ = reg;
975    }
976}
977
978/// `name` as a SQL identifier: double-quoted, an embedded `"` doubled. Quoted, any name
979/// reaches the engine as itself: a keyword (`order`, `group`), a space, a quote, an emoji.
980/// Every statement the container writes names its tables and columns this way, and raw SQL
981/// written against them should too.
982pub fn ident(name: &str) -> String {
983    let mut out = String::with_capacity(name.len() + 2);
984    out.push('"');
985    for ch in name.chars() {
986        if ch == '"' {
987            out.push('"');
988        }
989        out.push(ch);
990    }
991    out.push('"');
992    out
993}
994
995/// A table's name in the two halves SQL quotes separately: the attached database's alias,
996/// if any, and the table's own name. [`Model::TABLE`] (`notes`, `catalog.stations`) stays the
997/// plain key every lookup, fingerprint and `_day_schema` row uses; this is what a statement
998/// gets.
999#[derive(Clone, Copy)]
1000pub(crate) struct TableName {
1001    pub(crate) schema: Option<&'static str>,
1002    pub(crate) bare: &'static str,
1003}
1004
1005impl TableName {
1006    pub(crate) fn of<M: Model>() -> Self {
1007        let bare = match M::SCHEMA {
1008            Some(schema) => M::TABLE
1009                .strip_prefix(schema)
1010                .and_then(|rest| rest.strip_prefix('.'))
1011                .unwrap_or(M::TABLE),
1012            None => M::TABLE,
1013        };
1014        TableName {
1015            schema: M::SCHEMA,
1016            bare,
1017        }
1018    }
1019
1020    /// A table in the container's own file, named directly (a join table).
1021    pub(crate) fn plain(bare: &'static str) -> Self {
1022        TableName { schema: None, bare }
1023    }
1024
1025    /// The quoted SQL that names the table in a statement: `"notes"`, `"catalog"."stations"`.
1026    pub(crate) fn sql(&self) -> String {
1027        table_sql(self.schema, self.bare)
1028    }
1029
1030    /// The quoted SQL naming a shadow beside the table (`_fts`, `_geo`), in the same schema.
1031    pub(crate) fn shadow(&self, suffix: &str) -> String {
1032        table_sql(self.schema, &format!("{}{suffix}", self.bare))
1033    }
1034}
1035
1036pub(crate) fn table_sql(schema: Option<&str>, bare: &str) -> String {
1037    match schema {
1038        Some(schema) => format!("{}.{}", ident(schema), ident(bare)),
1039        None => ident(bare),
1040    }
1041}
1042
1043/// The declared schema's fingerprint: table, columns, types, flags, indexes. Equal fingerprints
1044/// open instantly; a difference migrates or refuses, never silently.
1045pub fn model_fingerprint<M: Model>() -> u64 {
1046    let mut h: u64 = 0xcbf2_9ce4_8422_2325;
1047    let mut eat = |bytes: &[u8]| {
1048        for b in bytes {
1049            h ^= u64::from(*b);
1050            h = h.wrapping_mul(0x1000_0000_01b3);
1051        }
1052    };
1053    // The DDL generation this fingerprint vouches for. Bumped when what the file must look
1054    // like changes (constrained column additions, unique indexes), so a fingerprint an older
1055    // build recorded no longer matches and the file gets one comparison pass, which rebuilds
1056    // what the older path left unconstrained and records the new fingerprint.
1057    eat(b"ddl-v2|");
1058    eat(M::TABLE.as_bytes());
1059    eat(M::KEY.as_bytes());
1060    for c in M::COLUMNS {
1061        eat(c.name.as_bytes());
1062        eat(c.sql.ddl().as_bytes());
1063        eat(&[
1064            u8::from(c.not_null),
1065            u8::from(c.unique),
1066            u8::from(c.indexed),
1067        ]);
1068    }
1069    for idx in M::COMPOSITE_INDEXES {
1070        for c in *idx {
1071            eat(c.as_bytes());
1072        }
1073        eat(b"|");
1074    }
1075    for idx in M::UNIQUE_INDEXES {
1076        eat(b"uniq:");
1077        for c in *idx {
1078            eat(c.as_bytes());
1079        }
1080        eat(b"|");
1081    }
1082    for c in M::FTS_COLUMNS {
1083        eat(b"fts:");
1084        eat(c.as_bytes());
1085    }
1086    if !M::FTS_COLUMNS.is_empty()
1087        && M::COLUMNS
1088            .iter()
1089            .any(|c| c.name == M::KEY && c.sql != SqlType::Integer)
1090    {
1091        eat(b"wide-fts-key-map-v1");
1092    }
1093    if let Some(t) = M::FTS_TOKENIZE {
1094        eat(b"tok:");
1095        eat(t.as_bytes());
1096    }
1097    if let Some(s) = M::SPATIAL {
1098        eat(b"geo:");
1099        eat(s.lat.as_bytes());
1100        eat(s.lon.as_bytes());
1101    }
1102    h
1103}
1104
1105// ---------------------------------------------------------------------------
1106// Schema sets and migrations
1107// ---------------------------------------------------------------------------
1108
1109/// The set of models a container manages; build with [`schema!`].
1110#[derive(Default)]
1111pub struct Schema {
1112    installers: Vec<Installer>,
1113    /// Run after every table is attached, because relation wiring needs both ends present.
1114    wirers: Vec<Installer>,
1115    /// Foreign-key clauses each declared relation contributes to its target's table.
1116    fk_specs: Vec<FkSpec>,
1117}
1118
1119type Installer = Box<dyn FnOnce(&ModelContainer) -> Result<(), DbError>>;
1120/// Decode raw rows into the cache silently, answering their handles.
1121type SettleFn = Rc<dyn Fn(&[u64]) -> Vec<(u64, &'static str, std::sync::Arc<Vec<u8>>)>>;
1122type AbsorbFn = Rc<dyn Fn(Vec<Vec<Value>>) -> Result<Vec<u64>, DbError>>;
1123/// Diff raw rows (selected by the resident keys) against the cache: per-field announcements
1124/// authored [`ModelContainer::EXTERNAL_AUTHOR`], resident rows missing from the input deleted
1125/// the same way. Returns whether anything differed.
1126type RefreshFn = Rc<dyn Fn(Vec<Vec<Value>>) -> Result<bool, DbError>>;
1127/// A `WHERE` clause plus its parameters, addressing one row by handle.
1128type KeyWhere = Rc<dyn Fn(u64) -> (String, Vec<Value>)>;
1129/// A `WHERE` clause plus its parameters, addressing a set of rows by handle.
1130type KeysWhere = Rc<dyn Fn(&[u64]) -> (String, Vec<Value>)>;
1131/// Evict up to N clean rows from an ordered candidate list, answering how many left.
1132type EvictFn = Rc<dyn Fn(&[u64], usize) -> usize>;
1133
1134/// One foreign-key clause, resolved from a `RelationDef` at `Schema::with` time: the child's
1135/// column carries `REFERENCES parent(key) ON DELETE …` in its generated DDL.
1136#[derive(Clone, Debug)]
1137struct FkSpec {
1138    child_table: &'static str,
1139    child_field: &'static str,
1140    parent_table: &'static str,
1141    /// `parent_table` as the `REFERENCES` clause names it.
1142    parent_sql: String,
1143    /// `parent_table` as `PRAGMA foreign_key_list` reports it: the bare, unquoted name.
1144    parent_bare: &'static str,
1145    parent_key: &'static str,
1146    rule: DeleteRule,
1147}
1148
1149impl Schema {
1150    pub fn new() -> Self {
1151        Self::default()
1152    }
1153    #[must_use]
1154    pub fn with<M: Model>(mut self) -> Self {
1155        for def in M::RELATIONS {
1156            if def.join.is_none() {
1157                self.fk_specs.push(FkSpec {
1158                    child_table: def.target_table,
1159                    child_field: def.inverse,
1160                    parent_table: M::TABLE,
1161                    parent_sql: TableName::of::<M>().sql(),
1162                    parent_bare: TableName::of::<M>().bare,
1163                    parent_key: M::KEY,
1164                    rule: def.delete,
1165                });
1166            }
1167        }
1168        self.installers.push(Box::new(ModelContainer::attach::<M>));
1169        self.wirers.push(Box::new(|c| {
1170            let mut reg = Registrar {
1171                container: c,
1172                error: None,
1173            };
1174            M::wire(&mut reg);
1175            match reg.error {
1176                Some(e) => Err(e),
1177                None => Ok(()),
1178            }
1179        }));
1180        self
1181    }
1182}
1183
1184/// `schema![Trip, Lodging]`: the models a container manages.
1185#[macro_export]
1186macro_rules! schema {
1187    ($($m:ty),+ $(,)?) => {{
1188        let s = $crate::Schema::new();
1189        $(let s = s.with::<$m>();)+
1190        s
1191    }};
1192}
1193
1194/// Staged migrations: anything with intent behind it (a rename, a split) gets a stage the app
1195/// writes; each stage runs in its own transaction against the raw connection, and versions only
1196/// go up. Lightweight migration (added/dropped columns and indexes) runs afterward, inferred
1197/// from the schema fingerprint.
1198#[derive(Default)]
1199pub struct MigrationPlan {
1200    stages: Vec<Stage>,
1201}
1202
1203type StageFn = Box<dyn FnOnce(&mut dyn SqliteConnection) -> Result<(), DbError>>;
1204
1205struct Stage {
1206    from: u32,
1207    to: u32,
1208    run: StageFn,
1209}
1210
1211impl MigrationPlan {
1212    pub fn new() -> Self {
1213        Self::default()
1214    }
1215    /// A hand-written stage taking the database from `from` to `to`. Stages run in ascending
1216    /// order from the file's current version; a gap is an error at open.
1217    #[must_use]
1218    pub fn custom(
1219        mut self,
1220        from: u32,
1221        to: u32,
1222        f: impl FnOnce(&mut dyn SqliteConnection) -> Result<(), DbError> + 'static,
1223    ) -> Self {
1224        self.stages.push(Stage {
1225            from,
1226            to,
1227            run: Box::new(f),
1228        });
1229        self
1230    }
1231    fn target_version(&self) -> u32 {
1232        self.stages.iter().map(|s| s.to).max().unwrap_or(0)
1233    }
1234}
1235
1236// ---------------------------------------------------------------------------
1237// Dirty tracking: the fold, applied live
1238// ---------------------------------------------------------------------------
1239
1240/// What one row needs at the next flush. The merge rules are the change-log fold: same-row
1241/// changes coalesce, an INSERT absorbs later field writes, a DELETE absorbs everything.
1242#[derive(Clone, Debug, PartialEq)]
1243pub(crate) enum DirtyRow {
1244    Insert,
1245    Update(Vec<&'static str>),
1246    Delete,
1247}
1248
1249#[derive(Clone, Default)]
1250struct DirtyState {
1251    /// (store, key) → pending statement kind, in first-touch order.
1252    rows: HashMap<(u64, u64), DirtyRow>,
1253    order: Vec<(u64, u64)>,
1254    /// store → its dirty keys, so a relation's mid-turn overlay reads one store's rows
1255    /// instead of scanning every store's.
1256    by_store: HashMap<u64, Vec<u64>>,
1257    /// Stores whose whole value was rewritten (a wholesale `Store::update`), flushed by
1258    /// upserting every resident row. With a lazy cache no "delete the rest" is possible (the
1259    /// rest was never loaded), so wholesale rewrites are a resync of the working set only.
1260    full: Vec<u64>,
1261    /// (store, key) → the distinct author tags of the changes folded into that row, in
1262    /// first-seen order; the unauthored user edit (`None`) leaves no tag.
1263    authors: HashMap<(u64, u64), Vec<&'static str>>,
1264    /// store → the author tags of its wholesale rewrites.
1265    store_authors: HashMap<u64, Vec<&'static str>>,
1266}
1267
1268/// Record one author tag once, in first-seen order; `None` (the user) is not a tag.
1269fn add_author(tags: &mut Vec<&'static str>, author: Option<&'static str>) {
1270    if let Some(author) = author
1271        && !tags.contains(&author)
1272    {
1273        tags.push(author);
1274    }
1275}
1276
1277impl DirtyState {
1278    fn merge(&mut self, newer: Self) {
1279        for store in newer.full {
1280            if !self.full.contains(&store) {
1281                self.full.push(store);
1282            }
1283        }
1284        for (store, tags) in newer.store_authors {
1285            let mine = self.store_authors.entry(store).or_default();
1286            for tag in tags {
1287                add_author(mine, Some(tag));
1288            }
1289        }
1290        for (id, tags) in newer.authors {
1291            let mine = self.authors.entry(id).or_default();
1292            for tag in tags {
1293                add_author(mine, Some(tag));
1294            }
1295        }
1296        for id in newer.order {
1297            let next = newer.rows[&id].clone();
1298            if !self.rows.contains_key(&id) {
1299                self.order.push(id);
1300                self.by_store.entry(id.0).or_default().push(id.1);
1301            }
1302            match (self.rows.get_mut(&id), next) {
1303                (Some(DirtyRow::Update(old)), DirtyRow::Update(cols)) => {
1304                    for col in cols {
1305                        if !old.contains(&col) {
1306                            old.push(col);
1307                        }
1308                    }
1309                }
1310                (Some(DirtyRow::Insert | DirtyRow::Delete), DirtyRow::Update(_)) => {}
1311                (_, next) => {
1312                    self.rows.insert(id, next);
1313                }
1314            }
1315        }
1316    }
1317
1318    fn is_empty(&self) -> bool {
1319        self.rows.is_empty() && self.full.is_empty()
1320    }
1321
1322    fn note(&mut self, change: &day_model::Change, known_store: impl Fn(u64) -> bool) {
1323        let Some(&store) = change.components.first() else {
1324            return;
1325        };
1326        if !known_store(store) {
1327            return;
1328        }
1329        let Some(&key) = change.components.get(1) else {
1330            // A store-level change: the whole value may have been replaced.
1331            if !self.full.contains(&store) {
1332                self.full.push(store);
1333            }
1334            add_author(self.store_authors.entry(store).or_default(), change.author);
1335            return;
1336        };
1337        if key == day_model::STRUCTURE {
1338            return; // the shape path is for the UI; the row paths carry the work
1339        }
1340        let id = (store, key);
1341        if !self.rows.contains_key(&id) {
1342            self.order.push(id);
1343            self.by_store.entry(store).or_default().push(key);
1344        }
1345        add_author(self.authors.entry(id).or_default(), change.author);
1346        let slot = self.rows.entry(id).or_insert(DirtyRow::Update(Vec::new()));
1347        match (change.op, &mut *slot) {
1348            (Op::Delete, _) => *slot = DirtyRow::Delete,
1349            (Op::Insert, _) => *slot = DirtyRow::Insert,
1350            (Op::Set, DirtyRow::Insert | DirtyRow::Delete) | (Op::Move, _) => {}
1351            (Op::Set, DirtyRow::Update(columns)) => {
1352                if change.components.len() >= 3 && !columns.contains(&change.label) {
1353                    columns.push(change.label);
1354                }
1355            }
1356        }
1357    }
1358}
1359
1360// ---------------------------------------------------------------------------
1361// The container
1362// ---------------------------------------------------------------------------
1363
1364/// Everything the container knows how to do for one attached model type, monomorphized at
1365/// [`Schema::with`] and stored type-erased. Join tables (relations.rs) hand-build one over
1366/// their pair-keyed membership cache, which is why keys are plural and clause-shaped here.
1367pub(crate) struct TableHooks {
1368    pub(crate) table: &'static str,
1369    /// `table` split for SQL: the statements below name it through `name.sql()`.
1370    pub(crate) name: TableName,
1371    /// The key column(s), in order: the key SELECT list, the upsert's conflict target, and
1372    /// what its DO UPDATE leaves alone. One entry everywhere but join tables. Plain names,
1373    /// like `columns`; [`ident`] quotes them where a statement is written.
1374    pub(crate) key_cols: Vec<String>,
1375    /// `WHERE` clause + params addressing one row by its key handle.
1376    pub(crate) key_where: KeyWhere,
1377    pub(crate) columns: Vec<String>,
1378    /// `columns` again, as the model's static names: what a [`RowChange`] reports.
1379    pub(crate) column_names: Vec<&'static str>,
1380    /// Same order as `columns`; what a change's label matches against.
1381    pub(crate) fields: Vec<String>,
1382    /// Per column, whether it is a [`Blob`]: named as `NULL` in a row select, protected by
1383    /// `COALESCE` in the upsert, skipped by an update that carries no bytes for it.
1384    pub(crate) lazy: Vec<bool>,
1385    /// After a flush wrote these rows: settle their lazy fields to stored and return the
1386    /// pending bytes they carried, for the blob memo.
1387    pub(crate) settle: SettleFn,
1388    /// The FTS5 shadow's table name, when the model declares `fts(…)`.
1389    pub(crate) fts: Option<String>,
1390    pub(crate) fts_key: &'static str,
1391    /// The R*Tree shadow: (lat column, lon column, shadow table name).
1392    pub(crate) spatial: Option<(String, String, String)>,
1393    /// Current row values by key, read from the cache at flush time; the change log carries
1394    /// which rows and columns moved, never their contents.
1395    pub(crate) row_for: Rc<dyn Fn(u64) -> Option<Vec<Value>>>,
1396    /// Every (key, row) currently resident, for wholesale resyncs.
1397    pub(crate) all_rows: Rc<dyn Fn() -> Vec<(u64, Vec<Value>)>>,
1398    /// The resident keys, in cache (≈ fault) order: what eviction walks.
1399    pub(crate) resident_keys: Rc<dyn Fn() -> Vec<u64>>,
1400    /// The resident count, O(1): the fault path's eviction gate.
1401    pub(crate) resident_len: Rc<dyn Fn() -> usize>,
1402    pub(crate) is_resident: Rc<dyn Fn(u64) -> bool>,
1403    /// Decode raw rows into the cache silently (a fault landing).
1404    pub(crate) absorb: AbsorbFn,
1405    /// Diff raw rows against the resident set, announcing as the external author.
1406    pub(crate) refresh: RefreshFn,
1407    /// Silently drop up to `want` clean rows from the cache, taking candidates in order and
1408    /// skipping anything observed; answers how many left. One retain over the store, so an
1409    /// eviction pass costs O(cache), not O(cache × evicted). (Dirtiness is the container's
1410    /// knowledge, checked before this is called.)
1411    pub(crate) evict: EvictFn,
1412    #[cfg(not(target_arch = "wasm32"))]
1413    pub(crate) dispose_cache: Rc<dyn Fn()>,
1414    /// Bring this store under an undo history; captured here because the model type is known
1415    /// only at attach time.
1416    pub(crate) watch_undo: Rc<dyn Fn(&day_model::UndoStack)>,
1417}
1418
1419/// Why a durable transaction is being performed. Cache refreshes use `EXTERNAL_AUTHOR` and
1420/// do not constitute a write transaction. Remote imports persist but must not echo outbound.
1421#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
1422pub enum WriteOrigin {
1423    #[default]
1424    Local,
1425    RemoteImport,
1426    Maintenance,
1427}
1428
1429/// One row of a transaction's folded change set, as [`TransactionParticipant::before_commit`]
1430/// receives it: every row the ORM's pipeline wrote in the transaction (model edits, relation
1431/// upkeep, in-process cascades), one entry per (table, key). Rows the engine's own `ON
1432/// DELETE` clause removed without the process faulting them are not in the set.
1433#[derive(Clone, Debug, PartialEq)]
1434pub struct RowChange {
1435    /// The model's table, as [`Model::TABLE`] names it (a join table by its generated name).
1436    pub table: &'static str,
1437    /// The key as stored: one value, or the (parent, child) pair of a join row.
1438    pub key: Vec<Value>,
1439    pub kind: RowChangeKind,
1440    /// The distinct author tags ([`day_model::with_author`]) of the changes folded into
1441    /// the row, first seen first. The user's own edits carry no tag; the container's
1442    /// external-merge author never reaches the set.
1443    pub authors: Vec<&'static str>,
1444}
1445
1446/// What happened to a [`RowChange`]'s row over the transaction. Same-row changes coalesce
1447/// the way the autosave fold does: an insert absorbs later updates, a delete absorbs
1448/// everything before it, updates union their columns, and a wholesale write supersedes a
1449/// column list.
1450#[derive(Clone, Debug, PartialEq, Eq)]
1451pub enum RowChangeKind {
1452    Inserted,
1453    /// The changed columns' names, in first-touch order; transient fields never appear.
1454    Updated(Vec<&'static str>),
1455    /// The key only; the removed row's values are not retained.
1456    Deleted,
1457    /// Every column was written: a row-level replacement, or a row of a store rewritten
1458    /// wholesale (`Store::update`), which resyncs the resident rows.
1459    Rewritten,
1460}
1461
1462impl RowChange {
1463    /// Fold a later change to the same row into this one, by the autosave fold's rules.
1464    fn absorb(&mut self, later: RowChange) {
1465        for author in later.authors {
1466            add_author(&mut self.authors, Some(author));
1467        }
1468        self.kind = match (
1469            std::mem::replace(&mut self.kind, RowChangeKind::Deleted),
1470            later.kind,
1471        ) {
1472            (RowChangeKind::Updated(mut cols), RowChangeKind::Updated(more)) => {
1473                for col in more {
1474                    if !cols.contains(&col) {
1475                        cols.push(col);
1476                    }
1477                }
1478                RowChangeKind::Updated(cols)
1479            }
1480            (RowChangeKind::Updated(_), RowChangeKind::Rewritten)
1481            | (RowChangeKind::Rewritten, RowChangeKind::Updated(_) | RowChangeKind::Rewritten) => {
1482                RowChangeKind::Rewritten
1483            }
1484            (
1485                kept @ (RowChangeKind::Inserted | RowChangeKind::Deleted),
1486                RowChangeKind::Updated(_) | RowChangeKind::Rewritten,
1487            ) => kept,
1488            (_, next @ (RowChangeKind::Inserted | RowChangeKind::Deleted)) => next,
1489        };
1490    }
1491}
1492
1493/// The cache refresh a participant asks for after its own SQL may have changed model rows
1494/// that were already flushed from the caches.
1495#[derive(Clone, Debug, PartialEq, Eq, Default)]
1496pub enum Refresh {
1497    /// The participant wrote nothing the caches hold.
1498    #[default]
1499    None,
1500    /// Re-read and diff the resident rows of these tables only, drop the relation memos
1501    /// that involve them, and re-run the live queries that read them.
1502    Tables(Vec<&'static str>),
1503    /// Every resident row, every relation memo, every live query.
1504    All,
1505}
1506
1507/// Queue-confined extension participating in the SAME SQL transaction as model changes.
1508/// Callbacks must use only the supplied connection, never re-enter the container, commit,
1509/// roll back, perform network I/O, or mutate reactive models. Returning an error aborts the
1510/// transaction. `before_commit` runs after worker reactions and intermediate saves settle.
1511/// Persist extension state transactionally; callbacks may run again following a rollback.
1512pub trait TransactionParticipant: 'static {
1513    /// Runs when a transaction opens, on its connection.
1514    ///
1515    /// # Errors
1516    /// An error aborts the transaction.
1517    fn begin(
1518        &self,
1519        connection: &mut dyn SqliteConnection,
1520        origin: WriteOrigin,
1521    ) -> Result<(), DbError>;
1522    /// The transaction's folded change set is `changes`: accumulated across every flush in
1523    /// the transaction (a worker job's intermediate saves included), empty again after a
1524    /// rollback. The answer names the cache refresh to run after a successful COMMIT, before
1525    /// the work is acknowledged; a refresh failure surfaces as
1526    /// [`DbErrorKind::CommittedRefreshFailed`] (the data is durable, it cannot roll back).
1527    ///
1528    /// # Errors
1529    /// An error aborts the transaction.
1530    fn before_commit(
1531        &self,
1532        connection: &mut dyn SqliteConnection,
1533        origin: WriteOrigin,
1534        changes: &[RowChange],
1535    ) -> Result<Refresh, DbError>;
1536}
1537
1538pub(crate) struct ContainerInner {
1539    conn: RefCell<Box<dyn SqliteConnection>>,
1540    caps: Capabilities,
1541    /// store root id → hooks.
1542    tables: RefCell<HashMap<u64, TableHooks>>,
1543    /// model `TypeId` → the `Store<Keyed<M>>` handle, boxed.
1544    stores: RefCell<HashMap<TypeId, Box<dyn Any>>>,
1545    dirty: RefCell<DirtyState>,
1546    /// Live query result sets, marked stale from the change sink. Weak: the app's `Query`
1547    /// handles own the state; dead entries are pruned on dispatch.
1548    queries: RefCell<Vec<std::rc::Weak<QueryState>>>,
1549    /// The engine's cross-connection change counter as of the last look (`PRAGMA
1550    /// data_version`; it moves only when another connection commits to the file). `None`
1551    /// where the driver reports no external-change detection.
1552    data_version: Cell<Option<i64>>,
1553    /// Foreign-key clauses relations contribute to their targets' tables, set before attach.
1554    fk_specs: RefCell<Vec<FkSpec>>,
1555    /// The wired relations, maintained from the change sink and read by `RelationRef`s.
1556    pub(crate) relations: RefCell<Vec<Rc<relations::ToOneRel>>>,
1557    /// The wired many-to-manys, each over its own membership cache.
1558    pub(crate) joins: RefCell<Vec<Rc<relations::JoinRel>>>,
1559    /// The declared value links (`#[model(link(…))]`), from every attached model.
1560    pub(crate) links: RefCell<Vec<LinkInfo>>,
1561    /// The databases `attach_database` holds open, by alias.
1562    attached: RefCell<Vec<String>>,
1563    sink: Cell<Option<day_model::ChangeSinkId>>,
1564    autosave: Cell<bool>,
1565    /// An outer worker transaction owns commit/rollback; inner model saves only execute SQL.
1566    worker_transaction: Cell<bool>,
1567    participant: RefCell<Option<Rc<dyn TransactionParticipant>>>,
1568    /// The open transaction's folded change set, kept only while a participant is installed:
1569    /// appended per flush, handed to `before_commit`, cleared at begin, commit and rollback.
1570    transaction_changes: RefCell<Vec<RowChange>>,
1571    /// What `before_commit` asked for, acted on once the COMMIT has succeeded.
1572    refresh_scope: RefCell<Refresh>,
1573    refresh_pending: Cell<bool>,
1574    /// Soft per-table bound on resident rows. Dirty and observed rows never evict, so the
1575    /// working set can exceed it; everything else does not.
1576    cache_limit: Cell<usize>,
1577    /// Retry eviction after growth when dirty/observed rows prevent reaching the limit.
1578    eviction_retry: RefCell<HashMap<u64, usize>>,
1579    /// Bytes of lazy blob columns read or written recently, bounded by `blob_cache_limit`.
1580    blob_memo: RefCell<blob::BlobMemo>,
1581    blob_cache_limit: Cell<usize>,
1582    /// Child tables whose rows the engine's `ON DELETE` clauses are taking or nullifying in
1583    /// the flush under way, with no undo stack to serve: their resident rows are re-read
1584    /// after the commit (relations.rs, `parent_deleted`).
1585    engine_cascades: RefCell<Vec<&'static str>>,
1586    /// Set while `reconcile_engine_cascades` re-reads the child tables. Every row it finds
1587    /// gone or re-pointed got that way because its parent was deleted, so relation upkeep
1588    /// skips the point lookups that would ask the file for a parent that no longer exists.
1589    pub(crate) reconciling: Cell<bool>,
1590    /// The last autosave failure, observable by the UI (`when(container.last_error()…)`).
1591    pub(crate) error: day_reactive::Signal<Option<String>>,
1592    /// Guards re-entrant flushes (a turn-end firing during an explicit save).
1593    flushing: Cell<bool>,
1594}
1595
1596/// An open database and the working-set caches over it. Clone is shallow: clones share the
1597/// connection and the dirty state.
1598#[derive(Clone)]
1599pub struct ModelContainer {
1600    pub(crate) inner: Rc<ContainerInner>,
1601}
1602
1603/// The default per-table bound on resident rows ([`ModelContainer::set_cache_limit`]).
1604pub const DEFAULT_CACHE_LIMIT: usize = 8_192;
1605
1606/// How many bytes of lazy [`Blob`] values a container keeps in memory by default (32 MiB):
1607/// the memo behind [`BlobField::bytes`], bounded in bytes rather than rows.
1608pub const DEFAULT_BLOB_CACHE_LIMIT: usize = 32 * 1024 * 1024;
1609
1610impl ModelContainer {
1611    /// Open through `driver`, migrate, and attach every model in `schema`. Attaching creates
1612    /// or migrates the table and nothing more: no rows load, so open cost does not grow with
1613    /// the file. Autosave is on: any turn that touched a store flushes at its end.
1614    ///
1615    /// # Errors
1616    /// Returns an error when the driver cannot open the database, or when a table cannot be created
1617    /// or migrated to the declared schema.
1618    pub fn open<D: SqliteDriver>(driver: D, schema: Schema) -> Result<ModelContainer, DbError>
1619    where
1620        D::Connection: 'static,
1621    {
1622        Self::open_with(driver, schema, MigrationPlan::new())
1623    }
1624
1625    /// [`ModelContainer::open`], with hand-written migration stages that run first.
1626    ///
1627    /// # Errors
1628    /// Returns an error when the driver cannot open the database, a migration stage fails, or a
1629    /// table cannot be created or migrated to the declared schema.
1630    pub fn open_with<D: SqliteDriver>(
1631        driver: D,
1632        schema: Schema,
1633        plan: MigrationPlan,
1634    ) -> Result<ModelContainer, DbError>
1635    where
1636        D::Connection: 'static,
1637    {
1638        let caps = driver.capabilities();
1639        let conn: Box<dyn SqliteConnection> = Box::new(driver.open()?);
1640        let error = day_reactive::Scope::detached().enter(|| day_reactive::Signal::new(None));
1641        let container = ModelContainer {
1642            inner: Rc::new(ContainerInner {
1643                conn: RefCell::new(conn),
1644                caps,
1645                tables: RefCell::new(HashMap::new()),
1646                stores: RefCell::new(HashMap::new()),
1647                dirty: RefCell::new(DirtyState::default()),
1648                queries: RefCell::new(Vec::new()),
1649                data_version: Cell::new(None),
1650                fk_specs: RefCell::new(Vec::new()),
1651                relations: RefCell::new(Vec::new()),
1652                joins: RefCell::new(Vec::new()),
1653                links: RefCell::new(Vec::new()),
1654                attached: RefCell::new(Vec::new()),
1655                sink: Cell::new(None),
1656                autosave: Cell::new(true),
1657                worker_transaction: Cell::new(false),
1658                refresh_pending: Cell::new(false),
1659                participant: RefCell::new(None),
1660                transaction_changes: RefCell::new(Vec::new()),
1661                refresh_scope: RefCell::new(Refresh::None),
1662                cache_limit: Cell::new(DEFAULT_CACHE_LIMIT),
1663                eviction_retry: RefCell::new(HashMap::new()),
1664                blob_memo: RefCell::new(blob::BlobMemo::default()),
1665                blob_cache_limit: Cell::new(DEFAULT_BLOB_CACHE_LIMIT),
1666                engine_cascades: RefCell::new(Vec::new()),
1667                reconciling: Cell::new(false),
1668                error,
1669                flushing: Cell::new(false),
1670            }),
1671        };
1672
1673        container.ensure_schema_table()?;
1674        container.run_stages(plan)?;
1675        let Schema {
1676            installers,
1677            wirers,
1678            fk_specs,
1679        } = schema;
1680        *container.inner.fk_specs.borrow_mut() = fk_specs;
1681        for install in installers {
1682            install(&container)?;
1683        }
1684        // Relations wire once every table is attached (both ends exist), and only then does
1685        // the sink go live.
1686        for wire in wirers {
1687            wire(&container)?;
1688        }
1689        relations::register_container(&container.inner);
1690        container.install_sink();
1691        container.install_autosave();
1692        if caps.external_changes {
1693            container
1694                .inner
1695                .data_version
1696                .set(container.file_data_version()?);
1697        }
1698        Ok(container)
1699    }
1700
1701    /// The author tag on changes [`ModelContainer::check_external`] merges in: the database's
1702    /// own contents arriving, distinguishable by every change consumer and never persisted
1703    /// back. Reserved: an app writing stores under this tag would have its writes dropped by
1704    /// the autosave fold.
1705    pub const EXTERNAL_AUTHOR: &'static str = "database";
1706
1707    /// The working-set cache for `M`, an ordinary day-model store holding the rows currently
1708    /// resident and not the table: its `keys()` are whatever happens to be faulted in. Bindings
1709    /// and accessors work on it unchanged; enumerate rows through a [`ModelContainer::query`],
1710    /// never through the cache.
1711    ///
1712    /// # Panics
1713    /// Panics only if `M` was not in the container's `schema!`, which is a wiring bug worth
1714    /// stopping on.
1715    pub fn cache<M: Model>(&self) -> Store<Keyed<M>> {
1716        let stores = self.inner.stores.borrow();
1717        let any = stores
1718            .get(&TypeId::of::<M>())
1719            .unwrap_or_else(|| panic!("model `{}` is not in this container's schema!", M::TABLE));
1720        *any.downcast_ref::<Store<Keyed<M>>>()
1721            .expect("store map holds the exact type it was keyed by")
1722    }
1723
1724    /// What the driver can and cannot do.
1725    pub fn capabilities(&self) -> Capabilities {
1726        self.inner.caps
1727    }
1728
1729    /// Autosave on/off (default on). Off, changes accumulate until [`ModelContainer::save`],
1730    /// and live queries answer from the last save, since only the file can answer them.
1731    pub fn set_autosave(&self, on: bool) {
1732        self.inner.autosave.set(on);
1733    }
1734
1735    /// The soft per-table bound on resident rows (default [`DEFAULT_CACHE_LIMIT`]). Dirty
1736    /// rows and rows something observes never evict; beyond that, faulting past the limit
1737    /// releases the oldest resident rows. `usize::MAX` disables eviction. Lowering the limit
1738    /// enforces it immediately.
1739    pub fn set_cache_limit(&self, rows: usize) {
1740        self.inner.cache_limit.set(rows.max(1));
1741        let store_ids: Vec<u64> = self.inner.tables.borrow().keys().copied().collect();
1742        for id in store_ids {
1743            self.enforce_cache_limit(id, &[], true);
1744        }
1745    }
1746
1747    // --- lazy blobs (blob.rs) ------------------------------------------------------------
1748
1749    /// Bound, in bytes, on the lazy [`Blob`] values kept in memory after a read or a flush
1750    /// (default [`DEFAULT_BLOB_CACHE_LIMIT`]). Lowering it releases the oldest entries now;
1751    /// a value larger than the limit is served but never kept.
1752    pub fn set_blob_cache_limit(&self, bytes: usize) {
1753        self.inner.blob_cache_limit.set(bytes);
1754        let mut memo = self.inner.blob_memo.borrow_mut();
1755        if bytes == 0 {
1756            memo.clear();
1757        } else {
1758            // Re-inserting nothing: the memo trims itself on the next insert, so trim here by
1759            // inserting an empty value under a key no row has, then forgetting it.
1760            memo.insert(
1761                (u64::MAX, u64::MAX, ""),
1762                std::sync::Arc::new(Vec::new()),
1763                bytes,
1764            );
1765            memo.forget_row(u64::MAX, u64::MAX);
1766        }
1767    }
1768
1769    /// Bytes the blob memo currently holds.
1770    pub fn blob_cache_bytes(&self) -> usize {
1771        self.inner.blob_memo.borrow().bytes()
1772    }
1773
1774    /// Values the blob memo currently holds.
1775    pub fn blob_cache_len(&self) -> usize {
1776        self.inner.blob_memo.borrow().len()
1777    }
1778
1779    /// Whether a delete may leave its rule to the engine's `ON DELETE` clause: no undo stack
1780    /// is capturing (it needs the removed rows to restore them) and no transaction
1781    /// participant is installed (its change set would miss the rows the engine removed).
1782    pub(crate) fn engine_cascade_allowed(&self) -> bool {
1783        !day_model::record_capture_active() && self.inner.participant.borrow().is_none()
1784    }
1785
1786    /// A relation whose parent row is being deleted this turn, with no undo stack to serve:
1787    /// the engine's `ON DELETE` clause will take or nullify the children in the flush, and
1788    /// the resident ones are re-read afterwards.
1789    pub(crate) fn note_engine_cascade(&self, child_table: &'static str) {
1790        let mut tables = self.inner.engine_cascades.borrow_mut();
1791        if !tables.contains(&child_table) {
1792            tables.push(child_table);
1793        }
1794    }
1795
1796    /// After a flush whose parent deletes the engine cascaded: re-read the resident rows of
1797    /// the child tables (vanished rows announce as deletes, nullified keys as field changes,
1798    /// both under the database's own author), drop their relation memos and re-run the live
1799    /// queries that read them. A vanished child is itself a parent whose children the engine
1800    /// took too, so the pass repeats while the announcements name new tables.
1801    fn reconcile_engine_cascades(&self) {
1802        for _ in 0..16 {
1803            let tables: Vec<&'static str> =
1804                std::mem::take(&mut *self.inner.engine_cascades.borrow_mut());
1805            if tables.is_empty() {
1806                return;
1807            }
1808            self.inner.reconciling.set(true);
1809            let refreshed = self.refresh_resident_in(Some(&tables));
1810            self.inner.reconciling.set(false);
1811            if let Err(error) = refreshed {
1812                self.inner.error.set(Some(error.to_string()));
1813                return;
1814            }
1815            self.invalidate_relation_memos_of(&tables);
1816            self.requery_tables(&tables);
1817        }
1818    }
1819
1820    /// After a successful flush: settle every written row's lazy fields (pending → stored),
1821    /// keep the bytes they carried in the memo, and drop the memo of every deleted row.
1822    fn settle_blobs(&self, dirty: &DirtyState) {
1823        let mut written: HashMap<u64, Vec<u64>> = HashMap::new();
1824        let mut deleted: Vec<(u64, u64)> = Vec::new();
1825        for id in &dirty.order {
1826            match dirty.rows.get(id) {
1827                Some(DirtyRow::Delete) => deleted.push(*id),
1828                Some(_) => written.entry(id.0).or_default().push(id.1),
1829                None => {}
1830            }
1831        }
1832        for store in &dirty.full {
1833            let keys = {
1834                let tables = self.inner.tables.borrow();
1835                tables.get(store).map_or_default(|h| (h.resident_keys)())
1836            };
1837            written.entry(*store).or_default().extend(keys);
1838        }
1839        let limit = self.inner.blob_cache_limit.get();
1840        for (store, keys) in written {
1841            let settle = {
1842                let tables = self.inner.tables.borrow();
1843                match tables.get(&store) {
1844                    Some(h) if h.lazy.iter().any(|l| *l) => h.settle.clone(),
1845                    _ => continue,
1846                }
1847            };
1848            for (key, field, bytes) in settle(&keys) {
1849                self.inner
1850                    .blob_memo
1851                    .borrow_mut()
1852                    .insert((store, key, field), bytes, limit);
1853            }
1854        }
1855        if !deleted.is_empty() {
1856            let mut memo = self.inner.blob_memo.borrow_mut();
1857            for (store, key) in deleted {
1858                memo.forget_row(store, key);
1859            }
1860        }
1861    }
1862
1863    /// The SQL half of a lazy read: the table, the key clause and the stored column name.
1864    fn blob_target(
1865        &self,
1866        store: u64,
1867        key: u64,
1868        column: &str,
1869    ) -> Option<(String, String, Vec<Value>)> {
1870        let tables = self.inner.tables.borrow();
1871        let hooks = tables.get(&store)?;
1872        let (clause, params) = (hooks.key_where)(key);
1873        Some((
1874            hooks.name.sql(),
1875            format!("{} FROM {} WHERE {clause}", ident(column), hooks.name.sql()),
1876            params,
1877        ))
1878    }
1879
1880    /// The whole value of a lazy column, from the memo or one point `SELECT`. Pending writes
1881    /// flush first with autosave, like every other read.
1882    pub(crate) fn blob_bytes(
1883        &self,
1884        store: u64,
1885        key: u64,
1886        column: &'static str,
1887    ) -> Result<std::sync::Arc<Vec<u8>>, DbError> {
1888        if self.inner.autosave.get() && !self.inner.dirty.borrow().is_empty() {
1889            self.save()?;
1890        }
1891        if let Some(hit) = self.inner.blob_memo.borrow_mut().get((store, key, column)) {
1892            return Ok(hit);
1893        }
1894        let Some((_, tail, params)) = self.blob_target(store, key, column) else {
1895            return Ok(std::sync::Arc::new(Vec::new()));
1896        };
1897        let mut bytes = Vec::new();
1898        self.conn()
1899            .query(&format!("SELECT {tail}"), &params, &mut |row| {
1900                bytes = row.get(0).as_blob().map_or_default(<[u8]>::to_vec);
1901            })?;
1902        let bytes = std::sync::Arc::new(bytes);
1903        let limit = self.inner.blob_cache_limit.get();
1904        self.inner
1905            .blob_memo
1906            .borrow_mut()
1907            .insert((store, key, column), bytes.clone(), limit);
1908        Ok(bytes)
1909    }
1910
1911    /// `length(col)`: the value's size without reading it (the memo answers when it can).
1912    pub(crate) fn blob_len(
1913        &self,
1914        store: u64,
1915        key: u64,
1916        column: &'static str,
1917    ) -> Result<usize, DbError> {
1918        if self.inner.autosave.get() && !self.inner.dirty.borrow().is_empty() {
1919            self.save()?;
1920        }
1921        if let Some(hit) = self.inner.blob_memo.borrow_mut().get((store, key, column)) {
1922            return Ok(hit.len());
1923        }
1924        let Some((table, _, params)) = self.blob_target(store, key, column) else {
1925            return Ok(0);
1926        };
1927        let clause = {
1928            let tables = self.inner.tables.borrow();
1929            tables.get(&store).map_or_default(|h| (h.key_where)(key).0)
1930        };
1931        let mut len = 0usize;
1932        self.conn().query(
1933            &format!(
1934                "SELECT length({}) FROM {table} WHERE {clause}",
1935                ident(column)
1936            ),
1937            &params,
1938            &mut |row| len = row.get(0).as_int().unwrap_or(0).max(0) as usize,
1939        )?;
1940        Ok(len)
1941    }
1942
1943    /// `substr(col, start + 1, len)`: one range of the value, clamped to its length, so only
1944    /// the asked-for bytes cross from the engine.
1945    pub(crate) fn blob_range(
1946        &self,
1947        store: u64,
1948        key: u64,
1949        column: &'static str,
1950        range: std::ops::Range<usize>,
1951    ) -> Result<Vec<u8>, DbError> {
1952        if self.inner.autosave.get() && !self.inner.dirty.borrow().is_empty() {
1953            self.save()?;
1954        }
1955        if let Some(hit) = self.inner.blob_memo.borrow_mut().get((store, key, column)) {
1956            let start = range.start.min(hit.len());
1957            let end = range.end.clamp(start, hit.len());
1958            return Ok(hit[start..end].to_vec());
1959        }
1960        if range.end <= range.start {
1961            return Ok(Vec::new());
1962        }
1963        let Some((table, _, mut params)) = self.blob_target(store, key, column) else {
1964            return Ok(Vec::new());
1965        };
1966        let clause = {
1967            let tables = self.inner.tables.borrow();
1968            tables.get(&store).map_or_default(|h| (h.key_where)(key).0)
1969        };
1970        let mut all = vec![
1971            Value::Int(range.start as i64 + 1),
1972            Value::Int((range.end - range.start) as i64),
1973        ];
1974        all.append(&mut params);
1975        let mut bytes = Vec::new();
1976        self.conn().query(
1977            &format!(
1978                "SELECT substr({}, ?, ?) FROM {table} WHERE {clause}",
1979                ident(column)
1980            ),
1981            &all,
1982            &mut |row| bytes = row.get(0).as_blob().map_or_default(<[u8]>::to_vec),
1983        )?;
1984        Ok(bytes)
1985    }
1986
1987    /// Drop the memo of one row's lazy values (the row left).
1988    #[allow(dead_code)]
1989    pub(crate) fn blob_forget(&self, store: u64, key: u64) {
1990        self.inner.blob_memo.borrow_mut().forget_row(store, key);
1991    }
1992
1993    /// Drop the memo of one lazy value (a write replaced it).
1994    pub(crate) fn blob_forget_field(&self, store: u64, key: u64, column: &'static str) {
1995        self.inner
1996            .blob_memo
1997            .borrow_mut()
1998            .forget_one((store, key, column));
1999    }
2000
2001    /// Install one transaction extension before exposing the container to writers. Registration
2002    /// does not initialize its schema; do that in the worker factory first. No replacement or
2003    /// removal is supported, preventing a sync-enabled connection silently losing journaling.
2004    ///
2005    /// # Errors
2006    /// Returns a `Deny` error when the container has pending changes, is inside a worker
2007    /// transaction, or already has a participant.
2008    pub fn install_transaction_participant(
2009        &self,
2010        participant: impl TransactionParticipant,
2011    ) -> Result<(), DbError> {
2012        if self.inner.worker_transaction.get()
2013            || !self.inner.dirty.borrow().is_empty()
2014            || self.inner.participant.borrow().is_some()
2015        {
2016            return Err(DbError::new(
2017                DbErrorKind::Deny,
2018                "transaction participant must be installed on an idle, clean container",
2019            ));
2020        }
2021        *self.inner.participant.borrow_mut() = Some(Rc::new(participant));
2022        Ok(())
2023    }
2024
2025    fn participant_begin(
2026        &self,
2027        conn: &mut dyn SqliteConnection,
2028        origin: WriteOrigin,
2029    ) -> Result<(), DbError> {
2030        let participant = self.inner.participant.borrow().clone();
2031        if let Some(p) = participant {
2032            self.inner.transaction_changes.borrow_mut().clear();
2033            *self.inner.refresh_scope.borrow_mut() = Refresh::None;
2034            p.begin(conn, origin)?;
2035        }
2036        Ok(())
2037    }
2038
2039    /// Hand the participant the transaction's change set and keep its refresh request for
2040    /// [`ModelContainer::participant_refresh`], which runs once the COMMIT has succeeded.
2041    fn participant_commit(
2042        &self,
2043        conn: &mut dyn SqliteConnection,
2044        origin: WriteOrigin,
2045    ) -> Result<(), DbError> {
2046        let participant = self.inner.participant.borrow().clone();
2047        if let Some(p) = participant {
2048            let changes = std::mem::take(&mut *self.inner.transaction_changes.borrow_mut());
2049            let scope = p.before_commit(conn, origin, &changes)?;
2050            *self.inner.refresh_scope.borrow_mut() = scope;
2051        }
2052        Ok(())
2053    }
2054
2055    /// A transaction is ending without a COMMIT: its change set is not anyone's to see.
2056    fn participant_rollback(&self) {
2057        self.inner.transaction_changes.borrow_mut().clear();
2058        *self.inner.refresh_scope.borrow_mut() = Refresh::None;
2059    }
2060
2061    /// Append one flush's folded rows to the open transaction's change set, coalescing with
2062    /// the rows earlier flushes of the same transaction already recorded.
2063    fn record_changes(&self, changes: Vec<RowChange>) {
2064        if changes.is_empty() {
2065            return;
2066        }
2067        let mut set = self.inner.transaction_changes.borrow_mut();
2068        for change in changes {
2069            match set
2070                .iter_mut()
2071                .find(|c| c.table == change.table && c.key == change.key)
2072            {
2073                None => set.push(change),
2074                Some(prior) => prior.absorb(change),
2075            }
2076        }
2077    }
2078
2079    /// The last autosave failure, if any, as a tracked signal the UI can watch.
2080    pub fn last_error(&self) -> day_reactive::Signal<Option<String>> {
2081        self.inner.error
2082    }
2083
2084    /// Flush every pending change now, in one transaction. Autosave calls this at turn end;
2085    /// call it directly where the error matters at a known point.
2086    ///
2087    /// # Errors
2088    /// Returns the storage error when the flush fails.
2089    pub fn save(&self) -> Result<(), DbError> {
2090        self.save_now().map(|_| ())
2091    }
2092
2093    /// Run `f` and return the SQL one flush of everything it changed issues: the headless
2094    /// persistence assert (autosave is suspended for the duration so the statements land
2095    /// here). Twenty keystrokes into one field come back as one `UPDATE`.
2096    ///
2097    /// # Errors
2098    /// Returns the storage error when the flush fails.
2099    pub fn record_sql(&self, f: impl FnOnce()) -> Result<Vec<String>, DbError> {
2100        let prior = self.inner.autosave.replace(false);
2101        f();
2102        let result = self.save_now();
2103        self.inner.autosave.set(prior);
2104        result.map(|stmts| stmts.into_iter().map(|(sql, _)| sql).collect())
2105    }
2106
2107    fn save_now(&self) -> Result<Vec<(String, Vec<Value>)>, DbError> {
2108        if self.inner.flushing.get() {
2109            return Ok(Vec::new());
2110        }
2111        if self.inner.refresh_pending.get() {
2112            self.refresh_committed()?;
2113        }
2114        let dirty = {
2115            let mut d = self.inner.dirty.borrow_mut();
2116            if d.is_empty() {
2117                drop(d);
2118                // Nothing to flush, but a stale query may still be waiting on its deferred
2119                // requery (a fetch swap on a clean container lands here).
2120                self.run_deferred_requeries(&[]);
2121                return Ok(Vec::new());
2122            }
2123            std::mem::take(&mut *d)
2124        };
2125        self.inner.flushing.set(true);
2126        let had_writes = !dirty.is_empty();
2127        let result = self.flush(&dirty);
2128        self.inner.flushing.set(false);
2129        match &result {
2130            Ok(stmts) => {
2131                // Previously dirty rows can now be evicted on the next fault.
2132                self.inner.eviction_retry.borrow_mut().clear();
2133                // And a pass runs now for every store the flush touched: an import that only
2134                // inserts never faults, so without this every row it wrote would stay
2135                // resident, however far past the limit (a ten-million-row seed sat at 3 GB).
2136                let mut touched: Vec<u64> = dirty.order.iter().map(|(store, _)| *store).collect();
2137                touched.extend(dirty.full.iter().copied());
2138                touched.sort_unstable();
2139                touched.dedup();
2140                for store in touched {
2141                    self.enforce_cache_limit(store, &[], false);
2142                }
2143                self.settle_blobs(&dirty);
2144                self.reconcile_engine_cascades();
2145                self.inner.error.set_if_changed(None);
2146                self.run_deferred_requeries(stmts);
2147            }
2148            Err(e) => {
2149                // Keep the optimistic cache and its pending writes together. A retry must
2150                // persist the same edits, including any edits made after the failed save.
2151                {
2152                    let mut pending = self.inner.dirty.borrow_mut();
2153                    let newer = std::mem::take(&mut *pending);
2154                    *pending = dirty;
2155                    pending.merge(newer);
2156                }
2157                self.inner.error.set(Some(e.to_string()));
2158            }
2159        }
2160        if result.is_ok() && !self.inner.worker_transaction.get() && had_writes {
2161            self.participant_refresh()?;
2162        }
2163        result
2164    }
2165
2166    /// After a successful COMMIT: the refresh `before_commit` asked for. A failure here is
2167    /// [`DbErrorKind::CommittedRefreshFailed`]; the refresh stays pending (as a full one) for
2168    /// the next save to retry, since the file already holds the participant's writes.
2169    fn participant_refresh(&self) -> Result<(), DbError> {
2170        let scope = std::mem::take(&mut *self.inner.refresh_scope.borrow_mut());
2171        let tables = match scope {
2172            Refresh::None => return Ok(()),
2173            Refresh::Tables(tables) => Some(tables),
2174            Refresh::All => None,
2175        };
2176        self.inner.refresh_pending.set(true);
2177        self.refresh_committed_in(tables.as_deref()).map_err(|e| {
2178            let error = DbError::new(DbErrorKind::CommittedRefreshFailed, e.to_string());
2179            self.inner.error.set(Some(error.to_string()));
2180            error
2181        })
2182    }
2183
2184    fn refresh_committed(&self) -> Result<(), DbError> {
2185        self.refresh_committed_in(None)
2186    }
2187
2188    /// Re-read the resident rows of `tables` (every table when `None`) and bring the memos
2189    /// and live queries that depend on them up to date.
2190    fn refresh_committed_in(&self, tables: Option<&[&'static str]>) -> Result<(), DbError> {
2191        if !self.inner.dirty.borrow().is_empty() {
2192            return Err(DbError::new(
2193                DbErrorKind::Deny,
2194                "refresh committed state before making further model edits",
2195            ));
2196        }
2197        self.refresh_resident_in(tables)?;
2198        match tables {
2199            None => {
2200                self.invalidate_relation_memos();
2201                self.requery_all();
2202            }
2203            Some(tables) => {
2204                self.invalidate_relation_memos_of(tables);
2205                self.requery_tables(tables);
2206            }
2207        }
2208        if self.inner.caps.external_changes {
2209            self.inner.data_version.set(self.file_data_version()?);
2210        }
2211        self.inner.refresh_pending.set(false);
2212        self.inner.error.set_if_changed(None);
2213        Ok(())
2214    }
2215
2216    // --- faulting ----------------------------------------------------------------------------
2217
2218    /// One row, made resident (faulted from the file if it was not). `None` when no such row
2219    /// exists (or a driver error surfaced, observable through [`ModelContainer::last_error`]).
2220    pub fn get<M: Model>(&self, id: impl Into<ModelId<M>>) -> Option<day_model::Elem<M>> {
2221        match self.try_get::<M>(id) {
2222            Ok(row) => row,
2223            Err(error) => {
2224                self.inner.error.set(Some(error.to_string()));
2225                None
2226            }
2227        }
2228    }
2229
2230    /// Read a row, distinguishing an absent key from a storage or decoding failure.
2231    ///
2232    /// # Errors
2233    /// Returns the storage or decoding error when the row cannot be read.
2234    pub fn try_get<M: Model>(
2235        &self,
2236        id: impl Into<ModelId<M>>,
2237    ) -> Result<Option<day_model::Elem<M>>, DbError> {
2238        let store = self.cache::<M>();
2239        let h = id.into().handle();
2240        if !store.with_untracked(|k| k.get(h).is_some()) {
2241            self.ensure_resident::<M>(&[h])?;
2242        }
2243        Ok(store
2244            .with_untracked(|k| k.get(h).is_some())
2245            .then(|| store.elem(h)))
2246    }
2247
2248    /// Make these rows resident, faulting the missing ones in one chunked `SELECT`. Rows the
2249    /// file does not have are not resident afterwards; rows deleted this turn are not
2250    /// resurrected.
2251    ///
2252    /// # Errors
2253    /// Returns the storage or decoding error when the rows cannot be read.
2254    pub fn ensure_resident<M: Model>(&self, keys: &[u64]) -> Result<(), DbError> {
2255        let store = self.cache::<M>();
2256        self.fault_into(store.store_id(), keys)
2257    }
2258
2259    /// [`ModelContainer::ensure_resident`], typed ids in.
2260    ///
2261    /// # Errors
2262    /// Returns the storage or decoding error when the rows cannot be read.
2263    pub fn ensure<M: Model>(
2264        &self,
2265        ids: impl IntoIterator<Item = impl Into<ModelId<M>>>,
2266    ) -> Result<(), DbError> {
2267        let keys: Vec<u64> = ids.into_iter().map(|i| i.into().handle()).collect();
2268        self.ensure_resident::<M>(&keys)
2269    }
2270
2271    /// Fault every row of `M`'s table in: the document pattern, said explicitly. A sketch's
2272    /// scene or a settings table is the working set, and the app that draws all of it warms
2273    /// it once at open. Raise the cache limit first (`set_cache_limit(usize::MAX)` for a
2274    /// document container) so the warmed rows are not immediately eligible to leave; rows
2275    /// deleted this turn stay deleted. Returns how many rows are resident afterward.
2276    ///
2277    /// # Errors
2278    /// Returns the storage or decoding error when the rows cannot be read.
2279    pub fn warm<M: Model>(&self) -> Result<usize, DbError> {
2280        let store = self.cache::<M>();
2281        let key = ident(M::KEY);
2282        let keys = self.select_id_column_checked(&format!(
2283            "SELECT {key} FROM {} ORDER BY {key}",
2284            TableName::of::<M>().sql()
2285        ))?;
2286        self.fault_into(store.store_id(), &keys)?;
2287        Ok(store.with_untracked(day_model::Keyed::len))
2288    }
2289
2290    /// Insert one row through the front door: it becomes resident and dirty, announces, and
2291    /// folds to one `INSERT` at the next flush.
2292    pub fn insert<M: Model>(&self, row: M) {
2293        let store = self.cache::<M>();
2294        let h = row.handle();
2295        store.restructure("create", Op::Insert, h, move |k| k.push(row));
2296    }
2297
2298    /// `SELECT COUNT(*)`: the table's true size, which the cache cannot know. Settles
2299    /// pending writes first (with autosave), like every other read.
2300    ///
2301    /// # Errors
2302    /// Returns the storage error when the read fails.
2303    pub fn table_count<M: Model>(&self) -> Result<u64, DbError> {
2304        if self.inner.autosave.get() && !self.inner.dirty.borrow().is_empty() {
2305            self.save()?;
2306        }
2307        let mut n = 0i64;
2308        self.conn().query(
2309            &format!("SELECT COUNT(*) FROM {}", TableName::of::<M>().sql()),
2310            &[],
2311            &mut |row| {
2312                n = row.get(0).as_int().unwrap_or(0);
2313            },
2314        )?;
2315        Ok(n.max(0) as u64)
2316    }
2317
2318    /// The untyped fault: bring `keys` into `store_id`'s cache.
2319    pub(crate) fn fault_into(&self, store_id: u64, keys: &[u64]) -> Result<(), DbError> {
2320        let (is_resident, absorb, columns, table, keys_where) = {
2321            let tables = self.inner.tables.borrow();
2322            let Some(hooks) = tables.get(&store_id) else {
2323                return Ok(());
2324            };
2325            (
2326                hooks.is_resident.clone(),
2327                hooks.absorb.clone(),
2328                select_list(hooks),
2329                hooks.name.sql(),
2330                keys_where_of(hooks),
2331            )
2332        };
2333        let missing: Vec<u64> = {
2334            let dirty = self.inner.dirty.borrow();
2335            keys.iter()
2336                .copied()
2337                .filter(|k| !is_resident(*k))
2338                // A row deleted this turn still exists in the file until the flush; faulting
2339                // it back would resurrect it.
2340                .filter(|k| dirty.rows.get(&(store_id, *k)) != Some(&DirtyRow::Delete))
2341                .collect()
2342        };
2343        if missing.is_empty() {
2344            return Ok(());
2345        }
2346        let mut faulted: Vec<u64> = Vec::with_capacity(missing.len());
2347        for chunk in missing.chunks(MAX_BOUND_PARAMS / 2) {
2348            let (clause, params) = keys_where(chunk);
2349            let mut rows: Vec<Vec<Value>> = Vec::new();
2350            self.conn().query(
2351                &format!("SELECT {columns} FROM {table} WHERE {clause}"),
2352                &params,
2353                &mut |row| {
2354                    let n = Row::len(row);
2355                    rows.push((0..n).map(|i| row.get(i)).collect());
2356                },
2357            )?;
2358            faulted.extend(absorb(rows)?);
2359        }
2360        // The Recorder serves whole fixture tables whatever the `WHERE` says; keep only what
2361        // was asked for so faulting stays precise there too. (Through a `HashSet`; a linear
2362        // scan here made warming half a million rows quadratic.)
2363        let wanted: std::collections::HashSet<u64> = missing.iter().copied().collect();
2364        faulted.retain(|k| wanted.contains(k));
2365        self.enforce_cache_limit(store_id, &faulted, false);
2366        Ok(())
2367    }
2368
2369    /// Release resident rows beyond the cache limit: oldest first, never a dirty row, never
2370    /// an observed one, never one just faulted for the caller still holding it.
2371    ///
2372    /// The pass walks every resident key, so it runs with hysteresis on the fault path:
2373    /// nothing happens until the cache overshoots the limit by a slack margin, and one pass
2374    /// then brings it back to the limit, amortizing the walk over the faults that filled the
2375    /// slack, instead of paying O(resident) per fault batch. (Measured: a 50k-row relation
2376    /// traversal over a 500k-row table spent 99% of its time in the un-amortized walk.)
2377    /// `strict` skips the slack, so `set_cache_limit` enforces its new bound immediately.
2378    fn enforce_cache_limit(&self, store_id: u64, protect: &[u64], strict: bool) {
2379        let limit = self.inner.cache_limit.get();
2380        if limit == usize::MAX {
2381            return;
2382        }
2383        let (resident_len, evict) = {
2384            let tables = self.inner.tables.borrow();
2385            let Some(hooks) = tables.get(&store_id) else {
2386                return;
2387            };
2388            ((hooks.resident_len)(), hooks.evict.clone())
2389        };
2390        let slack = if strict {
2391            0
2392        } else {
2393            (limit / 8).clamp(64, 4096)
2394        };
2395        let threshold = if strict {
2396            limit
2397        } else {
2398            self.inner
2399                .eviction_retry
2400                .borrow()
2401                .get(&store_id)
2402                .copied()
2403                .unwrap_or_else(|| limit.saturating_add(slack))
2404        };
2405        if resident_len <= threshold {
2406            return;
2407        }
2408        // Back off even when every row is pinned. A fixed limit gate otherwise scans an
2409        // ever-growing dirty transaction on EVERY single-row fault (quadratic work).
2410        self.inner.eviction_retry.borrow_mut().insert(
2411            store_id,
2412            resident_len.saturating_add(slack.max(resident_len / 8).max(1)),
2413        );
2414        let resident = {
2415            let tables = self.inner.tables.borrow();
2416            match tables.get(&store_id) {
2417                Some(hooks) => (hooks.resident_keys)(),
2418                None => return,
2419            }
2420        };
2421        let excess = resident.len().saturating_sub(limit);
2422        let candidates: Vec<u64> = {
2423            let dirty = self.inner.dirty.borrow();
2424            if dirty.full.contains(&store_id) {
2425                return; // a wholesale rewrite is pending; every row is implicitly dirty
2426            }
2427            let protect: std::collections::HashSet<u64> = protect.iter().copied().collect();
2428            resident
2429                .into_iter()
2430                .filter(|key| !protect.contains(key) && !dirty.rows.contains_key(&(store_id, *key)))
2431                .collect()
2432        };
2433        let evicted = evict(&candidates, excess);
2434        if evicted > 0 {
2435            let is_resident = {
2436                let tables = self.inner.tables.borrow();
2437                tables.get(&store_id).map(|h| h.is_resident.clone())
2438            };
2439            if let Some(is_resident) = is_resident {
2440                let mut memo = self.inner.blob_memo.borrow_mut();
2441                for key in candidates.iter().filter(|k| !is_resident(**k)) {
2442                    memo.forget_row(store_id, *key);
2443                }
2444            }
2445        }
2446        let remaining = resident_len.saturating_sub(evicted);
2447        self.inner.eviction_retry.borrow_mut().insert(
2448            store_id,
2449            remaining
2450                .max(limit)
2451                .saturating_add(slack.max(remaining / 8).max(1)),
2452        );
2453    }
2454
2455    // --- internals ---------------------------------------------------------------------------
2456
2457    pub(crate) fn conn(&self) -> std::cell::RefMut<'_, Box<dyn SqliteConnection>> {
2458        self.inner.conn.borrow_mut()
2459    }
2460
2461    /// Run a one-column SELECT and hand back the values as key handles. Errors surface on the
2462    /// error signal (these run on read paths with no caller to give a `Result` to).
2463    /// One-column key SELECT with the error returned, for callers that have a `Result` to
2464    /// give it to (the read paths use [`ModelContainer::select_id_column`] instead).
2465    fn select_id_column_checked(&self, sql: &str) -> Result<Vec<u64>, DbError> {
2466        let mut ids = Vec::new();
2467        self.conn().query(sql, &[], &mut |row| {
2468            if let Some(h) = value_to_handle(&row.get(0)) {
2469                ids.push(h);
2470            }
2471        })?;
2472        Ok(ids)
2473    }
2474
2475    fn select_ids_checked(&self, sql: &str, params: &[Value]) -> Result<Vec<u64>, DbError> {
2476        let mut ids = Vec::new();
2477        self.conn().query(sql, params, &mut |row| {
2478            if let Some(h) = value_to_handle(&row.get(0)) {
2479                ids.push(h);
2480            }
2481        })?;
2482        Ok(ids)
2483    }
2484    pub(crate) fn select_id_column(&self, sql: &str, params: &[Value]) -> Vec<u64> {
2485        match self.select_ids_checked(sql, params) {
2486            Ok(ids) => ids,
2487            Err(e) => {
2488                self.inner.error.set(Some(e.to_string()));
2489                Vec::new()
2490            }
2491        }
2492    }
2493
2494    /// Run a one-column SELECT for a single REAL.
2495    pub(crate) fn select_real(&self, sql: &str, params: &[Value]) -> Option<f64> {
2496        let mut v = None;
2497        if let Err(e) = self.conn().query(sql, params, &mut |row| {
2498            v = row.get(0).as_real().ok();
2499        }) {
2500            self.inner.error.set(Some(e.to_string()));
2501        }
2502        v
2503    }
2504
2505    /// This turn's unflushed rows of one store: what relation reads overlay.
2506    pub(crate) fn dirty_rows_of(&self, store_id: u64) -> Vec<(u64, DirtyRow)> {
2507        let dirty = self.inner.dirty.borrow();
2508        let Some(keys) = dirty.by_store.get(&store_id) else {
2509            return Vec::new();
2510        };
2511        keys.iter()
2512            .filter_map(|k| {
2513                dirty
2514                    .rows
2515                    .get(&(store_id, *k))
2516                    .map(|state| (*k, state.clone()))
2517            })
2518            .collect()
2519    }
2520
2521    /// One row's pending state, if any.
2522    pub(crate) fn dirty_state_of(&self, store_id: u64, key: u64) -> Option<DirtyRow> {
2523        self.inner
2524            .dirty
2525            .borrow()
2526            .rows
2527            .get(&(store_id, key))
2528            .cloned()
2529    }
2530
2531    fn ensure_schema_table(&self) -> Result<(), DbError> {
2532        self.conn().execute(
2533            "CREATE TABLE IF NOT EXISTS _day_schema (\
2534             table_name TEXT PRIMARY KEY, fingerprint TEXT NOT NULL, version INTEGER NOT NULL)",
2535            &[],
2536        )?;
2537        Ok(())
2538    }
2539
2540    fn db_version(&self) -> Result<u32, DbError> {
2541        let mut version = 0u32;
2542        self.conn().query(
2543            "SELECT version FROM _day_schema WHERE table_name = '_db'",
2544            &[],
2545            &mut |row| {
2546                version = row.get(0).as_int().unwrap_or(0) as u32;
2547            },
2548        )?;
2549        Ok(version)
2550    }
2551
2552    /// The `_db` version row, written on the caller's connection so it can ride inside the
2553    /// transaction whose effects it labels.
2554    fn write_db_version(conn: &mut dyn SqliteConnection, v: u32) -> Result<(), DbError> {
2555        conn.execute(
2556            "INSERT INTO _day_schema (table_name, fingerprint, version) VALUES ('_db', '', ?) \
2557             ON CONFLICT(table_name) DO UPDATE SET version = excluded.version",
2558            &[Value::Int(i64::from(v))],
2559        )?;
2560        Ok(())
2561    }
2562
2563    /// `f` and the version row it leads to, in one transaction: the stage's effects and its
2564    /// label commit together or not at all. A failure in the stage, in the version write, or
2565    /// in the commit itself takes the one rollback path.
2566    fn versioned_transaction(
2567        &self,
2568        version: u32,
2569        f: impl FnOnce(&mut dyn SqliteConnection) -> Result<(), DbError>,
2570    ) -> Result<(), DbError> {
2571        let mut conn = self.conn();
2572        conn.begin()?;
2573        let result = f(conn.as_mut())
2574            .and_then(|()| Self::write_db_version(conn.as_mut(), version))
2575            .and_then(|()| conn.commit());
2576        if result.is_err() {
2577            let _ = conn.rollback();
2578        }
2579        result
2580    }
2581
2582    fn run_stages(&self, plan: MigrationPlan) -> Result<(), DbError> {
2583        let target = plan.target_version();
2584        let mut current = self.db_version()?;
2585        if current > target && target > 0 {
2586            return Err(DbError::new(
2587                DbErrorKind::Schema,
2588                format!(
2589                    "database is at version {current}, newer than this build's {target} — \
2590                     refusing to open (an old app writing a new schema is how data rots)"
2591                ),
2592            ));
2593        }
2594        let mut stages = plan.stages;
2595        stages.sort_by_key(|s| s.from);
2596        for stage in stages {
2597            if stage.from != current {
2598                if stage.to <= current {
2599                    continue; // already applied
2600                }
2601                return Err(DbError::new(
2602                    DbErrorKind::Schema,
2603                    format!(
2604                        "migration stage {}→{} does not start at the database's version {current}",
2605                        stage.from, stage.to
2606                    ),
2607                ));
2608            }
2609            self.versioned_transaction(stage.to, stage.run)?;
2610            current = stage.to;
2611        }
2612        if target > 0 && current != target {
2613            self.versioned_transaction(target, |_| Ok(()))?;
2614        }
2615        Ok(())
2616    }
2617
2618    /// Create (or lightweight-migrate) `M`'s table and register its hooks. The cache starts
2619    /// empty: this is the line where the old engine read the whole table, and the lazy one
2620    /// exists so that nothing does.
2621    fn attach<M: Model>(&self) -> Result<(), DbError> {
2622        // FTS5's external-content table and the R*Tree address rows by ROWID (an i64), so
2623        // both need the key column to be the integer rowid. A wide-keyed model declaring one
2624        // is refused at open, naming the constraint, rather than silently mis-indexing.
2625        let key_sql = M::COLUMNS
2626            .iter()
2627            .find(|c| c.name == M::KEY)
2628            .map_or(SqlType::Integer, |c| c.sql);
2629        if key_sql != SqlType::Integer && M::SPATIAL.is_some() {
2630            return Err(DbError::new(
2631                DbErrorKind::Unsupported,
2632                format!(
2633                    "`{}` declares spatial(…), which addresses rows by ROWID — those \
2634                     need an integer `#[model(id)]`, not a Uuid or String key",
2635                    M::TABLE
2636                ),
2637            ));
2638        }
2639        // A second attach of the same model (a re-attached database after a catalog swap)
2640        // keeps its store and hooks; only the resident rows are stale, and they are dropped
2641        // so the next fault reads the new file.
2642        if self.inner.stores.borrow().contains_key(&TypeId::of::<M>()) {
2643            let store = self.cache::<M>();
2644            let keys: Vec<u64> = store.with_untracked(day_model::Keyed::keys);
2645            store.depopulate_many(&keys);
2646            return Ok(());
2647        }
2648        if !M::EXTERNAL {
2649            self.ensure_table::<M>()?;
2650        }
2651        for l in M::LINKS {
2652            self.inner.links.borrow_mut().push(LinkInfo {
2653                owner_table: M::TABLE,
2654                field: l.field,
2655                local: l.local,
2656                target_table: l.target_table,
2657                remote: l.remote,
2658            });
2659        }
2660
2661        let store: Store<Keyed<M>> = Store::new(Keyed::new(Vec::new()));
2662        let store_id = store.store_id();
2663
2664        let columns: Vec<String> = M::COLUMNS.iter().map(|c| c.name.to_string()).collect();
2665        let fields: Vec<String> = M::COLUMNS.iter().map(|c| c.field.to_string()).collect();
2666        let row_for = {
2667            Rc::new(move |key: u64| store.with_untracked(|k| k.get(key).map(Model::to_row)))
2668                as Rc<dyn Fn(u64) -> Option<Vec<Value>>>
2669        };
2670        let all_rows = {
2671            Rc::new(move || {
2672                store.with_untracked(|k| {
2673                    k.items().iter().map(|m| (m.handle(), m.to_row())).collect()
2674                })
2675            }) as Rc<dyn Fn() -> Vec<(u64, Vec<Value>)>>
2676        };
2677        let resident_keys = Rc::new(move || store.with_untracked(day_model::Keyed::keys))
2678            as Rc<dyn Fn() -> Vec<u64>>;
2679        let resident_len =
2680            Rc::new(move || store.with_untracked(day_model::Keyed::len)) as Rc<dyn Fn() -> usize>;
2681        let is_resident = Rc::new(move |h: u64| store.with_untracked(|k| k.get(h).is_some()))
2682            as Rc<dyn Fn(u64) -> bool>;
2683
2684        let absorb = {
2685            Rc::new(
2686                move |raw_rows: Vec<Vec<Value>>| -> Result<Vec<u64>, DbError> {
2687                    let mut decoded: Vec<M> = Vec::with_capacity(raw_rows.len());
2688                    for r in raw_rows {
2689                        decoded.push(M::from_row(&r)?);
2690                    }
2691                    let keys: Vec<u64> =
2692                        decoded.iter().map(day_model::Identified::handle).collect();
2693                    store.populate(decoded);
2694                    Ok(keys)
2695                },
2696            ) as AbsorbFn
2697        };
2698        let refresh = {
2699            Rc::new(move |raw_rows: Vec<Vec<Value>>| -> Result<bool, DbError> {
2700                let mut fresh: Vec<M> = Vec::with_capacity(raw_rows.len());
2701                for r in raw_rows {
2702                    fresh.push(M::from_row(&r)?);
2703                }
2704                let existing: Vec<u64> = store.with_untracked(day_model::Keyed::keys);
2705                let fresh_keys: std::collections::HashSet<u64> =
2706                    fresh.iter().map(day_model::Identified::handle).collect();
2707                let mut changed = false;
2708                day_model::with_author(ModelContainer::EXTERNAL_AUTHOR, || {
2709                    for m in fresh {
2710                        let key = m.handle();
2711                        match store.with_untracked(|k| k.get(key).map(Model::to_row)) {
2712                            // Selected by resident keys, so an unknown row means the cache
2713                            // moved underneath us; treat it as an arrival.
2714                            None => {
2715                                changed = true;
2716                                store.restructure("external", Op::Insert, key, |k| k.push(m));
2717                            }
2718                            Some(old) => {
2719                                let new_row = m.to_row();
2720                                let labels: Vec<&'static str> = M::COLUMNS
2721                                    .iter()
2722                                    .enumerate()
2723                                    .filter(|(i, _)| Row::get(&old, *i) != Row::get(&new_row, *i))
2724                                    .map(|(_, c)| c.field)
2725                                    .collect();
2726                                if !labels.is_empty() {
2727                                    changed = true;
2728                                    store.merge_row(key, m, &labels);
2729                                }
2730                            }
2731                        }
2732                    }
2733                    for gone in existing.into_iter().filter(|k| !fresh_keys.contains(k)) {
2734                        changed = true;
2735                        store.restructure("external", Op::Delete, gone, |k| {
2736                            k.remove(gone);
2737                        });
2738                    }
2739                });
2740                Ok(changed)
2741            }) as RefreshFn
2742        };
2743        let evict = Rc::new(move |candidates: &[u64], want: usize| {
2744            let doomed: Vec<u64> = candidates
2745                .iter()
2746                .copied()
2747                .filter(|h| !store.is_observed(*h))
2748                .take(want)
2749                .collect();
2750            store.depopulate_many(&doomed)
2751        }) as EvictFn;
2752        let watch_undo = Rc::new(move |stack: &day_model::UndoStack| stack.watch(store))
2753            as Rc<dyn Fn(&day_model::UndoStack)>;
2754        // After a flush: the pending bytes of these rows' lazy fields are in the file, so the
2755        // fields settle to stored, under the database's own author so nothing re-dirties.
2756        let settle = Rc::new(move |keys: &[u64]| {
2757            let mut out = Vec::new();
2758            if M::LAZY_FIELDS.is_empty() {
2759                return out;
2760            }
2761            day_model::with_author(ModelContainer::EXTERNAL_AUTHOR, || {
2762                for &key in keys {
2763                    let Some(mut row) = store.with_untracked(|k| k.get(key).cloned()) else {
2764                        continue;
2765                    };
2766                    let taken = row.settle_lazy();
2767                    if taken.is_empty() {
2768                        continue;
2769                    }
2770                    store.merge_row(key, row, M::LAZY_FIELDS);
2771                    out.extend(taken.into_iter().map(|(f, b)| (key, f, b)));
2772                }
2773            });
2774            out
2775        }) as SettleFn;
2776
2777        self.inner.tables.borrow_mut().insert(
2778            store_id,
2779            TableHooks {
2780                table: M::TABLE,
2781                name: TableName::of::<M>(),
2782                key_cols: vec![M::KEY.to_string()],
2783                key_where: Rc::new(|h| (format!("{} = ?", ident(M::KEY)), vec![key_param(h)])),
2784                columns,
2785                column_names: M::COLUMNS.iter().map(|c| c.name).collect(),
2786                fields,
2787                lazy: M::COLUMNS.iter().map(|c| c.lazy).collect(),
2788                settle,
2789                resident_len,
2790                fts: (!M::FTS_COLUMNS.is_empty()).then(|| format!("{}_fts", M::TABLE)),
2791                fts_key: if key_sql == SqlType::Integer {
2792                    "rowid"
2793                } else {
2794                    "day_key"
2795                },
2796                spatial: M::SPATIAL.map(|s| {
2797                    (
2798                        s.lat.to_string(),
2799                        s.lon.to_string(),
2800                        format!("{}_geo", M::TABLE),
2801                    )
2802                }),
2803                row_for,
2804                all_rows,
2805                resident_keys,
2806                is_resident,
2807                absorb,
2808                refresh,
2809                evict,
2810                #[cfg(not(target_arch = "wasm32"))]
2811                dispose_cache: Rc::new(move || store.dispose_contents()),
2812                watch_undo,
2813            },
2814        );
2815        self.inner
2816            .stores
2817            .borrow_mut()
2818            .insert(TypeId::of::<M>(), Box::new(store));
2819        Ok(())
2820    }
2821
2822    fn ensure_table<M: Model>(&self) -> Result<(), DbError> {
2823        // Relation-contributed clauses fold into the fingerprint: adopting or changing a
2824        // delete rule re-runs this path. An existing table cannot gain the SQL-level clause
2825        // in place (SQLite has no `ALTER ADD CONSTRAINT`), so the comparison below routes
2826        // that difference through the table rebuild, which writes the declared DDL whole.
2827        let mut fp_val = model_fingerprint::<M>();
2828        for s in self
2829            .inner
2830            .fk_specs
2831            .borrow()
2832            .iter()
2833            .filter(|s| s.child_table == M::TABLE)
2834        {
2835            for b in s
2836                .child_field
2837                .bytes()
2838                .chain(s.parent_table.bytes())
2839                .chain(s.rule.sql().bytes())
2840            {
2841                fp_val ^= u64::from(b);
2842                fp_val = fp_val.wrapping_mul(0x1000_0000_01b3);
2843            }
2844        }
2845        let fp = format!("{fp_val:016x}");
2846        let stored = self.stored_fingerprint(M::TABLE)?;
2847        if stored.as_deref() == Some(fp.as_str()) {
2848            // The layout is known to match; the generated triggers are still reconciled, so
2849            // updated trigger logic reaches an existing file (an unchanged trigger is a read
2850            // and no write).
2851            self.conn().begin()?;
2852            let result = self
2853                .create_shadow_tables::<M>()
2854                .and_then(|()| self.conn().commit());
2855            if result.is_err() {
2856                let _ = self.conn().rollback();
2857            }
2858            return result;
2859        }
2860
2861        let name = TableName::of::<M>();
2862        // A missing table is created whole; an existing one is compared column by column,
2863        // index by index, and the comparison decides between in-place DDL and a rebuild,
2864        // or refuses before anything is touched.
2865        let delta = if self.table_exists(M::TABLE)? {
2866            let file = self.file_schema(name)?;
2867            Some((self.compare_schema::<M>(&file)?, file))
2868        } else {
2869            None
2870        };
2871        // A rebuild drops and recreates the table, which `PRAGMA foreign_keys` would treat
2872        // as a delete of every row (cascading into children). The pragma is a no-op inside
2873        // a transaction, so it toggles around one, and the rebuild's own
2874        // `foreign_key_check` is what guarantees nothing dangles afterwards.
2875        let rebuild = matches!(delta, Some((SchemaDelta::Rebuild, _)));
2876        let fk_was_on = rebuild && self.foreign_keys_enabled()?;
2877        if fk_was_on {
2878            self.conn().execute("PRAGMA foreign_keys = OFF", &[])?;
2879        }
2880
2881        // DDL, copy and backfill are one transaction: a failed migration leaves the old
2882        // table, index and fingerprint usable. Remove triggers before dropping indexed
2883        // columns.
2884        // Bound first: the `RefMut` the `begin` call holds must be gone before the closure
2885        // borrows the connection again.
2886        let began = self.conn().begin();
2887        let result = began.and_then(|()| {
2888            let result = (|| {
2889                let t = name.bare;
2890                if self.table_exists(&format!("{t}_fts"))? {
2891                    for suffix in ["ai", "ad", "au"] {
2892                        self.conn().execute(
2893                            &format!(
2894                                "DROP TRIGGER IF EXISTS {}",
2895                                ident(&format!("{t}_fts_{suffix}"))
2896                            ),
2897                            &[],
2898                        )?;
2899                    }
2900                    self.conn()
2901                        .execute(&format!("DROP TABLE {}", name.shadow("_fts")), &[])?;
2902                    self.conn().execute(
2903                        &format!("DROP TABLE IF EXISTS {}", name.shadow("_fts_keys")),
2904                        &[],
2905                    )?;
2906                }
2907                let mut unique_indexes: Vec<&'static str> = Vec::new();
2908                match delta {
2909                    None => self.create_table_named::<M>(&name.sql())?,
2910                    Some((SchemaDelta::Lightweight(plan), _)) => {
2911                        unique_indexes.clone_from(&plan.unique_indexes);
2912                        self.lightweight_migrate::<M>(&plan)?;
2913                    }
2914                    Some((SchemaDelta::Rebuild, file)) => {
2915                        self.rebuild_table::<M>(&file)?;
2916                    }
2917                }
2918                self.create_indexes::<M>(&unique_indexes)?;
2919                if rebuild {
2920                    let dangling = Self::foreign_key_check_on(self.conn().as_mut())?;
2921                    if let Some(first) = dangling.first() {
2922                        return Err(DbError::new(
2923                            DbErrorKind::Schema,
2924                            format!(
2925                                "rebuilding `{}` leaves {} row(s) with a dangling foreign \
2926                                 key ({first}); the file is unchanged",
2927                                M::TABLE,
2928                                dangling.len()
2929                            ),
2930                        ));
2931                    }
2932                }
2933                self.create_shadow_tables::<M>()?;
2934                // The fingerprint is the last write: it records a layout that has been
2935                // brought to the declaration and verified, never one still in flight.
2936                self.store_fingerprint(M::TABLE, &fp)?;
2937                self.conn().commit()
2938            })();
2939            if result.is_err() {
2940                let _ = self.conn().rollback();
2941            }
2942            result
2943        });
2944        if fk_was_on {
2945            let restored = self
2946                .conn()
2947                .execute("PRAGMA foreign_keys = ON", &[])
2948                .map(|_| ());
2949            return result.and(restored);
2950        }
2951        result
2952    }
2953
2954    /// Whether this connection enforces foreign keys right now (`PRAGMA foreign_keys`). A
2955    /// driver that answers nothing (the Recorder) counts as not enforcing.
2956    fn foreign_keys_enabled(&self) -> Result<bool, DbError> {
2957        let mut on = false;
2958        self.conn().query("PRAGMA foreign_keys", &[], &mut |row| {
2959            on = row.get(0).as_int().unwrap_or(0) != 0;
2960        })?;
2961        Ok(on)
2962    }
2963
2964    /// The FTS5 and R*Tree shadows a model declares: virtual tables plus `AFTER` triggers, so
2965    /// the indexes stay true inside the same transaction as every write, even a write made by
2966    /// another tool straight into the file. Backfilled when first created.
2967    fn create_shadow_tables<M: Model>(&self) -> Result<(), DbError> {
2968        let name = TableName::of::<M>();
2969        // The plain names, for messages and `sqlite_master` lookups; the quoted ones, for
2970        // every statement. A shadow's trigger is named after the shadow, quoted as one name.
2971        let bare = name.bare;
2972        let t = name.sql();
2973        let key = ident(M::KEY);
2974        let trigger = |suffix: &str| ident(&format!("{bare}{suffix}"));
2975        if !M::FTS_COLUMNS.is_empty() {
2976            if !self.inner.caps.full_text_search {
2977                return Err(DbError::new(
2978                    DbErrorKind::Unsupported,
2979                    format!(
2980                        "`{bare}` declares fts(…) but this driver's engine has no FTS5 — the \
2981                         bundled and cipher engines compile it in"
2982                    ),
2983                ));
2984            }
2985            let fts = name.shadow("_fts");
2986            // The hidden column FTS5 takes its commands through carries the index's own name.
2987            let fts_command = ident(&format!("{bare}_fts"));
2988            let cols = column_list(M::FTS_COLUMNS);
2989            let fresh = !self.table_exists(&format!("{bare}_fts"))?;
2990            // The tokenizer rides in the DDL; its value is a compile-time literal, quoted
2991            // through SQL's own escape so an apostrophe cannot break the statement.
2992            let tokenize = match M::FTS_TOKENIZE {
2993                Some(tok) => format!(", tokenize='{}'", tok.replace('\'', "''")),
2994                None => String::new(),
2995            };
2996            let new_cols = prefixed_list("new", M::FTS_COLUMNS);
2997            let wide = M::COLUMNS
2998                .iter()
2999                .find(|c| c.name == M::KEY)
3000                .is_some_and(|c| c.sql != SqlType::Integer);
3001            if wide {
3002                if M::FTS_COLUMNS.contains(&"day_key") {
3003                    return Err(DbError::new(
3004                        DbErrorKind::Unsupported,
3005                        "day_key is reserved in a non-integer-key FTS index",
3006                    ));
3007                }
3008                let key_type = M::COLUMNS
3009                    .iter()
3010                    .find(|c| c.name == M::KEY)
3011                    .expect("key column")
3012                    .sql
3013                    .ddl();
3014                let fts_keys = name.shadow("_fts_keys");
3015                // FTS rowids stay private. An indexed mapping makes update/delete O(log n)
3016                // rather than scanning the unindexed public key in every FTS row.
3017                self.conn().execute(&format!("CREATE TABLE IF NOT EXISTS {fts_keys}(day_rowid INTEGER PRIMARY KEY, day_key {key_type} NOT NULL UNIQUE)"), &[])?;
3018                self.conn().execute(&format!("CREATE VIRTUAL TABLE IF NOT EXISTS {fts} USING fts5(day_key UNINDEXED, {cols}{tokenize})"), &[])?;
3019                let insert = format!(
3020                    "INSERT INTO {fts_keys}(day_key) VALUES (new.{key}); INSERT INTO {fts}(rowid, day_key, {cols}) VALUES ((SELECT day_rowid FROM {fts_keys} WHERE day_key = new.{key}), new.{key}, {new_cols});"
3021                );
3022                let delete = format!(
3023                    "DELETE FROM {fts} WHERE rowid = (SELECT day_rowid FROM {fts_keys} WHERE day_key = old.{key}); DELETE FROM {fts_keys} WHERE day_key = old.{key};"
3024                );
3025                let (update_of, changed) = fts_update_guard(M::KEY, M::FTS_COLUMNS);
3026                for (suffix, event, body) in [
3027                    ("_fts_ai", "INSERT".to_string(), insert.clone()),
3028                    ("_fts_ad", "DELETE".to_string(), delete.clone()),
3029                    (
3030                        "_fts_au",
3031                        format!("UPDATE OF {update_of}"),
3032                        format!("{delete} {insert}"),
3033                    ),
3034                ] {
3035                    let when = if suffix == "_fts_au" {
3036                        format!(" WHEN {changed}")
3037                    } else {
3038                        String::new()
3039                    };
3040                    self.ensure_trigger(
3041                        &format!("{bare}{suffix}"),
3042                        name.schema,
3043                        &format!(
3044                            "CREATE TRIGGER {} AFTER {event} ON {t}{when} BEGIN {body} END",
3045                            trigger(suffix)
3046                        ),
3047                    )?;
3048                }
3049                if fresh {
3050                    self.conn().execute(
3051                        &format!("INSERT INTO {fts_keys}(day_key) SELECT {key} FROM {t}"),
3052                        &[],
3053                    )?;
3054                    let qualified = prefixed_list("b", M::FTS_COLUMNS);
3055                    self.conn().execute(&format!("INSERT INTO {fts}(rowid, day_key, {cols}) SELECT k.day_rowid, b.{key}, {qualified} FROM {t} b JOIN {fts_keys} k ON k.day_key = b.{key}"), &[])?;
3056                }
3057            } else {
3058                // FTS5 reads its options as SQL string literals; the content table and its
3059                // rowid column are plain names inside them.
3060                self.conn().execute(
3061                    &format!(
3062                        "CREATE VIRTUAL TABLE IF NOT EXISTS {fts} USING fts5({cols}, \
3063                     content='{}', content_rowid='{}'{tokenize})",
3064                        bare.replace('\'', "''"),
3065                        M::KEY.replace('\'', "''")
3066                    ),
3067                    &[],
3068                )?;
3069                let old_cols = prefixed_list("old", M::FTS_COLUMNS);
3070                self.ensure_trigger(
3071                    &format!("{bare}_fts_ai"),
3072                    name.schema,
3073                    &format!(
3074                        "CREATE TRIGGER {} AFTER INSERT ON {t} BEGIN \
3075                     INSERT INTO {fts}(rowid, {cols}) VALUES (new.{key}, {new_cols}); END",
3076                        trigger("_fts_ai")
3077                    ),
3078                )?;
3079                self.ensure_trigger(
3080                    &format!("{bare}_fts_ad"),
3081                    name.schema,
3082                    &format!(
3083                        "CREATE TRIGGER {} AFTER DELETE ON {t} BEGIN \
3084                     INSERT INTO {fts}({fts_command}, rowid, {cols}) VALUES ('delete', old.{key}, {old_cols}); END",
3085                        trigger("_fts_ad")
3086                    ),
3087                )?;
3088                // Only a change to indexed content or to the row's identity re-indexes it: an
3089                // update that touches other columns, or assigns an indexed column its own
3090                // value, leaves the index alone.
3091                let (update_of, changed) = fts_update_guard(M::KEY, M::FTS_COLUMNS);
3092                self.ensure_trigger(
3093                    &format!("{bare}_fts_au"),
3094                    name.schema,
3095                    &format!(
3096                        "CREATE TRIGGER {} AFTER UPDATE OF {update_of} ON {t} WHEN {changed} BEGIN \
3097                     INSERT INTO {fts}({fts_command}, rowid, {cols}) VALUES ('delete', old.{key}, {old_cols}); \
3098                     INSERT INTO {fts}(rowid, {cols}) VALUES (new.{key}, {new_cols}); END",
3099                        trigger("_fts_au")
3100                    ),
3101                )?;
3102                if fresh {
3103                    // External-content rebuild: index whatever rows predate the declaration.
3104                    self.conn().execute(
3105                        &format!("INSERT INTO {fts}({fts_command}) VALUES ('rebuild')"),
3106                        &[],
3107                    )?;
3108                }
3109            }
3110        }
3111        if let Some(s) = M::SPATIAL {
3112            if !self.inner.caps.rtree {
3113                return Err(DbError::new(
3114                    DbErrorKind::Unsupported,
3115                    format!(
3116                        "`{bare}` declares spatial(…) but this driver's engine has no R*Tree — \
3117                         the bundled and cipher engines compile it in"
3118                    ),
3119                ));
3120            }
3121            let (lat, lon) = (ident(s.lat), ident(s.lon));
3122            let geo = name.shadow("_geo");
3123            let fresh = !self.table_exists(&format!("{bare}_geo"))?;
3124            self.conn().execute(
3125                &format!(
3126                    "CREATE VIRTUAL TABLE IF NOT EXISTS {geo} USING rtree({key}, \
3127                     min_lat, max_lat, min_lon, max_lon)"
3128                ),
3129                &[],
3130            )?;
3131            self.ensure_trigger(
3132                &format!("{bare}_geo_ai"),
3133                name.schema,
3134                &format!(
3135                    "CREATE TRIGGER {} AFTER INSERT ON {t} BEGIN \
3136                     INSERT OR REPLACE INTO {geo} VALUES (new.{key}, new.{lat}, new.{lat}, new.{lon}, new.{lon}); END",
3137                    trigger("_geo_ai")
3138                ),
3139            )?;
3140            self.ensure_trigger(
3141                &format!("{bare}_geo_au"),
3142                name.schema,
3143                &format!(
3144                    "CREATE TRIGGER {} AFTER UPDATE OF {lat}, {lon} ON {t} BEGIN \
3145                     INSERT OR REPLACE INTO {geo} VALUES (new.{key}, new.{lat}, new.{lat}, new.{lon}, new.{lon}); END",
3146                    trigger("_geo_au")
3147                ),
3148            )?;
3149            self.ensure_trigger(
3150                &format!("{bare}_geo_ad"),
3151                name.schema,
3152                &format!(
3153                    "CREATE TRIGGER {} AFTER DELETE ON {t} BEGIN \
3154                     DELETE FROM {geo} WHERE {key} = old.{key}; END",
3155                    trigger("_geo_ad")
3156                ),
3157            )?;
3158            if fresh {
3159                self.conn().execute(
3160                    &format!(
3161                        "INSERT OR REPLACE INTO {geo} \
3162                         SELECT {key}, {lat}, {lat}, {lon}, {lon} FROM {t}"
3163                    ),
3164                    &[],
3165                )?;
3166            }
3167        }
3168        Ok(())
3169    }
3170
3171    /// Install a generated trigger, replacing one whose stored text differs: updated trigger
3172    /// logic reaches an existing file on the open that carries it, and a trigger that is
3173    /// already right costs one `sqlite_master` read and no write. `create_sql` is the plain
3174    /// `CREATE TRIGGER` statement; the engine stores that text verbatim, which is what the
3175    /// comparison reads back.
3176    fn ensure_trigger(
3177        &self,
3178        bare_name: &str,
3179        schema: Option<&str>,
3180        create_sql: &str,
3181    ) -> Result<(), DbError> {
3182        let master = table_sql(schema, "sqlite_master");
3183        let mut stored: Option<String> = None;
3184        self.conn().query(
3185            &format!("SELECT sql FROM {master} WHERE type = 'trigger' AND name = ?"),
3186            &[Value::Text(bare_name.into())],
3187            &mut |row| {
3188                if let Ok(sql) = row.get(0).as_text() {
3189                    stored = Some(sql.to_string());
3190                }
3191            },
3192        )?;
3193        match stored {
3194            Some(sql) if sql == create_sql => return Ok(()),
3195            Some(_) => {
3196                self.conn().execute(
3197                    &format!("DROP TRIGGER IF EXISTS {}", table_sql(schema, bare_name)),
3198                    &[],
3199                )?;
3200            }
3201            None => {}
3202        }
3203        self.conn().execute(create_sql, &[])?;
3204        Ok(())
3205    }
3206
3207    fn table_exists(&self, table: &str) -> Result<bool, DbError> {
3208        let mut found = false;
3209        self.conn().query(
3210            "SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?",
3211            &[Value::Text(table.into())],
3212            &mut |_| found = true,
3213        )?;
3214        Ok(found)
3215    }
3216
3217    /// `M`'s declared DDL under `table_sql` (its own quoted name, or the `_new` name a
3218    /// rebuild copies into). The fresh form carries every constraint inline; the upgrade
3219    /// paths produce the same enforced constraints by other means (a `NOT NULL DEFAULT`
3220    /// addition, a `day_uidx_` unique index), and the comparison treats them as equal.
3221    fn create_table_named<M: Model>(&self, table_sql: &str) -> Result<(), DbError> {
3222        let cols: Vec<String> = self
3223            .declared_columns::<M>()
3224            .iter()
3225            .map(|d| {
3226                let c = &d.def;
3227                let mut s = format!("{} {}", ident(c.name), c.sql.ddl());
3228                if d.pk {
3229                    s.push_str(" PRIMARY KEY");
3230                } else if c.not_null {
3231                    s.push_str(" NOT NULL");
3232                }
3233                if c.unique && !d.pk {
3234                    s.push_str(" UNIQUE");
3235                }
3236                // A declared relation's foreign key: enforced by the engine too, so another
3237                // process honors the same delete rule, and so a cascade's reach into rows
3238                // this process never faulted still lands in the file. Deferred, so
3239                // within-transaction statement order (a cascade's children, an undo's
3240                // re-inserts) never trips it.
3241                if let Some(fk) = &d.fk {
3242                    s.push_str(&fk.clause());
3243                }
3244                s
3245            })
3246            .collect();
3247        let body = format!("CREATE TABLE {table_sql} ({})", cols.join(", "));
3248        // `STRICT` where the engine allows it (always, for the bundled build); older system
3249        // SQLites reject the keyword, and the plain form is the fallback.
3250        if self.conn().execute(&format!("{body} STRICT"), &[]).is_err() {
3251            self.conn().execute(&body, &[])?;
3252        }
3253        Ok(())
3254    }
3255
3256    /// The indexes `M` owns in the file, by name: one per `indexed` column, one per declared
3257    /// relation's foreign-key column (what `children_of` selects on, what relation predicates
3258    /// correlate on, and what the engine scans to enforce `ON DELETE`; unindexed, each is a
3259    /// full scan of the child table), one per composite declaration, and a unique index for
3260    /// each column in `unique_indexes`, the declared-unique columns whose table carries no
3261    /// inline `UNIQUE` because they arrived through `ADD COLUMN`.
3262    fn declared_indexes<M: Model>(&self, unique_indexes: &[&'static str]) -> Vec<DeclaredIndex> {
3263        let fk_cols: Vec<&str> = self
3264            .inner
3265            .fk_specs
3266            .borrow()
3267            .iter()
3268            .filter(|s| s.child_table == M::TABLE)
3269            .filter_map(|s| {
3270                M::COLUMNS
3271                    .iter()
3272                    .find(|c| c.field == s.child_field)
3273                    .map(|c| c.name)
3274            })
3275            .collect();
3276        let bare = TableName::of::<M>().bare;
3277        let mut out: Vec<DeclaredIndex> = M::COLUMNS
3278            .iter()
3279            .filter(|c| c.indexed || (fk_cols.contains(&c.name) && c.name != M::KEY))
3280            .map(|c| DeclaredIndex {
3281                name: format!("day_idx_{bare}_{}", c.name),
3282                columns: vec![c.name.to_string()],
3283                unique: false,
3284            })
3285            .collect();
3286        out.extend(M::COMPOSITE_INDEXES.iter().map(|idx| DeclaredIndex {
3287            name: format!("day_idx_{bare}_{}", idx.join("_")),
3288            columns: idx.iter().map(std::string::ToString::to_string).collect(),
3289            unique: false,
3290        }));
3291        out.extend(unique_indexes.iter().map(|c| DeclaredIndex {
3292            name: format!("day_uidx_{bare}_{c}"),
3293            columns: vec![c.to_string()],
3294            unique: true,
3295        }));
3296        out.extend(M::UNIQUE_INDEXES.iter().map(|idx| DeclaredIndex {
3297            name: format!("day_uidx_{bare}_{}", idx.join("_")),
3298            columns: idx.iter().map(std::string::ToString::to_string).collect(),
3299            unique: true,
3300        }));
3301        out
3302    }
3303
3304    fn create_indexes<M: Model>(&self, unique_indexes: &[&'static str]) -> Result<(), DbError> {
3305        let name = TableName::of::<M>();
3306        let t = name.sql();
3307        for idx in self.declared_indexes::<M>(unique_indexes) {
3308            self.conn().execute(
3309                &format!(
3310                    "CREATE {}INDEX IF NOT EXISTS {} ON {t}({})",
3311                    if idx.unique { "UNIQUE " } else { "" },
3312                    table_sql(name.schema, &idx.name),
3313                    column_list(&idx.columns)
3314                ),
3315                &[],
3316            )?;
3317        }
3318        Ok(())
3319    }
3320
3321    /// `M`'s columns as declared, with the relation-contributed foreign key resolved: the
3322    /// declared side of [`ModelContainer::compare_schema`], and what the DDL is written from.
3323    fn declared_columns<M: Model>(&self) -> Vec<DeclaredColumn> {
3324        let specs = self.inner.fk_specs.borrow();
3325        M::COLUMNS
3326            .iter()
3327            .enumerate()
3328            .map(|(index, c)| DeclaredColumn {
3329                def: *c,
3330                index,
3331                pk: c.name == M::KEY,
3332                fk: specs
3333                    .iter()
3334                    .find(|f| f.child_table == M::TABLE && f.child_field == c.field)
3335                    .map(|s| DeclaredFk {
3336                        parent_sql: s.parent_sql.clone(),
3337                        parent_bare: s.parent_bare,
3338                        parent_key: s.parent_key,
3339                        rule: s.rule,
3340                    }),
3341            })
3342            .collect()
3343    }
3344
3345    /// The table as the file has it: `PRAGMA table_info`, `foreign_key_list`, `index_list`
3346    /// and `index_info`, read into one value the comparison and the rebuild both consume.
3347    fn file_schema(&self, name: TableName) -> Result<FileSchema, DbError> {
3348        let mut columns: Vec<FileColumn> = Vec::new();
3349        self.conn().query(
3350            &pragma_sql(name.schema, "table_info", name.bare),
3351            &[],
3352            &mut |row| {
3353                columns.push(FileColumn {
3354                    name: row.get(1).as_text().unwrap_or("").to_string(),
3355                    decl_type: row.get(2).as_text().unwrap_or("").to_ascii_uppercase(),
3356                    not_null: row.get(3).as_int().unwrap_or(0) != 0,
3357                    pk: row.get(5).as_int().unwrap_or(0) != 0,
3358                    fk: None,
3359                });
3360            },
3361        )?;
3362        self.conn().query(
3363            &pragma_sql(name.schema, "foreign_key_list", name.bare),
3364            &[],
3365            &mut |row| {
3366                let from = row.get(3).as_text().unwrap_or("").to_string();
3367                if let Some(col) = columns.iter_mut().find(|c| c.name == from) {
3368                    col.fk = Some(FileFk {
3369                        parent: row.get(2).as_text().unwrap_or("").to_string(),
3370                        parent_key: row.get(4).as_text().unwrap_or("").to_string(),
3371                        on_delete: row.get(6).as_text().unwrap_or("").to_ascii_uppercase(),
3372                    });
3373                }
3374            },
3375        )?;
3376        let mut indexes: Vec<FileIndex> = Vec::new();
3377        self.conn().query(
3378            &pragma_sql(name.schema, "index_list", name.bare),
3379            &[],
3380            &mut |row| {
3381                indexes.push(FileIndex {
3382                    name: row.get(1).as_text().unwrap_or("").to_string(),
3383                    unique: row.get(2).as_int().unwrap_or(0) != 0,
3384                    origin: row.get(3).as_text().unwrap_or("").to_string(),
3385                    columns: Vec::new(),
3386                });
3387            },
3388        )?;
3389        for idx in &mut indexes {
3390            let mut cols: Vec<(i64, String)> = Vec::new();
3391            self.conn().query(
3392                &pragma_sql(name.schema, "index_info", &idx.name),
3393                &[],
3394                &mut |row| {
3395                    cols.push((
3396                        row.get(0).as_int().unwrap_or(0),
3397                        row.get(2).as_text().unwrap_or("").to_string(),
3398                    ));
3399                },
3400            )?;
3401            cols.sort_by_key(|(seq, _)| *seq);
3402            idx.columns = cols.into_iter().map(|(_, c)| c).collect();
3403        }
3404        Ok(FileSchema { columns, indexes })
3405    }
3406
3407    /// Decide how the file's table reaches the declaration. Column and index additions and
3408    /// drops are closed in place; any other difference (a type, a nullability, a unique
3409    /// constraint, a foreign key or its delete action) is a rebuild; a change the rebuild
3410    /// could not fill truthfully (the key column, a required column with no usable default)
3411    /// is refused here, before anything is touched.
3412    fn compare_schema<M: Model>(&self, file: &FileSchema) -> Result<SchemaDelta, DbError> {
3413        let declared = self.declared_columns::<M>();
3414        let defaults = M::default_row();
3415        let bare = TableName::of::<M>().bare;
3416        let refuse = |column: &str, what: String| {
3417            Err(DbError::new(
3418                DbErrorKind::Schema,
3419                format!(
3420                    "`{}`.`{column}`: {what}; this is not a change the container can make to \
3421                     an existing table, and it refuses to open rather than guess (a staged \
3422                     migration can make it)",
3423                    M::TABLE
3424                ),
3425            ))
3426        };
3427        let mut plan = LightweightPlan::default();
3428        // Any one of these differences needs the copy; which one is immaterial to the work.
3429        let mut rebuild = false;
3430        let mut note = |_why: String| rebuild = true;
3431        let inline_unique = |col: &str| {
3432            file.indexes
3433                .iter()
3434                .any(|i| i.origin == "u" && i.columns.len() == 1 && i.columns[0] == col)
3435        };
3436        let unique_index_name = |col: &str| format!("day_uidx_{bare}_{col}");
3437        let has_unique_index = |col: &str| {
3438            let name = unique_index_name(col);
3439            file.indexes
3440                .iter()
3441                .any(|i| i.name == name && i.unique && i.columns.len() == 1 && i.columns[0] == col)
3442        };
3443
3444        for d in &declared {
3445            let c = &d.def;
3446            let default = Row::get(&defaults, d.index);
3447            let required = c.not_null || d.pk;
3448            match file.columns.iter().find(|f| f.name == c.name) {
3449                None => {
3450                    if d.pk {
3451                        return refuse(c.name, "the declared key column is not in the file".into());
3452                    }
3453                    if required && default == Value::Null {
3454                        return refuse(
3455                            c.name,
3456                            "a required column whose Rust default stores NULL cannot be \
3457                             added to existing rows"
3458                                .into(),
3459                        );
3460                    }
3461                    if d.fk.is_some() && required {
3462                        // `ADD COLUMN … REFERENCES` must default to NULL while foreign keys
3463                        // are enforced; a required reference needs the copy.
3464                        note(format!("required foreign-key column `{}` added", c.name));
3465                    } else {
3466                        plan.add.push(d.index);
3467                        if c.unique {
3468                            plan.unique_indexes.push(c.name);
3469                        }
3470                    }
3471                }
3472                Some(f) => {
3473                    if f.decl_type != c.sql.ddl() {
3474                        if d.pk {
3475                            return refuse(
3476                                c.name,
3477                                format!(
3478                                    "the key column is {} in the file and {} in the declaration",
3479                                    f.decl_type,
3480                                    c.sql.ddl()
3481                                ),
3482                            );
3483                        }
3484                        note(format!(
3485                            "`{}` is {} in the file and {} in the declaration",
3486                            c.name,
3487                            f.decl_type,
3488                            c.sql.ddl()
3489                        ));
3490                    }
3491                    if f.pk != d.pk {
3492                        return refuse(
3493                            c.name,
3494                            if d.pk {
3495                                "declared as the key but not the file's primary key".into()
3496                            } else {
3497                                "the file's primary key but not the declared key".into()
3498                            },
3499                        );
3500                    }
3501                    // The key's NOT NULL is implied by PRIMARY KEY (and reported either way
3502                    // by the engine, depending on its type), so only the other columns compare.
3503                    if !d.pk && f.not_null != c.not_null {
3504                        if c.not_null && default == Value::Null {
3505                            return refuse(
3506                                c.name,
3507                                "becomes required, but its Rust default stores NULL, so \
3508                                 existing NULLs cannot be filled"
3509                                    .into(),
3510                            );
3511                        }
3512                        note(format!(
3513                            "`{}` is {} in the file and {} in the declaration",
3514                            c.name,
3515                            if f.not_null { "NOT NULL" } else { "nullable" },
3516                            if c.not_null { "NOT NULL" } else { "nullable" }
3517                        ));
3518                    }
3519                    let want_unique = c.unique && !d.pk;
3520                    // A declared unique without the inline constraint owns its index,
3521                    // whether that index exists yet or not (`IF NOT EXISTS` keeps one).
3522                    match (want_unique, inline_unique(c.name), has_unique_index(c.name)) {
3523                        (true, false, _) => plan.unique_indexes.push(c.name),
3524                        (false, true, _) => note(format!(
3525                            "`{}` carries UNIQUE but is not declared unique",
3526                            c.name
3527                        )),
3528                        (false, false, true) => plan.drop_indexes.push(unique_index_name(c.name)),
3529                        _ => {}
3530                    }
3531                    let fk_matches = match (&d.fk, &f.fk) {
3532                        (None, None) => true,
3533                        (Some(want), Some(have)) => {
3534                            have.parent == want.parent_bare
3535                                && have.parent_key == want.parent_key
3536                                && have.on_delete == want.rule.sql()
3537                        }
3538                        _ => false,
3539                    };
3540                    if !fk_matches {
3541                        note(format!("`{}`'s foreign key differs", c.name));
3542                    }
3543                }
3544            }
3545        }
3546        for f in file
3547            .columns
3548            .iter()
3549            .filter(|f| !declared.iter().any(|d| d.def.name == f.name))
3550        {
3551            if f.pk {
3552                return refuse(&f.name, "the file's primary key is not declared".into());
3553            }
3554            // `DROP COLUMN` refuses a column under a constraint or a foreign index; those go
3555            // through the copy, which simply leaves them out.
3556            let covered_by_foreign_index = file.indexes.iter().any(|i| {
3557                i.origin == "c" && !is_day_index(&i.name, bare) && i.columns.contains(&f.name)
3558            });
3559            if f.fk.is_some() || inline_unique(&f.name) || covered_by_foreign_index {
3560                note(format!("constrained column `{}` dropped", f.name));
3561            } else {
3562                plan.drop.push(f.name.clone());
3563            }
3564        }
3565        if let Some(extra) = file
3566            .indexes
3567            .iter()
3568            .find(|i| i.origin == "u" && i.columns.len() > 1)
3569        {
3570            note(format!(
3571                "undeclared multi-column UNIQUE constraint over ({})",
3572                extra.columns.join(", ")
3573            ));
3574        }
3575        // Day's own indexes: one that no longer matches a declaration is dropped in place;
3576        // a missing one is created after the column work.
3577        let want = self.declared_indexes::<M>(&plan.unique_indexes);
3578        for i in file
3579            .indexes
3580            .iter()
3581            .filter(|i| i.origin == "c" && is_day_index(&i.name, bare))
3582        {
3583            let matches = want
3584                .iter()
3585                .any(|w| w.name == i.name && w.columns == i.columns && w.unique == i.unique);
3586            if !matches && !plan.drop_indexes.contains(&i.name) {
3587                plan.drop_indexes.push(i.name.clone());
3588            }
3589        }
3590        Ok(if rebuild {
3591            SchemaDelta::Rebuild
3592        } else {
3593            SchemaDelta::Lightweight(plan)
3594        })
3595    }
3596
3597    /// Close the gap in place: indexes that no longer match are dropped, missing columns are
3598    /// added with their declared constraints (`NOT NULL DEFAULT <Rust default>`; a nullable
3599    /// foreign key with its `REFERENCES`), and columns the declaration lost are dropped.
3600    /// Declared-unique additions get their unique index from `create_indexes`, since
3601    /// `ADD COLUMN` cannot carry `UNIQUE`.
3602    fn lightweight_migrate<M: Model>(&self, plan: &LightweightPlan) -> Result<(), DbError> {
3603        let name = TableName::of::<M>();
3604        let t = name.sql();
3605        let declared = self.declared_columns::<M>();
3606        let defaults = M::default_row();
3607        for idx in &plan.drop_indexes {
3608            self.conn().execute(
3609                &format!("DROP INDEX IF EXISTS {}", table_sql(name.schema, idx)),
3610                &[],
3611            )?;
3612        }
3613        for &i in &plan.add {
3614            let d = &declared[i];
3615            let c = &d.def;
3616            let default = Row::get(&defaults, i);
3617            let mut ddl = format!(
3618                "ALTER TABLE {t} ADD COLUMN {} {}",
3619                ident(c.name),
3620                c.sql.ddl()
3621            );
3622            if c.not_null {
3623                let _ = write!(ddl, " NOT NULL DEFAULT {}", sql_literal(&default)?);
3624            }
3625            if let Some(fk) = &d.fk {
3626                ddl.push_str(&fk.clause());
3627            }
3628            self.conn().execute(&ddl, &[])?;
3629            // A nullable column whose Rust default is a value still backfills, as before.
3630            if !c.not_null && default != Value::Null {
3631                self.conn()
3632                    .execute(&format!("UPDATE {t} SET {} = ?", ident(c.name)), &[default])?;
3633            }
3634        }
3635        for gone in &plan.drop {
3636            self.conn()
3637                .execute(&format!("ALTER TABLE {t} DROP COLUMN {}", ident(gone)), &[])?;
3638        }
3639        Ok(())
3640    }
3641
3642    /// SQLite's documented procedure for every other change
3643    /// (<https://www.sqlite.org/lang_altertable.html#otheralter>): the declared table is
3644    /// created under a temporary name, the rows copied across (shared columns as they are,
3645    /// a required column that was nullable through `COALESCE` with its Rust default, a
3646    /// missing one from that default), the old table dropped, the new one renamed into
3647    /// place, and the application's own indexes that still apply re-created. The caller
3648    /// holds the transaction, has foreign keys off, and runs `foreign_key_check` after.
3649    /// Application-owned triggers on the table go with it; their owner re-creates them.
3650    fn rebuild_table<M: Model>(&self, file: &FileSchema) -> Result<(), DbError> {
3651        let name = TableName::of::<M>();
3652        let t = name.sql();
3653        let bare = name.bare;
3654        let tmp_bare = format!("{bare}_new");
3655        let tmp = table_sql(name.schema, &tmp_bare);
3656        let declared = self.declared_columns::<M>();
3657        let defaults = M::default_row();
3658        let master = table_sql(name.schema, "sqlite_master");
3659
3660        // The application's own indexes, to re-create when every column they name survives.
3661        let mut foreign_indexes: Vec<String> = Vec::new();
3662        self.conn().query(
3663            &format!(
3664                "SELECT name, sql FROM {master} WHERE type = 'index' AND tbl_name = ? \
3665                 AND sql IS NOT NULL"
3666            ),
3667            &[Value::Text(bare.into())],
3668            &mut |row| {
3669                let idx_name = row.get(0).as_text().unwrap_or("").to_string();
3670                if is_day_index(&idx_name, bare) {
3671                    return;
3672                }
3673                let survives = file
3674                    .indexes
3675                    .iter()
3676                    .find(|i| i.name == idx_name)
3677                    .is_some_and(|i| {
3678                        i.columns
3679                            .iter()
3680                            .all(|c| declared.iter().any(|d| d.def.name == c))
3681                    });
3682                if survives && let Ok(sql) = row.get(1).as_text() {
3683                    foreign_indexes.push(sql.to_string());
3684                }
3685            },
3686        )?;
3687
3688        // Triggers on the table go with `DROP TABLE`; the application's come back verbatim
3689        // after the rename when every column they name survives (Day's generated ones are
3690        // reconciled afterwards anyway). Checked by a dry parse against the new table.
3691        let mut triggers: Vec<String> = Vec::new();
3692        self.conn().query(
3693            &format!(
3694                "SELECT sql FROM {master} WHERE type = 'trigger' AND tbl_name = ? \
3695                 AND sql IS NOT NULL ORDER BY rowid"
3696            ),
3697            &[Value::Text(bare.into())],
3698            &mut |row| {
3699                if let Ok(sql) = row.get(0).as_text() {
3700                    triggers.push(sql.to_string());
3701                }
3702            },
3703        )?;
3704
3705        let wrap =
3706            |e: DbError| DbError::new(e.kind, format!("rebuilding `{}`: {}", M::TABLE, e.message));
3707        self.conn()
3708            .execute(&format!("DROP TABLE IF EXISTS {tmp}"), &[])
3709            .map_err(wrap)?;
3710        self.create_table_named::<M>(&tmp).map_err(wrap)?;
3711        let targets = column_list(&declared.iter().map(|d| d.def.name).collect::<Vec<_>>());
3712        let mut sources: Vec<String> = Vec::with_capacity(declared.len());
3713        for d in &declared {
3714            let c = &d.def;
3715            let default = Row::get(&defaults, d.index);
3716            let required = c.not_null || d.pk;
3717            let shared = file.columns.iter().find(|f| f.name == c.name);
3718            sources.push(match shared {
3719                Some(f) if !required || f.not_null => ident(c.name),
3720                Some(_) => format!(
3721                    "COALESCE({}, {})",
3722                    ident(c.name),
3723                    sql_literal(&default).map_err(wrap)?
3724                ),
3725                None => sql_literal(&default).map_err(wrap)?,
3726            });
3727        }
3728        self.conn()
3729            .execute(
3730                &format!(
3731                    "INSERT INTO {tmp} ({targets}) SELECT {} FROM {t}",
3732                    sources.join(", ")
3733                ),
3734                &[],
3735            )
3736            .map_err(wrap)?;
3737        self.conn()
3738            .execute(&format!("DROP TABLE {t}"), &[])
3739            .map_err(wrap)?;
3740        // Other tables' triggers and views may name this table (an adapter's capture trigger
3741        // on a child table reads the parent), and a rename checks every such reference
3742        // against a schema the table is momentarily absent from. The legacy rename leaves
3743        // those references as written, which is what a rebuild under the same name wants.
3744        self.conn()
3745            .execute_batch("PRAGMA legacy_alter_table = ON")
3746            .map_err(wrap)?;
3747        let renamed = self
3748            .conn()
3749            .execute(&format!("ALTER TABLE {tmp} RENAME TO {}", ident(bare)), &[]);
3750        self.conn()
3751            .execute_batch("PRAGMA legacy_alter_table = OFF")
3752            .map_err(wrap)?;
3753        renamed.map_err(wrap)?;
3754        for sql in foreign_indexes {
3755            self.conn().execute(&sql, &[]).map_err(wrap)?;
3756        }
3757        for sql in triggers {
3758            // A trigger naming a dropped column cannot come back; its owner recreates it.
3759            let _ = self.conn().execute(&sql, &[]);
3760        }
3761        Ok(())
3762    }
3763
3764    fn stored_fingerprint(&self, table: &str) -> Result<Option<String>, DbError> {
3765        let mut fp = None;
3766        self.conn().query(
3767            "SELECT fingerprint FROM _day_schema WHERE table_name = ?",
3768            &[Value::Text(table.into())],
3769            &mut |row| {
3770                if let Ok(t) = row.get(0).as_text() {
3771                    fp = Some(t.to_string());
3772                }
3773            },
3774        )?;
3775        Ok(fp)
3776    }
3777
3778    fn store_fingerprint(&self, table: &str, fp: &str) -> Result<(), DbError> {
3779        self.conn().execute(
3780            "INSERT INTO _day_schema (table_name, fingerprint, version) VALUES (?, ?, 0) \
3781             ON CONFLICT(table_name) DO UPDATE SET fingerprint = excluded.fingerprint",
3782            &[Value::Text(table.into()), Value::Text(fp.into())],
3783        )?;
3784        Ok(())
3785    }
3786
3787    fn install_sink(&self) {
3788        let inner = Rc::downgrade(&self.inner);
3789        let sink = day_model::install_change_sink(move |change| {
3790            if let Some(inner) = inner.upgrade() {
3791                // An external merge's changes are the database's own contents arriving:
3792                // queries go stale on them like any edit, but nothing goes back to the file.
3793                if change.author != Some(ModelContainer::EXTERNAL_AUTHOR) {
3794                    let tables = inner.tables.borrow();
3795                    inner
3796                        .dirty
3797                        .borrow_mut()
3798                        .note(change, |store| tables.contains_key(&store));
3799                }
3800                let container = ModelContainer { inner };
3801                container.mark_queries_stale(change);
3802                container.relations_on_change(change);
3803            }
3804        });
3805        self.inner.sink.set(Some(sink));
3806    }
3807
3808    fn install_autosave(&self) {
3809        // Weak: `on_turn_end` registrations live for the process, and a strong clone here
3810        // would keep the container (and its connection) alive forever.
3811        let inner = Rc::downgrade(&self.inner);
3812        day_reactive::on_turn_end(move || {
3813            let Some(inner) = inner.upgrade() else { return };
3814            let this = ModelContainer { inner };
3815            if this.inner.autosave.get() && !this.inner.dirty.borrow().is_empty() {
3816                // The error lands in the signal; a turn end has no caller to give a Result to.
3817                let _ = this.save();
3818            }
3819        });
3820    }
3821
3822    fn flush(&self, dirty: &DirtyState) -> Result<Vec<(String, Vec<Value>)>, DbError> {
3823        let capture = self.inner.participant.borrow().is_some();
3824        let (stmts, changes) = self.fold(dirty, capture);
3825        if stmts.is_empty() {
3826            return Ok(stmts);
3827        }
3828        let mut conn = self.conn();
3829        let outer = self.inner.worker_transaction.get();
3830        if !outer {
3831            conn.begin()?;
3832            if let Err(error) = self.participant_begin(conn.as_mut(), WriteOrigin::Local) {
3833                let _ = conn.rollback();
3834                return Err(error);
3835            }
3836        }
3837        // After begin (which clears the set) and before the statements: an outer worker
3838        // transaction accumulates every inner flush, and rolls the set back with the rest.
3839        self.record_changes(changes);
3840        for (sql, params) in &stmts {
3841            if let Err(e) = conn.execute(sql, params) {
3842                if !outer {
3843                    let _ = conn.rollback();
3844                    self.participant_rollback();
3845                }
3846                return Err(e);
3847            }
3848        }
3849        if !outer
3850            && let Err(error) = self
3851                .participant_commit(conn.as_mut(), WriteOrigin::Local)
3852                .and_then(|()| conn.commit())
3853        {
3854            let _ = conn.rollback();
3855            self.participant_rollback();
3856            return Err(error);
3857        }
3858        Ok(stmts)
3859    }
3860
3861    #[cfg(not(target_arch = "wasm32"))]
3862    fn worker_dispose_caches(&self) {
3863        for hooks in self.inner.tables.borrow().values() {
3864            (hooks.dispose_cache)();
3865        }
3866        // Reactive callbacks can retain a container until thread teardown. Release transaction
3867        // resources (including replication's writer lease) before acknowledging worker close.
3868        self.inner.participant.borrow_mut().take();
3869    }
3870
3871    #[cfg(not(target_arch = "wasm32"))]
3872    fn worker_write<R>(
3873        &self,
3874        write: impl FnOnce(&Self) -> Result<R, DbError>,
3875    ) -> Result<R, DbError> {
3876        self.worker_write_origin(WriteOrigin::Local, write)
3877    }
3878
3879    #[cfg(not(target_arch = "wasm32"))]
3880    fn worker_write_origin<R>(
3881        &self,
3882        origin: WriteOrigin,
3883        write: impl FnOnce(&Self) -> Result<R, DbError>,
3884    ) -> Result<R, DbError> {
3885        self.save()?;
3886        self.conn().begin()?;
3887        let admission = self.participant_begin(self.conn().as_mut(), origin);
3888        if let Err(error) = admission {
3889            self.conn().rollback().map_err(|rollback| {
3890                DbError::new(
3891                    DbErrorKind::Closed,
3892                    format!("transaction admission rollback failed: {rollback}"),
3893                )
3894            })?;
3895            return Err(error);
3896        }
3897        self.inner.worker_transaction.set(true);
3898        let result = day_reactive::batch(|| {
3899            let value = write(self)?;
3900            self.save()?;
3901            Ok(value)
3902        })
3903        .and_then(|value| {
3904            // Reactions may have made additional writes while the batch drained. Commit only
3905            // after those writes settle, never before turn-end autosave has run.
3906            self.save()?;
3907            self.participant_commit(self.conn().as_mut(), origin)?;
3908            self.conn().commit()?;
3909            Ok(value)
3910        });
3911        self.inner.worker_transaction.set(false);
3912        if result.is_err() {
3913            self.participant_rollback();
3914            self.conn().rollback().map_err(|error| {
3915                DbError::new(
3916                    DbErrorKind::Closed,
3917                    format!("transaction rollback failed: {error}"),
3918                )
3919            })?;
3920            self.worker_discard_changes()?;
3921        }
3922        if result.is_ok() {
3923            self.participant_refresh()?;
3924        }
3925        result
3926    }
3927
3928    #[cfg(not(target_arch = "wasm32"))]
3929    fn worker_read<R>(&self, read: impl FnOnce(&Self) -> Result<R, DbError>) -> Result<R, DbError> {
3930        self.save()?;
3931        self.conn().execute_batch("PRAGMA query_only = ON")?;
3932        self.conn().begin().map_err(|error| {
3933            DbError::new(
3934                DbErrorKind::Closed,
3935                format!("could not begin read snapshot: {error}"),
3936            )
3937        })?;
3938        let autosave = self.inner.autosave.replace(false);
3939        let mut result =
3940            day_model::with_author("worker-read", || day_reactive::batch(|| read(self)));
3941        let mutated = !self.inner.dirty.borrow().is_empty();
3942        self.conn().rollback().map_err(|error| {
3943            DbError::new(
3944                DbErrorKind::Closed,
3945                format!("could not end read snapshot: {error}"),
3946            )
3947        })?;
3948        self.conn()
3949            .execute_batch("PRAGMA query_only = OFF")
3950            .map_err(|error| {
3951                DbError::new(
3952                    DbErrorKind::Closed,
3953                    format!("could not restore write mode: {error}"),
3954                )
3955            })?;
3956        if mutated {
3957            self.worker_discard_changes()?;
3958            result = Err(DbError::driver(
3959                "a worker read attempted to mutate model state",
3960            ));
3961        }
3962        self.inner.autosave.set(autosave);
3963        result
3964    }
3965
3966    #[cfg(not(target_arch = "wasm32"))]
3967    fn worker_discard_changes(&self) -> Result<(), DbError> {
3968        let autosave = self.inner.autosave.replace(false);
3969        *self.inner.dirty.borrow_mut() = DirtyState::default();
3970        self.participant_rollback();
3971        let result = self.refresh_resident();
3972        self.invalidate_relation_memos();
3973        if result.is_ok() {
3974            self.requery_all();
3975        }
3976        self.inner.autosave.set(autosave);
3977        result.map(|_| ()).map_err(|error| {
3978            DbError::new(
3979                DbErrorKind::Closed,
3980                format!("transaction cache recovery failed: {error}"),
3981            )
3982        })
3983    }
3984
3985    /// The fold, materialized: the smallest statement list that expresses `dirty`, in
3986    /// first-touch order. Row values come from the caches now; the change log carried which
3987    /// rows and columns moved, never their contents. Reads only; [`ModelContainer::flush`]
3988    /// executes the list in one transaction.
3989    ///
3990    /// With `capture`, the second answer is the same work as a change set, one entry per row
3991    /// a statement writes (the participant's view of the flush); without it, empty.
3992    #[allow(clippy::type_complexity)]
3993    fn fold(
3994        &self,
3995        dirty: &DirtyState,
3996        capture: bool,
3997    ) -> (Vec<(String, Vec<Value>)>, Vec<RowChange>) {
3998        let tables = self.inner.tables.borrow();
3999        let mut stmts: Vec<(String, Vec<Value>)> = Vec::new();
4000        let mut changes: Vec<RowChange> = Vec::new();
4001        let authors_of = |store_id: u64, key: u64| -> Vec<&'static str> {
4002            dirty
4003                .authors
4004                .get(&(store_id, key))
4005                .cloned()
4006                .unwrap_or_default()
4007        };
4008
4009        // Wholesale rewrites first: upsert every resident row. (The cache is a working set;
4010        // rows outside it were never part of the rewrite, and deleting "the rest" would
4011        // delete data the rewrite never saw.)
4012        for store_id in &dirty.full {
4013            let Some(hooks) = tables.get(store_id) else {
4014                continue;
4015            };
4016            for (key, row) in (hooks.all_rows)() {
4017                stmts.push(upsert_stmt(hooks, &row));
4018                if capture {
4019                    let mut authors = dirty
4020                        .store_authors
4021                        .get(store_id)
4022                        .cloned()
4023                        .unwrap_or_default();
4024                    for tag in authors_of(*store_id, key) {
4025                        add_author(&mut authors, Some(tag));
4026                    }
4027                    changes.push(RowChange {
4028                        table: hooks.table,
4029                        key: (hooks.key_where)(key).1,
4030                        kind: RowChangeKind::Rewritten,
4031                        authors,
4032                    });
4033                }
4034            }
4035        }
4036
4037        // Per-row work, into batches that can merge. Deletes of one table merge into one
4038        // `IN`; updates merge when they write the same columns to the same values, which is
4039        // what a multi-selection edit produces ("set fill on twelve shapes").
4040        //
4041        // Deletes are held apart and emitted last. Dirty order interleaves rows by when each
4042        // first changed, which can put a parent row's delete between its children's detach
4043        // updates (the first detach dirties the parent through relation upkeep), and a
4044        // relation column's `ON DELETE CASCADE` fires per statement, `DEFERRABLE` or not, so
4045        // the mid-batch delete would take every still-attached child with it (found by Day
4046        // Sketch's ungroup: two members detached, one row left). With inserts and updates
4047        // first, a row still referencing a deleted parent at delete time is one the batch
4048        // never detached: an orphan the cascade rule is meant to take.
4049        let mut batches: Vec<Batch> = Vec::new();
4050        let mut deletes: Vec<Batch> = Vec::new();
4051        // (store, statement shape, a fingerprint of the `SET` values) → the batch it joins.
4052        let mut open: HashMap<(u64, String, u64), usize> = HashMap::new();
4053        let mut open_deletes: HashMap<(u64, String, u64), usize> = HashMap::new();
4054        // store → the insert batch collecting its new rows.
4055        let mut open_inserts: HashMap<u64, usize> = HashMap::new();
4056
4057        for id in &dirty.order {
4058            let Some(state) = dirty.rows.get(id) else {
4059                continue;
4060            };
4061            let (store_id, key) = *id;
4062            if dirty.full.contains(&store_id) {
4063                continue; // the resync above already covered it
4064            }
4065            let Some(hooks) = tables.get(&store_id) else {
4066                continue;
4067            };
4068            // A join row is addressed by a pair of columns, which no single-column `IN` can
4069            // express, so those keep one statement each.
4070            let batchable = hooks.key_cols.len() == 1;
4071            let mut capture_as = |kind: RowChangeKind| {
4072                if capture {
4073                    changes.push(RowChange {
4074                        table: hooks.table,
4075                        key: (hooks.key_where)(key).1,
4076                        kind,
4077                        authors: authors_of(store_id, key),
4078                    });
4079                }
4080            };
4081            match state {
4082                DirtyRow::Delete => {
4083                    capture_as(RowChangeKind::Deleted);
4084                    if !batchable {
4085                        let (clause, params) = (hooks.key_where)(key);
4086                        deletes.push(Batch::Single(
4087                            format!("DELETE FROM {} WHERE {clause}", hooks.name.sql()),
4088                            params,
4089                        ));
4090                        continue;
4091                    }
4092                    join_batch(&mut deletes, &mut open_deletes, store_id, None, key);
4093                }
4094                DirtyRow::Insert => {
4095                    if let Some(row) = (hooks.row_for)(key) {
4096                        capture_as(RowChangeKind::Inserted);
4097                        if let Some(&i) = open_inserts.get(&store_id) {
4098                            if let Batch::Inserts { rows, .. } = &mut batches[i] {
4099                                rows.push(row);
4100                            }
4101                        } else {
4102                            open_inserts.insert(store_id, batches.len());
4103                            batches.push(Batch::Inserts {
4104                                store: store_id,
4105                                rows: vec![row],
4106                            });
4107                        }
4108                    }
4109                }
4110                DirtyRow::Update(cols) => {
4111                    let Some(row) = (hooks.row_for)(key) else {
4112                        continue; // deleted since; its Delete entry handles it
4113                    };
4114                    if cols.is_empty() {
4115                        // A row-level replacement named no columns; write them all.
4116                        let (sql, params) = upsert_stmt(hooks, &row);
4117                        batches.push(Batch::Single(sql, params));
4118                        capture_as(RowChangeKind::Rewritten);
4119                        continue;
4120                    }
4121                    let mut sets: Vec<String> = Vec::with_capacity(cols.len());
4122                    let mut set_params: Vec<Value> = Vec::with_capacity(cols.len());
4123                    for c in cols {
4124                        let Some(i) = hooks.fields.iter().position(|n| n == c) else {
4125                            continue; // a transient field's label, never a column
4126                        };
4127                        let value = row.get(i);
4128                        if hooks.lazy.get(i).copied().unwrap_or(false) && value == Value::Null {
4129                            continue; // a stored blob the row did not rewrite: leave the bytes
4130                        }
4131                        sets.push(format!("{} = ?", ident(&hooks.columns[i])));
4132                        set_params.push(value);
4133                    }
4134                    if set_params.is_empty() {
4135                        continue; // only transient fields changed
4136                    }
4137                    if capture {
4138                        let written: Vec<&'static str> = cols
4139                            .iter()
4140                            .filter_map(|c| hooks.fields.iter().position(|n| n == c))
4141                            .map(|i| hooks.column_names[i])
4142                            .collect();
4143                        capture_as(RowChangeKind::Updated(written));
4144                    }
4145                    let clause = sets.join(", ");
4146                    if !batchable {
4147                        let (where_clause, mut where_params) = (hooks.key_where)(key);
4148                        let mut params = set_params;
4149                        params.append(&mut where_params);
4150                        batches.push(Batch::Single(
4151                            format!(
4152                                "UPDATE {} SET {clause} WHERE {where_clause}",
4153                                hooks.name.sql()
4154                            ),
4155                            params,
4156                        ));
4157                        continue;
4158                    }
4159                    join_batch(
4160                        &mut batches,
4161                        &mut open,
4162                        store_id,
4163                        Some((clause, set_params)),
4164                        key,
4165                    );
4166                }
4167            }
4168        }
4169
4170        for batch in batches.into_iter().chain(deletes) {
4171            match batch {
4172                Batch::Single(sql, params) => stmts.push((sql, params)),
4173                Batch::Inserts { store, rows } => {
4174                    let Some(hooks) = tables.get(&store) else {
4175                        continue;
4176                    };
4177                    insert_stmts(hooks, &rows, &mut stmts);
4178                }
4179                Batch::Keyed { store, set, keys } => {
4180                    let Some(hooks) = tables.get(&store) else {
4181                        continue;
4182                    };
4183                    materialize(hooks, set, &keys, &mut stmts);
4184                }
4185            }
4186        }
4187        (stmts, changes)
4188    }
4189
4190    // --- maintenance (docs/persistence.md) ---------------------------------------------------
4191
4192    /// `VACUUM INTO`: a transactionally consistent, already-compacted snapshot, taken while
4193    /// the app keeps writing. Restore is an open, not a verb.
4194    ///
4195    /// # Errors
4196    /// Returns the storage error when pending changes cannot be saved or the snapshot cannot be
4197    /// written.
4198    pub fn backup_to(&self, path: &Path) -> Result<(), DbError> {
4199        self.save()?;
4200        self.conn().execute(
4201            "VACUUM INTO ?",
4202            &[Value::Text(path.to_string_lossy().into_owned())],
4203        )?;
4204        Ok(())
4205    }
4206
4207    /// `PRAGMA integrity_check` and `PRAGMA foreign_key_check`, parsed: an empty list means
4208    /// the database is sound. A foreign-key violation reads
4209    /// `foreign key: <child> row <rowid> references missing <parent>`; the structured form is
4210    /// [`ModelContainer::foreign_key_check`].
4211    ///
4212    /// # Errors
4213    /// Returns the storage error when a check cannot run. Findings are not errors.
4214    pub fn integrity_check(&self) -> Result<Vec<String>, DbError> {
4215        let mut findings = Vec::new();
4216        self.conn()
4217            .query("PRAGMA integrity_check", &[], &mut |row| {
4218                if let Ok(t) = row.get(0).as_text()
4219                    && t != "ok"
4220                {
4221                    findings.push(t.to_string());
4222                }
4223            })?;
4224        for v in self.foreign_key_check()? {
4225            findings.push(v.to_string());
4226        }
4227        Ok(findings)
4228    }
4229
4230    /// `PRAGMA foreign_key_check`, parsed: every row whose foreign key names a parent that
4231    /// does not exist. Empty when every reference resolves. Runs on the whole file, which is
4232    /// what a migration's post-check and a settings-page validation both want.
4233    ///
4234    /// # Errors
4235    /// Returns the storage error when the check cannot run. Violations are not errors.
4236    pub fn foreign_key_check(&self) -> Result<Vec<ForeignKeyViolation>, DbError> {
4237        Self::foreign_key_check_on(self.conn().as_mut())
4238    }
4239
4240    fn foreign_key_check_on(
4241        conn: &mut dyn SqliteConnection,
4242    ) -> Result<Vec<ForeignKeyViolation>, DbError> {
4243        let mut violations = Vec::new();
4244        conn.query("PRAGMA foreign_key_check", &[], &mut |row| {
4245            violations.push(ForeignKeyViolation {
4246                table: row.get(0).as_text().unwrap_or("").to_string(),
4247                rowid: row.get(1).as_int().ok(),
4248                parent: row.get(2).as_text().unwrap_or("").to_string(),
4249                constraint: row.get(3).as_int().unwrap_or(0),
4250            });
4251        })?;
4252        Ok(violations)
4253    }
4254
4255    /// Fold the WAL into the main file (`PRAGMA wal_checkpoint(TRUNCATE)`), before the OS
4256    /// copies it whole (device backup). The engine's own answer comes back: whether a reader
4257    /// kept the checkpoint from completing, and how much of the log it folded.
4258    ///
4259    /// # Errors
4260    /// Returns the storage error when the checkpoint cannot run.
4261    pub fn checkpoint(&self) -> Result<Checkpoint, DbError> {
4262        let mut result = Checkpoint::default();
4263        self.conn()
4264            .query("PRAGMA wal_checkpoint(TRUNCATE)", &[], &mut |row| {
4265                result = Checkpoint {
4266                    busy: row.get(0).as_int().unwrap_or(0) != 0,
4267                    log_frames: row.get(1).as_int().unwrap_or(-1),
4268                    checkpointed_frames: row.get(2).as_int().unwrap_or(-1),
4269                };
4270            })?;
4271        Ok(result)
4272    }
4273
4274    /// Save pending changes, then rebuild the file compactly (`VACUUM`).
4275    ///
4276    /// # Errors
4277    /// Returns the storage error when pending changes cannot be saved or the engine refuses the
4278    /// `VACUUM`.
4279    pub fn vacuum(&self) -> Result<(), DbError> {
4280        self.save()?;
4281        self.conn().execute("VACUUM", &[]).map(|_| ())
4282    }
4283
4284    /// Re-encrypt in place under a new key (`PRAGMA rekey`; feature `cipher`).
4285    #[cfg(feature = "cipher")]
4286    pub fn rekey(&self, new: Secret) -> Result<(), DbError> {
4287        self.save()?;
4288        let quoted = new.reveal().replace('\'', "''");
4289        // The PRAGMA answers with a status row; consume it as a query.
4290        self.conn()
4291            .query(&format!("PRAGMA rekey = '{quoted}'"), &[], &mut |_| {})
4292    }
4293
4294    /// Write an encrypted copy at `path` through SQLCipher's conversion path (`ATTACH` +
4295    /// `sqlcipher_export`); plaintext↔encrypted cannot happen in place.
4296    #[cfg(feature = "cipher")]
4297    pub fn encrypt_to(&self, path: &Path, key: Secret) -> Result<(), DbError> {
4298        self.export_to(path, Some(key))
4299    }
4300
4301    /// Write a plaintext copy at `path`, same mechanism with an empty key.
4302    #[cfg(feature = "cipher")]
4303    pub fn decrypt_to(&self, path: &Path) -> Result<(), DbError> {
4304        self.export_to(path, None)
4305    }
4306
4307    #[cfg(feature = "cipher")]
4308    fn export_to(&self, path: &Path, key: Option<Secret>) -> Result<(), DbError> {
4309        self.save()?;
4310        let quoted_path = path.to_string_lossy().replace('\'', "''");
4311        let quoted_key = key
4312            .as_ref()
4313            .map_or_default(|k| k.reveal().replace('\'', "''"));
4314        let mut conn = self.conn();
4315        conn.execute(
4316            &format!("ATTACH DATABASE '{quoted_path}' AS day_export KEY '{quoted_key}'"),
4317            &[],
4318        )?;
4319        conn.query("SELECT sqlcipher_export('day_export')", &[], &mut |_| {})
4320            .and_then(|_| conn.execute("DETACH DATABASE day_export", &[]).map(|_| ()))
4321    }
4322
4323    /// Total size in bytes (`page_count` × `page_size`).
4324    ///
4325    /// # Errors
4326    /// Returns the storage error when the read fails.
4327    pub fn size_bytes(&self) -> Result<u64, DbError> {
4328        let mut pages = 0i64;
4329        let mut page_size = 0i64;
4330        self.conn().query("PRAGMA page_count", &[], &mut |row| {
4331            pages = row.get(0).as_int().unwrap_or(0);
4332        })?;
4333        self.conn().query("PRAGMA page_size", &[], &mut |row| {
4334            page_size = row.get(0).as_int().unwrap_or(0);
4335        })?;
4336        Ok((pages * page_size).max(0) as u64)
4337    }
4338}
4339
4340/// The multi-key `WHERE` builder for a hooks entry: `key IN (…)` for single-column keys, an OR
4341/// list of pair clauses for join tables.
4342/// `a, b, c` as a quoted SELECT list or insert column list.
4343pub(crate) fn column_list<S: AsRef<str>>(columns: &[S]) -> String {
4344    columns
4345        .iter()
4346        .map(|c| ident(c.as_ref()))
4347        .collect::<Vec<_>>()
4348        .join(", ")
4349}
4350
4351/// `prefix.a, prefix.b`: the same list through a table alias or a trigger's `new`/`old`.
4352pub(crate) fn prefixed_list<S: AsRef<str>>(prefix: &str, columns: &[S]) -> String {
4353    columns
4354        .iter()
4355        .map(|c| format!("{prefix}.{}", ident(c.as_ref())))
4356        .collect::<Vec<_>>()
4357        .join(", ")
4358}
4359
4360/// The two halves of an FTS update trigger's gate: the `UPDATE OF` column list (the key and
4361/// every indexed column, once each) and the `WHEN` expression that is true only if one of
4362/// them actually changed (`IS NOT`, so a NULL on either side compares like any value).
4363fn fts_update_guard(key: &str, fts_columns: &[&str]) -> (String, String) {
4364    let mut cols: Vec<&str> = vec![key];
4365    cols.extend(fts_columns.iter().copied().filter(|c| *c != key));
4366    let update_of = column_list(&cols);
4367    let changed = cols
4368        .iter()
4369        .map(|c| {
4370            let c = ident(c);
4371            format!("new.{c} IS NOT old.{c}")
4372        })
4373        .collect::<Vec<_>>()
4374        .join(" OR ");
4375    (update_of, changed)
4376}
4377
4378// ---------------------------------------------------------------------------
4379// Schema comparison: the declaration against the file
4380// ---------------------------------------------------------------------------
4381
4382/// One column as the model declares it, with the relation-contributed foreign key resolved.
4383struct DeclaredColumn {
4384    def: ColumnDef,
4385    /// Position in `Model::COLUMNS`, which is also the position in `Model::default_row()`.
4386    index: usize,
4387    pk: bool,
4388    fk: Option<DeclaredFk>,
4389}
4390
4391/// A declared relation's foreign key on a child column.
4392struct DeclaredFk {
4393    parent_sql: String,
4394    parent_bare: &'static str,
4395    parent_key: &'static str,
4396    rule: DeleteRule,
4397}
4398
4399impl DeclaredFk {
4400    /// The column constraint, deferred so within-transaction statement order (a cascade's
4401    /// children, an undo's re-inserts) never trips it.
4402    fn clause(&self) -> String {
4403        format!(
4404            " REFERENCES {}({}) ON DELETE {} DEFERRABLE INITIALLY DEFERRED",
4405            self.parent_sql,
4406            ident(self.parent_key),
4407            self.rule.sql()
4408        )
4409    }
4410}
4411
4412/// An index the container owns, as it is declared.
4413struct DeclaredIndex {
4414    name: String,
4415    columns: Vec<String>,
4416    unique: bool,
4417}
4418
4419/// A column as `PRAGMA table_info` and `PRAGMA foreign_key_list` report it.
4420struct FileColumn {
4421    name: String,
4422    /// Upper-cased declared type (`INTEGER`, `TEXT`, …).
4423    decl_type: String,
4424    not_null: bool,
4425    pk: bool,
4426    fk: Option<FileFk>,
4427}
4428
4429struct FileFk {
4430    parent: String,
4431    parent_key: String,
4432    on_delete: String,
4433}
4434
4435/// An index as `PRAGMA index_list` and `index_info` report it. `origin` is `c` for a
4436/// `CREATE INDEX`, `u` for an inline `UNIQUE`, `pk` for the primary key's.
4437struct FileIndex {
4438    name: String,
4439    unique: bool,
4440    origin: String,
4441    columns: Vec<String>,
4442}
4443
4444struct FileSchema {
4445    columns: Vec<FileColumn>,
4446    indexes: Vec<FileIndex>,
4447}
4448
4449/// What `lightweight_migrate` does in place: positions in `COLUMNS` to add, file columns to
4450/// drop, Day-owned index names to drop, and declared-unique columns that need a unique index.
4451#[derive(Default, Clone)]
4452struct LightweightPlan {
4453    add: Vec<usize>,
4454    drop: Vec<String>,
4455    drop_indexes: Vec<String>,
4456    unique_indexes: Vec<&'static str>,
4457}
4458
4459/// The comparison's verdict for an existing table.
4460enum SchemaDelta {
4461    Lightweight(LightweightPlan),
4462    Rebuild,
4463}
4464
4465/// Whether an index name is one the container writes for `table` (`day_idx_`, `day_uidx_`).
4466fn is_day_index(name: &str, table: &str) -> bool {
4467    name.starts_with(&format!("day_idx_{table}_"))
4468        || name.starts_with(&format!("day_uidx_{table}_"))
4469}
4470
4471/// `PRAGMA [schema.]name("arg")`.
4472fn pragma_sql(schema: Option<&str>, name: &str, arg: &str) -> String {
4473    match schema {
4474        Some(s) => format!("PRAGMA {}.{name}({})", ident(s), ident(arg)),
4475        None => format!("PRAGMA {name}({})", ident(arg)),
4476    }
4477}
4478
4479/// A [`Value`] as an SQL literal, for the places a bound parameter cannot go (a column's
4480/// `DEFAULT`, a copy's `SELECT` list): text single-quoted with `'` doubled, a blob as
4481/// `X'…'`, numbers as themselves. A non-finite real has no literal and is refused.
4482fn sql_literal(v: &Value) -> Result<String, DbError> {
4483    Ok(match v {
4484        Value::Null => "NULL".into(),
4485        Value::Int(i) => i.to_string(),
4486        Value::Real(r) if r.is_finite() => format!("{r:?}"),
4487        Value::Real(r) => {
4488            return Err(DbError::new(
4489                DbErrorKind::Schema,
4490                format!("{r} has no SQL literal to use as a column default"),
4491            ));
4492        }
4493        Value::Text(t) => format!("'{}'", t.replace('\'', "''")),
4494        Value::Blob(b) => {
4495            let mut s = String::with_capacity(b.len() * 2 + 3);
4496            s.push_str("X'");
4497            for byte in b {
4498                let _ = write!(s, "{byte:02x}");
4499            }
4500            s.push('\'');
4501            s
4502        }
4503    })
4504}
4505
4506fn keys_where_of(hooks: &TableHooks) -> KeysWhere {
4507    if hooks.key_cols.len() == 1 {
4508        let col = ident(&hooks.key_cols[0]);
4509        Rc::new(move |keys: &[u64]| {
4510            let marks = vec!["?"; keys.len()].join(", ");
4511            (
4512                format!("{col} IN ({marks})"),
4513                keys.iter().map(|k| key_param(*k)).collect(),
4514            )
4515        })
4516    } else {
4517        let key_where = hooks.key_where.clone();
4518        Rc::new(move |keys: &[u64]| {
4519            let mut clauses = Vec::with_capacity(keys.len());
4520            let mut params = Vec::with_capacity(keys.len() * 2);
4521            for k in keys {
4522                let (c, mut p) = key_where(*k);
4523                clauses.push(format!("({c})"));
4524                params.append(&mut p);
4525            }
4526            (clauses.join(" OR "), params)
4527        })
4528    }
4529}
4530
4531/// One statement the fold will write, before the mergeable ones have merged.
4532enum Batch {
4533    /// Stands alone: a row-level rewrite (its values are the row's own), or a row addressed
4534    /// by a composite key, which no single-column `IN` can express.
4535    Single(String, Vec<Value>),
4536    /// One table's inserted rows: a multi-row upsert, chunked by the parameter ceiling.
4537    Inserts { store: u64, rows: Vec<Vec<Value>> },
4538    /// Same table, same operation, same values: one statement per chunk of keys.
4539    Keyed {
4540        store: u64,
4541        /// The `SET` clause and its parameters for an update; `None` for a delete.
4542        set: Option<(String, Vec<Value>)>,
4543        keys: Vec<u64>,
4544    },
4545}
4546
4547/// SQLite's conservative bound-parameter ceiling. Modern builds allow far more, but a
4548/// `system` engine may be an older one, and chunking costs nothing to write.
4549const MAX_BOUND_PARAMS: usize = 900;
4550
4551/// Add `key` to the batch sharing this table, clause and values, or open a new one.
4552///
4553/// The slot key separates a delete from an update (their clauses differ), and separates
4554/// updates writing different values, which must stay different statements. Grouping is by
4555/// hash so a large flush stays linear; the values are compared before merging, so a hash
4556/// collision costs a second statement rather than a wrong one.
4557fn join_batch(
4558    batches: &mut Vec<Batch>,
4559    open: &mut HashMap<(u64, String, u64), usize>,
4560    store: u64,
4561    set: Option<(String, Vec<Value>)>,
4562    key: u64,
4563) {
4564    let clause = set.as_ref().map_or_default(|(c, _)| c.clone());
4565    let print = set.as_ref().map_or(0, |(_, p)| fingerprint(p));
4566    let slot = (store, clause, print);
4567    if let Some(&i) = open.get(&slot)
4568        && let Batch::Keyed {
4569            set: existing,
4570            keys,
4571            ..
4572        } = &mut batches[i]
4573        && existing.as_ref().map(|(_, p)| p.as_slice()) == set.as_ref().map(|(_, p)| p.as_slice())
4574    {
4575        keys.push(key);
4576        return;
4577    }
4578    open.insert(slot, batches.len());
4579    batches.push(Batch::Keyed {
4580        store,
4581        set,
4582        keys: vec![key],
4583    });
4584}
4585
4586/// A cheap order-sensitive hash of bound values, for grouping identical updates. Equality is
4587/// still checked before two rows share a statement; this only decides who to compare with.
4588fn fingerprint(values: &[Value]) -> u64 {
4589    let mut h: u64 = 0xcbf2_9ce4_8422_2325;
4590    let mut eat = |bytes: &[u8]| {
4591        for b in bytes {
4592            h ^= u64::from(*b);
4593            h = h.wrapping_mul(0x1000_0000_01b3);
4594        }
4595    };
4596    for v in values {
4597        match v {
4598            Value::Null => eat(&[0]),
4599            Value::Int(i) => {
4600                eat(&[1]);
4601                eat(&i.to_le_bytes());
4602            }
4603            // Bit pattern, not numeric value: two values that hash apart are only compared
4604            // separately, never merged wrongly.
4605            Value::Real(r) => {
4606                eat(&[2]);
4607                eat(&r.to_bits().to_le_bytes());
4608            }
4609            Value::Text(t) => {
4610                eat(&[3]);
4611                eat(t.as_bytes());
4612            }
4613            Value::Blob(b) => {
4614                eat(&[4]);
4615                eat(b);
4616            }
4617        }
4618    }
4619    h
4620}
4621
4622/// Write a batch out: the familiar `= ?` form for one key, chunked `IN` statements for more.
4623fn materialize(
4624    hooks: &TableHooks,
4625    set: Option<(String, Vec<Value>)>,
4626    keys: &[u64],
4627    out: &mut Vec<(String, Vec<Value>)>,
4628) {
4629    // One key keeps the `= ?` form: it is the common case by far, it reads plainly in a
4630    // trace, and it is one more statement shape the cache can hold.
4631    let table = hooks.name.sql();
4632    if keys.len() == 1 {
4633        let (where_clause, where_params) = (hooks.key_where)(keys[0]);
4634        match set {
4635            Some((clause, mut params)) => {
4636                params.extend(where_params);
4637                out.push((
4638                    format!("UPDATE {table} SET {clause} WHERE {where_clause}"),
4639                    params,
4640                ));
4641            }
4642            None => out.push((
4643                format!("DELETE FROM {table} WHERE {where_clause}"),
4644                where_params,
4645            )),
4646        }
4647        return;
4648    }
4649
4650    let key_col = ident(&hooks.key_cols[0]);
4651    let set_len = set.as_ref().map_or(0, |(_, p)| p.len());
4652    let cap = MAX_BOUND_PARAMS.saturating_sub(set_len).max(1);
4653    for chunk in keys.chunks(cap) {
4654        let mut params: Vec<Value> = Vec::with_capacity(set_len + chunk.len());
4655        if let Some((_, set_params)) = &set {
4656            params.extend(set_params.iter().cloned());
4657        }
4658        params.extend(chunk.iter().map(|k| key_param(*k)));
4659        let marks = vec!["?"; chunk.len()].join(", ");
4660        let sql = match &set {
4661            Some((clause, _)) => {
4662                format!("UPDATE {table} SET {clause} WHERE {key_col} IN ({marks})")
4663            }
4664            None => format!("DELETE FROM {table} WHERE {key_col} IN ({marks})"),
4665        };
4666        out.push((sql, params));
4667    }
4668}
4669
4670/// The row select list: every column, with a lazy one named as a literal `NULL` so the
4671/// row's shape stays the same while the bytes stay in the file.
4672fn select_list(hooks: &TableHooks) -> String {
4673    hooks
4674        .columns
4675        .iter()
4676        .enumerate()
4677        .map(|(i, c)| {
4678            if hooks.lazy.get(i).copied().unwrap_or(false) {
4679                "NULL".to_string()
4680            } else {
4681                ident(c)
4682            }
4683        })
4684        .collect::<Vec<_>>()
4685        .join(", ")
4686}
4687
4688/// One table's inserted rows as multi-row upserts: as many rows per statement as the
4689/// parameter ceiling allows, so a thousand new rows are a few statements rather than a
4690/// thousand.
4691fn insert_stmts(hooks: &TableHooks, rows: &[Vec<Value>], out: &mut Vec<(String, Vec<Value>)>) {
4692    let per_row = hooks.columns.len().max(1);
4693    let per_stmt = (MAX_BOUND_PARAMS / per_row).max(1);
4694    for chunk in rows.chunks(per_stmt) {
4695        let sql = upsert_sql(hooks, chunk.len());
4696        let params: Vec<Value> = chunk.iter().flatten().cloned().collect();
4697        out.push((sql, params));
4698    }
4699}
4700
4701fn upsert_stmt(hooks: &TableHooks, row: &[Value]) -> (String, Vec<Value>) {
4702    (upsert_sql(hooks, 1), row.to_vec())
4703}
4704
4705/// `INSERT … VALUES (…), (…) ON CONFLICT(key) DO UPDATE SET …` for `rows` rows.
4706fn upsert_sql(hooks: &TableHooks, rows: usize) -> String {
4707    let cols = column_list(&hooks.columns);
4708    let one = format!("({})", vec!["?"; hooks.columns.len()].join(", "));
4709    let marks = vec![one.as_str(); rows.max(1)].join(", ");
4710    // A true upsert, not `INSERT OR REPLACE`: the conflict path fires `UPDATE` triggers, which
4711    // is what keeps the generated FTS/R*Tree shadows in sync (`OR REPLACE`'s implicit delete
4712    // skips `AFTER DELETE` triggers unless recursive_triggers is on). A lazy column keeps its
4713    // stored bytes when the row arrives without new ones (`NULL`).
4714    let sets = hooks
4715        .columns
4716        .iter()
4717        .enumerate()
4718        .filter(|(_, c)| !hooks.key_cols.contains(c))
4719        .map(|(i, c)| {
4720            if hooks.lazy.get(i).copied().unwrap_or(false) {
4721                format!("{0} = COALESCE(excluded.{0}, {0})", ident(c))
4722            } else {
4723                format!("{0} = excluded.{0}", ident(c))
4724            }
4725        })
4726        .collect::<Vec<_>>()
4727        .join(", ");
4728    let conflict = if sets.is_empty() {
4729        "DO NOTHING".to_string()
4730    } else {
4731        format!("DO UPDATE SET {sets}")
4732    };
4733    format!(
4734        "INSERT INTO {} ({cols}) VALUES {marks} ON CONFLICT({}) {conflict}",
4735        hooks.name.sql(),
4736        column_list(&hooks.key_cols)
4737    )
4738}
4739
4740impl Drop for ContainerInner {
4741    fn drop(&mut self) {
4742        if let Some(sink) = self.sink.take() {
4743            day_model::remove_change_sink(sink);
4744        }
4745    }
4746}
4747
4748// ---------------------------------------------------------------------------
4749// Live queries (docs/persistence.md; queries.rs holds the compiler and the diff)
4750// ---------------------------------------------------------------------------
4751
4752/// What a query's consumers have not yet seen. `Deltas` can animate a list row by row;
4753/// `Reload` means the whole set moved (a fetch swap, a tangled requery) and a reload is
4754/// the accurate answer.
4755#[derive(Clone, Debug, PartialEq)]
4756pub enum QueryEvents {
4757    None,
4758    Deltas(Vec<Delta>),
4759    Reload,
4760}
4761
4762impl QueryEvents {
4763    fn push(&mut self, deltas: &[Delta]) {
4764        match self {
4765            QueryEvents::None => *self = QueryEvents::Deltas(deltas.to_vec()),
4766            QueryEvents::Deltas(d) => d.extend_from_slice(deltas),
4767            QueryEvents::Reload => {}
4768        }
4769    }
4770    fn reload(&mut self) {
4771        *self = QueryEvents::Reload;
4772    }
4773}
4774
4775/// What one relation crossing subscribes a query to, precomputed at install (and at every
4776/// fetch swap) so the change sink's staleness check is a few comparisons.
4777struct RelWatch {
4778    /// The store on the far side of the crossing (the related table's cache).
4779    store: u64,
4780    /// The join membership store, when the crossing is a many-to-many; every link, unlink
4781    /// and reposition there is a membership change.
4782    join_store: Option<u64>,
4783    /// Field names on the far store whose write is a membership change (the foreign key, the
4784    /// order field) even though no predicate reads them.
4785    fields: Vec<&'static str>,
4786    /// Column names of the far table the inner predicate reads.
4787    columns: Vec<&'static str>,
4788}
4789
4790struct QueryState {
4791    store_id: u64,
4792    table: &'static str,
4793    /// The relation subscriptions, recomputed when the fetch is swapped.
4794    watches: RefCell<Vec<RelWatch>>,
4795    /// Extra local columns whose write is a membership change (a `One` column crossed by the
4796    /// query's own table).
4797    local_extra: RefCell<Vec<&'static str>>,
4798    set: RefCell<ResultSet>,
4799    /// Bumped whenever the result set changes; `ids()`/`count()` track it.
4800    version: day_reactive::Signal<u64>,
4801    pending: RefCell<QueryEvents>,
4802    /// A dependency-touching change arrived; the answer re-derives after the flush.
4803    needs_sql: Cell<bool>,
4804    // Reading the last committed result must not consume the post-commit invalidation.
4805    pending_commit: Cell<bool>,
4806    error: day_reactive::Signal<Option<DbError>>,
4807    /// `query_raw` only: the statement and the tables whose flush re-runs it.
4808    raw: Option<RawQuery>,
4809    /// A count-shaped query: `count` is the answer, the id set stays empty, and requeries
4810    /// run `SELECT COUNT(*)`, the badge query, O(1) memory at any result size.
4811    count_only: bool,
4812    count: Cell<usize>,
4813}
4814
4815struct RawQuery {
4816    sql: String,
4817    params: Vec<Value>,
4818    tables: Vec<String>,
4819}
4820
4821/// A live, typed result set over one model's table: ids only, answered by the engine and
4822/// kept current by dependency-gated requeries. Rows stay lazy: fault the ones you show
4823/// ([`Query::materialize`], or a list source doing it for you). Clone shares the same set.
4824pub struct Query<M: Model> {
4825    state: Rc<QueryState>,
4826    container: ModelContainer,
4827    _p: std::marker::PhantomData<fn() -> M>,
4828}
4829
4830impl<M: Model> Clone for Query<M> {
4831    fn clone(&self) -> Self {
4832        Query {
4833            state: self.state.clone(),
4834            container: self.container.clone(),
4835            _p: std::marker::PhantomData,
4836        }
4837    }
4838}
4839
4840impl<M: Model> Query<M> {
4841    /// This query's last read failure. A failed read preserves its previous result set.
4842    pub fn error(&self) -> day_reactive::Signal<Option<DbError>> {
4843        self.state.error
4844    }
4845
4846    /// Explicitly retry a query after a transient read failure (also useful after external SQL).
4847    pub fn refresh(&self) {
4848        self.state.needs_sql.set(true);
4849        self.refresh_if_stale();
4850    }
4851
4852    /// Checked form of `ids`, for imports and commands that must not mistake failure for absence.
4853    ///
4854    /// # Errors
4855    /// Returns the query's error when its last read failed.
4856    pub fn try_ids(&self) -> Result<Vec<ModelId<M>>, DbError> {
4857        let ids = self.ids();
4858        match self.state.error.get() {
4859            Some(e) => Err(e),
4860            None => Ok(ids),
4861        }
4862    }
4863
4864    /// A checked snapshot for non-UI work, fetched in batches. UI lists should use the query
4865    /// itself as their row source so they only materialize the visible working set.
4866    ///
4867    /// # Errors
4868    /// Returns the query's error when its last read failed, or the storage or decoding error of a
4869    /// row that cannot be read.
4870    pub fn try_collect(&self) -> Result<Vec<M>, DbError> {
4871        let ids = self.try_ids()?;
4872        let mut rows = Vec::with_capacity(ids.len());
4873        for chunk in ids.chunks(64) {
4874            self.container
4875                .ensure_resident::<M>(&chunk.iter().map(|id| id.handle()).collect::<Vec<_>>())?;
4876            #[expect(
4877                clippy::redundant_closure_for_method_calls,
4878                reason = "`Option::cloned` as a path is ambiguous between `Option<&T>` and `Option<&mut T>`, and naming the type fixes the reference lifetime"
4879            )]
4880            for id in chunk {
4881                if let Some(row) = self.container.try_get::<M>(*id)?
4882                    && let Some(value) = row.with_value_untracked(|v| v.cloned())
4883                {
4884                    rows.push(value);
4885                }
4886            }
4887        }
4888        Ok(rows)
4889    }
4890
4891    /// The result ids, typed and in query order, as a tracked read: the caller re-runs when
4892    /// the set changes, and only then.
4893    pub fn ids(&self) -> Vec<ModelId<M>> {
4894        self.refresh_if_stale();
4895        let _ = self.state.version.get();
4896        self.state
4897            .set
4898            .borrow()
4899            .ids()
4900            .iter()
4901            .map(|h| ModelId::from_handle(*h))
4902            .collect()
4903    }
4904
4905    /// The result count, tracked like [`Query::ids`].
4906    pub fn count(&self) -> usize {
4907        self.refresh_if_stale();
4908        let _ = self.state.version.get();
4909        self.state.set.borrow().ids().len()
4910    }
4911
4912    /// The first result, tracked.
4913    pub fn first(&self) -> Option<ModelId<M>> {
4914        self.refresh_if_stale();
4915        let _ = self.state.version.get();
4916        self.state
4917            .set
4918            .borrow()
4919            .ids()
4920            .first()
4921            .map(|h| ModelId::from_handle(*h))
4922    }
4923
4924    /// Tracked membership test.
4925    pub fn contains(&self, id: impl Into<ModelId<M>>) -> bool {
4926        self.refresh_if_stale();
4927        let _ = self.state.version.get();
4928        self.state.set.borrow().ids().contains(&id.into().handle())
4929    }
4930
4931    /// Untracked snapshot: reactively silent, but still current (pending staleness settles
4932    /// first, like every other read).
4933    pub fn ids_untracked(&self) -> Vec<ModelId<M>> {
4934        self.refresh_if_stale();
4935        self.state
4936            .set
4937            .borrow()
4938            .ids()
4939            .iter()
4940            .map(|h| ModelId::from_handle(*h))
4941            .collect()
4942    }
4943
4944    /// Bring this window of the result into the cache: what a list binding calls for the
4945    /// rows it is about to show, batched into one `SELECT`.
4946    pub fn materialize(&self, range: std::ops::Range<usize>) {
4947        let keys: Vec<u64> = {
4948            let set = self.state.set.borrow();
4949            let ids = set.ids();
4950            let end = range.end.min(ids.len());
4951            let start = range.start.min(end);
4952            ids[start..end].to_vec()
4953        };
4954        if let Err(e) = self.container.ensure_resident::<M>(&keys) {
4955            self.container.inner.error.set(Some(e.to_string()));
4956        }
4957    }
4958
4959    /// Swap the fetch (a changed filter or sort): requeries and reloads consumers. No-op when
4960    /// equal to the current one, so a `query_fn` closure can re-run cheaply.
4961    pub fn set_fetch(&self, fetch: Fetch) {
4962        if *self.state.set.borrow().fetch() == fetch {
4963            return;
4964        }
4965        self.container.rewire_watches(&self.state, &fetch);
4966        {
4967            let mut set = self.state.set.borrow_mut();
4968            *set = ResultSet::new(fetch);
4969        }
4970        // Resolve once pending statements land; mid-turn the flush has not run yet.
4971        self.state.needs_sql.set(true);
4972        self.state
4973            .pending_commit
4974            .set(!self.container.inner.dirty.borrow().is_empty());
4975        self.refresh_if_stale();
4976        self.state.pending.borrow_mut().reload();
4977        self.bump();
4978    }
4979
4980    /// Drain what changed since the last call: the list source's feed. Pending staleness
4981    /// settles first, so a drain right after an edit already narrates it.
4982    pub fn take_events(&self) -> QueryEvents {
4983        self.refresh_if_stale();
4984        std::mem::replace(&mut *self.state.pending.borrow_mut(), QueryEvents::None)
4985    }
4986
4987    /// A dependency-touching change arrived this turn and the flush has not run yet: settle
4988    /// it now. With autosave the pending statements flush (which requeries everything stale);
4989    /// with autosave off the query re-derives against the last save; the file is the only
4990    /// thing that can answer it.
4991    fn refresh_if_stale(&self) {
4992        if !self.state.needs_sql.get() {
4993            return;
4994        }
4995        if self.container.inner.autosave.get() {
4996            let _ = self.container.save();
4997        }
4998        if self.state.needs_sql.replace(false) {
4999            self.container.requery(&self.state);
5000        }
5001    }
5002
5003    fn bump(&self) {
5004        self.state
5005            .version
5006            .set(self.state.version.get_untracked().wrapping_add(1));
5007    }
5008}
5009
5010/// Builder for a [`Query`]: `container.query::<Trip>().filter(…).sort(…).live()`.
5011pub struct QueryBuilder<'c, M: Model> {
5012    container: &'c ModelContainer,
5013    fetch: Fetch,
5014    _p: std::marker::PhantomData<fn() -> M>,
5015}
5016
5017impl<M: Model> QueryBuilder<'_, M> {
5018    #[must_use]
5019    pub fn filter(mut self, p: Pred) -> Self {
5020        self.fetch = self.fetch.filter(p);
5021        self
5022    }
5023    #[must_use]
5024    pub fn sort(mut self, s: Sort) -> Self {
5025        self.fetch = self.fetch.sort(s);
5026        self
5027    }
5028    #[must_use]
5029    pub fn limit(mut self, n: usize) -> Self {
5030        self.fetch = self.fetch.limit(n);
5031        self
5032    }
5033    /// Run the fetch and keep it live against the change log.
5034    pub fn live(self) -> Query<M> {
5035        self.container.install_query::<M>(self.fetch, None)
5036    }
5037
5038    /// Keep only the count live, the badge form: no id vector, one `SELECT COUNT(*)` per
5039    /// requery, the same dependency gating.
5040    pub fn live_count(self) -> CountQuery<M> {
5041        CountQuery {
5042            state: self.container.install_state::<M>(&self.fetch, None, true),
5043            container: self.container.clone(),
5044            _p: std::marker::PhantomData,
5045        }
5046    }
5047}
5048
5049/// A live count over one model's table, the badge query: `SELECT COUNT(*)` behind the same
5050/// dependency-gated staleness as [`Query`], holding no id vector, so a count over a
5051/// million-row result costs O(1) memory. Clone shares the same state.
5052pub struct CountQuery<M: Model> {
5053    state: Rc<QueryState>,
5054    container: ModelContainer,
5055    _p: std::marker::PhantomData<fn() -> M>,
5056}
5057
5058impl<M: Model> Clone for CountQuery<M> {
5059    fn clone(&self) -> Self {
5060        CountQuery {
5061            state: self.state.clone(),
5062            container: self.container.clone(),
5063            _p: std::marker::PhantomData,
5064        }
5065    }
5066}
5067
5068impl<M: Model> CountQuery<M> {
5069    /// The last read error, independently of the last successful count.
5070    pub fn error(&self) -> day_reactive::Signal<Option<DbError>> {
5071        self.state.error
5072    }
5073    /// Checked count for operations that require a successful read.
5074    ///
5075    /// # Errors
5076    /// Returns the query's error when its last read failed.
5077    pub fn try_get(&self) -> Result<usize, DbError> {
5078        let n = self.get();
5079        match self.state.error.get() {
5080            Some(e) => Err(e),
5081            None => Ok(n),
5082        }
5083    }
5084    /// Retry after a transient read error.
5085    pub fn refresh(&self) {
5086        self.state.needs_sql.set(true);
5087        self.refresh_if_stale();
5088    }
5089
5090    /// The count, as a tracked read: the caller re-runs when it changes, and only then.
5091    pub fn get(&self) -> usize {
5092        self.refresh_if_stale();
5093        let _ = self.state.version.get();
5094        self.state.count.get()
5095    }
5096
5097    /// Untracked snapshot: reactively silent, still current.
5098    pub fn get_untracked(&self) -> usize {
5099        self.refresh_if_stale();
5100        self.state.count.get()
5101    }
5102
5103    /// Swap the fetch (a changed filter): re-counts. No-op when equal, so a `count_fn`
5104    /// closure can re-run cheaply.
5105    pub fn set_fetch(&self, fetch: Fetch) {
5106        if *self.state.set.borrow().fetch() == fetch {
5107            return;
5108        }
5109        self.container.rewire_watches(&self.state, &fetch);
5110        *self.state.set.borrow_mut() = ResultSet::new(fetch);
5111        self.state.needs_sql.set(true);
5112        self.state
5113            .pending_commit
5114            .set(!self.container.inner.dirty.borrow().is_empty());
5115        self.refresh_if_stale();
5116    }
5117
5118    fn refresh_if_stale(&self) {
5119        if !self.state.needs_sql.get() {
5120            return;
5121        }
5122        if self.container.inner.autosave.get() {
5123            let _ = self.container.save();
5124        }
5125        if self.state.needs_sql.replace(false) {
5126            self.container.requery(&self.state);
5127        }
5128    }
5129}
5130
5131/// A fallback row: the key plus the dependency columns the fallback `SELECT` carried.
5132struct FallbackRow<'a> {
5133    cols: &'a [&'static str],
5134    /// values[0] is the key; values[1..] align with `cols`.
5135    values: &'a [Value],
5136}
5137
5138impl RowView for FallbackRow<'_> {
5139    fn col(&self, c: &str) -> Option<Value> {
5140        self.cols
5141            .iter()
5142            .position(|n| *n == c)
5143            .and_then(|i| self.values.get(i + 1).cloned())
5144    }
5145}
5146
5147impl ModelContainer {
5148    /// Start a typed query over `M`'s rows. Panics (like [`ModelContainer::cache`]) if `M` is
5149    /// not in this container's `schema!`, a wiring bug worth stopping on.
5150    pub fn query<M: Model>(&self) -> QueryBuilder<'_, M> {
5151        let _ = self.cache::<M>();
5152        QueryBuilder {
5153            container: self,
5154            fetch: Fetch::new(),
5155            _p: std::marker::PhantomData,
5156        }
5157    }
5158
5159    /// The reactive-fetch form: `f` is a computation, so a query whose fetch depends on signals
5160    /// (a search term, a filter toggle) re-derives itself when they change.
5161    pub fn query_fn<M: Model>(&self, f: impl Fn() -> Fetch + 'static) -> Query<M> {
5162        let q = self.install_query::<M>(f(), None);
5163        let q2 = q.clone();
5164        day_reactive::bind(f, move |fetch| q2.set_fetch(fetch.clone()));
5165        q
5166    }
5167
5168    /// The reactive-fetch count: a badge whose fetch depends on signals (the "Today" cutoff,
5169    /// a filter toggle) re-counts itself when they change.
5170    pub fn count_fn<M: Model>(&self, f: impl Fn() -> Fetch + 'static) -> CountQuery<M> {
5171        let q = CountQuery::<M> {
5172            state: self.install_state::<M>(&f(), None, true),
5173            container: self.clone(),
5174            _p: std::marker::PhantomData,
5175        };
5176        let q2 = q.clone();
5177        day_reactive::bind(f, move |fetch| q2.set_fetch(fetch.clone()));
5178        q
5179    }
5180
5181    /// Raw SQL, declared: a read-only SELECT returning ids, re-run whenever a flush touches
5182    /// one of the named tables. The price of the escape hatch is whole-query invalidation.
5183    pub fn query_raw<M: Model>(
5184        &self,
5185        sql: impl Into<String>,
5186        params: Vec<Value>,
5187        tables: &[&str],
5188    ) -> Query<M> {
5189        self.install_query::<M>(
5190            Fetch::new().filter(Pred::Raw(String::new(), Vec::new())),
5191            Some(RawQuery {
5192                sql: sql.into(),
5193                params,
5194                tables: tables
5195                    .iter()
5196                    .map(std::string::ToString::to_string)
5197                    .collect(),
5198            }),
5199        )
5200    }
5201
5202    /// Opt into undo: One history over every store this container manages, `levels` deep.
5203    /// Undo/redo replay flows through the same change pipeline as the user's edits, so
5204    /// autosave writes the inverse statements and live queries hear rows come back. Call it
5205    /// once, after open; clear it on migration (`UndoStack::clear`). While a stack is
5206    /// installed, deletes materialize the rows they remove (a cascade included) so the
5207    /// history can restore them.
5208    pub fn undo(&self, levels: usize) -> day_model::UndoStack {
5209        let stack = day_model::UndoStack::new(levels);
5210        for hooks in self.inner.tables.borrow().values() {
5211            (hooks.watch_undo)(&stack);
5212        }
5213        stack
5214    }
5215
5216    /// The driver's connection, directly, for maintenance, imports, or an extension's
5217    /// statements. Pending changes flush first; writes made here bypass the change log, so
5218    /// call [`ModelContainer::rescan`] afterward if they touched Day's tables.
5219    ///
5220    /// # Panics
5221    /// Panics if the pending save fails. Use [`ModelContainer::try_with_connection`] when
5222    /// the caller needs to handle storage errors.
5223    pub fn with_connection<R>(&self, f: impl FnOnce(&mut dyn SqliteConnection) -> R) -> R {
5224        self.save().expect(
5225            "pending persistence save failed; use try_with_connection to handle storage errors",
5226        );
5227        f(self.conn().as_mut())
5228    }
5229
5230    /// Flush pending edits before accessing the connection. The closure is never run if
5231    /// that save fails; its own error is returned unchanged.
5232    ///
5233    /// # Errors
5234    /// Returns the error of the save, or the closure's own.
5235    pub fn try_with_connection<R>(
5236        &self,
5237        f: impl FnOnce(&mut dyn SqliteConnection) -> Result<R, DbError>,
5238    ) -> Result<R, DbError> {
5239        self.save()?;
5240        f(self.conn().as_mut())
5241    }
5242
5243    /// Re-read the resident rows from the file and re-run every query: the recovery from
5244    /// writes that bypassed the change log ([`ModelContainer::with_connection`], another
5245    /// process). O(working set + query results), never O(table).
5246    ///
5247    /// # Errors
5248    /// Returns the storage error when pending changes cannot be saved or the rows cannot be read
5249    /// again.
5250    pub fn rescan(&self) -> Result<(), DbError> {
5251        self.save()?;
5252        self.inner.blob_memo.borrow_mut().clear();
5253        self.refresh_resident()?;
5254        self.invalidate_relation_memos();
5255        self.requery_all();
5256        // The cache now equals the file; a later check_external should not re-merge for
5257        // whatever external commits this refresh already picked up.
5258        if self.inner.caps.external_changes {
5259            self.inner.data_version.set(self.file_data_version()?);
5260        }
5261        Ok(())
5262    }
5263
5264    /// Attach another SQLite file under `alias` and bring `schema`'s models (every one
5265    /// declared `external = "alias"`) into this container: their tables are read as they
5266    /// are, with no DDL, migration, or fingerprint, through the same caches, queries, and
5267    /// lists as the container's own. Writes to them fail at flush (the file is attached
5268    /// read-only where the engine honors `mode=ro`), which is the contract: the other
5269    /// database has its own owner.
5270    ///
5271    /// Calling it again with the same alias swaps the file: the old one is detached, every
5272    /// resident row of its models dropped, and every live query re-run: the shape a
5273    /// downloaded catalog update takes.
5274    ///
5275    /// # Errors
5276    /// Returns an error when `alias` is not a plain identifier, the file cannot be attached, or its
5277    /// tables do not fit `schema`.
5278    pub fn attach_database(&self, alias: &str, path: &str, schema: Schema) -> Result<(), DbError> {
5279        if alias.is_empty() || !alias.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') {
5280            return Err(DbError::new(
5281                DbErrorKind::Unsupported,
5282                format!("attach_database: {alias:?} is not a plain SQL schema name"),
5283            ));
5284        }
5285        let held = self.inner.attached.borrow().iter().any(|a| a == alias);
5286        if held {
5287            self.detach_database(alias)?;
5288        }
5289        // Flush first: a DETACH/ATTACH pair inside an open write transaction is refused.
5290        if !self.inner.dirty.borrow().is_empty() {
5291            self.save()?;
5292        }
5293        // `file:` URIs carry the read-only mode; a driver without URI filenames (the web
5294        // worker) takes the bare path and relies on nobody writing.
5295        let uri = if path.starts_with("file:") || path.contains("://") {
5296            path.to_string()
5297        } else {
5298            format!("file:{}?mode=ro", path.replace('?', "%3F"))
5299        };
5300        let attach = |target: &str| -> Result<(), DbError> {
5301            self.conn().execute(
5302                &format!("ATTACH DATABASE ? AS {}", ident(alias)),
5303                &[Value::Text(target.to_string())],
5304            )?;
5305            Ok(())
5306        };
5307        if let Err(uri_err) = attach(&uri) {
5308            attach(path).map_err(|_| uri_err)?;
5309        }
5310        self.inner.attached.borrow_mut().push(alias.to_string());
5311        let Schema {
5312            installers, wirers, ..
5313        } = schema;
5314        for install in installers {
5315            install(self)?;
5316        }
5317        for wire in wirers {
5318            wire(self)?;
5319        }
5320        self.rescan()
5321    }
5322
5323    /// Detach a database [`ModelContainer::attach_database`] holds, dropping every resident
5324    /// row of its models. Their stores stay registered (empty), so a query over them answers
5325    /// nothing until the alias is attached again.
5326    ///
5327    /// # Errors
5328    /// Returns the storage error when pending changes cannot be saved or the database cannot be
5329    /// detached.
5330    pub fn detach_database(&self, alias: &str) -> Result<(), DbError> {
5331        type Doomed = Vec<(Rc<dyn Fn() -> Vec<u64>>, EvictFn)>;
5332        let position = self.inner.attached.borrow().iter().position(|a| a == alias);
5333        let Some(position) = position else {
5334            return Ok(());
5335        };
5336        if !self.inner.dirty.borrow().is_empty() {
5337            self.save()?;
5338        }
5339        self.inner.blob_memo.borrow_mut().clear();
5340        let doomed: Doomed = self
5341            .inner
5342            .tables
5343            .borrow()
5344            .values()
5345            .filter(|h| h.name.schema == Some(alias))
5346            .map(|h| (h.resident_keys.clone(), h.evict.clone()))
5347            .collect();
5348        for (keys, evict) in &doomed {
5349            let resident = keys();
5350            let _ = evict(&resident, resident.len());
5351        }
5352        self.conn()
5353            .execute(&format!("DETACH DATABASE {}", ident(alias)), &[])?;
5354        self.inner.attached.borrow_mut().remove(position);
5355        self.rescan()
5356    }
5357
5358    /// Look for other connections' committed writes (another process, a sync engine, a CLI)
5359    /// and merge what changed. Detection is one `PRAGMA data_version` (the counter moves only
5360    /// when another connection commits, never for this one's own writes), so this is cheap
5361    /// enough to wire to app foreground, window focus, or a timer.
5362    ///
5363    /// When the counter moved, pending local edits flush first; then the resident rows are
5364    /// re-read and diffed: changed fields announce per column, disappearances take the
5365    /// structural path, every live query re-derives, all authored
5366    /// [`ModelContainer::EXTERNAL_AUTHOR`], so the autosave fold declines the echo and an
5367    /// installed undo stack skips them (another author's writes are not the user's history).
5368    /// Rows that arrived elsewhere become visible through the re-run queries and fault in
5369    /// like any other row. A row another connection rewrote arrives whole, so its
5370    /// `#[model(transient)]` fields reset to their defaults, exactly as at fault.
5371    ///
5372    /// Returns whether anything arrived. On a driver without detection
5373    /// ([`Capabilities::external_changes`]) this is `Ok(false)`; writes made
5374    /// through [`ModelContainer::with_connection`] are this connection's own and stay
5375    /// [`ModelContainer::rescan`]'s job.
5376    ///
5377    /// # Errors
5378    /// Returns the storage error when the file cannot be read.
5379    pub fn check_external(&self) -> Result<bool, DbError> {
5380        if !self.inner.caps.external_changes {
5381            return Ok(false);
5382        }
5383        let Some(current) = self.file_data_version()? else {
5384            return Ok(false);
5385        };
5386        if self.inner.data_version.get() == Some(current) {
5387            return Ok(false);
5388        }
5389        // Local edits flush first, so the diff compares the file against a cache with nothing
5390        // pending: an unflushed local edit must not read as the other side's deletion.
5391        self.save()?;
5392        self.refresh_resident()?;
5393        self.invalidate_relation_memos();
5394        self.requery_all();
5395        // A commit matters even when it inserted only nonresident rows: count/projection
5396        // observers still need invalidation. Advance only after a successful merge so errors
5397        // remain retryable.
5398        self.inner.data_version.set(Some(current));
5399        Ok(true)
5400    }
5401
5402    /// `PRAGMA data_version`, or `None` where the engine did not answer (the Recorder).
5403    fn file_data_version(&self) -> Result<Option<i64>, DbError> {
5404        let mut v = None;
5405        self.conn().query("PRAGMA data_version", &[], &mut |row| {
5406            if let Ok(i) = row.get(0).as_int() {
5407                v = Some(i);
5408            }
5409        })?;
5410        Ok(v)
5411    }
5412
5413    /// Re-select every resident row and feed the differences through the caches: per-field
5414    /// announcements, external deletions, all authored [`ModelContainer::EXTERNAL_AUTHOR`].
5415    fn refresh_resident(&self) -> Result<bool, DbError> {
5416        self.refresh_resident_in(None)
5417    }
5418
5419    /// [`ModelContainer::refresh_resident`] over the tables named (all of them for `None`).
5420    fn refresh_resident_in(&self, only: Option<&[&'static str]>) -> Result<bool, DbError> {
5421        struct RefreshJob {
5422            store: u64,
5423            resident_keys: Rc<dyn Fn() -> Vec<u64>>,
5424            refresh: RefreshFn,
5425            columns: String,
5426            table: String,
5427            keys_where: KeysWhere,
5428        }
5429        let jobs: Vec<RefreshJob> = {
5430            let tables = self.inner.tables.borrow();
5431            tables
5432                .iter()
5433                .filter(|(_, h)| only.is_none_or(|t| t.contains(&h.table)))
5434                .map(|(store, h)| RefreshJob {
5435                    store: *store,
5436                    resident_keys: h.resident_keys.clone(),
5437                    refresh: h.refresh.clone(),
5438                    columns: select_list(h),
5439                    table: h.name.sql(),
5440                    keys_where: keys_where_of(h),
5441                })
5442                .collect()
5443        };
5444        let mut changed = false;
5445        for RefreshJob {
5446            store,
5447            resident_keys,
5448            refresh,
5449            columns,
5450            table,
5451            keys_where,
5452        } in jobs
5453        {
5454            // Another writer may have replaced the bytes; the memo answers from the file next.
5455            self.inner.blob_memo.borrow_mut().forget_store(store);
5456            let resident = resident_keys();
5457            if resident.is_empty() {
5458                continue;
5459            }
5460            let mut rows: Vec<Vec<Value>> = Vec::new();
5461            for chunk in resident.chunks(MAX_BOUND_PARAMS / 2) {
5462                let (clause, params) = keys_where(chunk);
5463                self.conn().query(
5464                    &format!("SELECT {columns} FROM {table} WHERE {clause}"),
5465                    &params,
5466                    &mut |row| {
5467                        let n = Row::len(row);
5468                        rows.push((0..n).map(|i| row.get(i)).collect());
5469                    },
5470                )?;
5471            }
5472            // The connection borrow is released before refresh announces; a sink hearing
5473            // these changes may read back through SQL.
5474            if refresh(rows)? {
5475                changed = true;
5476            }
5477        }
5478        Ok(changed)
5479    }
5480
5481    fn install_state<M: Model>(
5482        &self,
5483        fetch: &Fetch,
5484        raw: Option<RawQuery>,
5485        count_only: bool,
5486    ) -> Rc<QueryState> {
5487        let store = self.cache::<M>();
5488        let state = Rc::new(QueryState {
5489            store_id: store.store_id(),
5490            table: M::TABLE,
5491            watches: RefCell::new(Vec::new()),
5492            local_extra: RefCell::new(Vec::new()),
5493            set: RefCell::new(ResultSet::new(fetch.clone())),
5494            version: day_reactive::Scope::detached().enter(|| day_reactive::Signal::new(0)),
5495            pending: RefCell::new(QueryEvents::None),
5496            needs_sql: Cell::new(false),
5497            pending_commit: Cell::new(!self.inner.dirty.borrow().is_empty()),
5498            error: day_reactive::Scope::detached().enter(|| day_reactive::Signal::new(None)),
5499            raw,
5500            count_only,
5501            count: Cell::new(0),
5502        });
5503        self.rewire_watches(&state, fetch);
5504        self.inner.queries.borrow_mut().push(Rc::downgrade(&state));
5505        // The seed: flush anything pending (with autosave; without it the query answers from
5506        // the last save, documented), then one SQL round trip.
5507        if self.inner.autosave.get() && !self.inner.dirty.borrow().is_empty() {
5508            let _ = self.save();
5509        }
5510        if count_only {
5511            state.count.set(self.answer_count(&state));
5512        } else {
5513            let ids = self.answer(&state);
5514            state.set.borrow_mut().reset(ids);
5515        }
5516        // The seed is the list's starting snapshot, not a stream of inserts to replay.
5517        *state.pending.borrow_mut() = QueryEvents::None;
5518        state
5519    }
5520
5521    #[expect(
5522        clippy::needless_pass_by_value,
5523        reason = "every caller builds the fetch for this call and hands it over"
5524    )]
5525    fn install_query<M: Model>(&self, fetch: Fetch, raw: Option<RawQuery>) -> Query<M> {
5526        Query {
5527            state: self.install_state::<M>(&fetch, raw, false),
5528            container: self.clone(),
5529            _p: std::marker::PhantomData,
5530        }
5531    }
5532
5533    /// (Re)compute a query's relation subscriptions from its fetch.
5534    fn rewire_watches(&self, state: &Rc<QueryState>, fetch: &Fetch) {
5535        let deps = fetch.dependencies();
5536        let mut watches: Vec<RelWatch> = Vec::new();
5537        let mut local_extra: Vec<&'static str> = Vec::new();
5538        let relations = self.inner.relations.borrow();
5539        let joins = self.inner.joins.borrow();
5540        for dep in &deps.related {
5541            let mut resolved = false;
5542            for r in relations.iter() {
5543                if r.parent_table == dep.owner && r.parent_field == dep.field {
5544                    watches.push(RelWatch {
5545                        store: r.child_store,
5546                        join_store: None,
5547                        fields: {
5548                            let mut f = vec![r.fk_field];
5549                            if let Some(o) = r.ordered {
5550                                f.push(o);
5551                            }
5552                            f
5553                        },
5554                        columns: dep.columns.clone(),
5555                    });
5556                    resolved = true;
5557                    break;
5558                }
5559                if r.child_table == dep.owner && r.fk_field == dep.field {
5560                    // The membership column lives on the query's table: a rewrite of the
5561                    // reference must mark it stale even though no predicate reads it.
5562                    if dep.owner == state.table && !local_extra.contains(&r.fk_col) {
5563                        local_extra.push(r.fk_col);
5564                    }
5565                    watches.push(RelWatch {
5566                        store: r.parent_store,
5567                        join_store: None,
5568                        fields: Vec::new(),
5569                        columns: dep.columns.clone(),
5570                    });
5571                    resolved = true;
5572                    break;
5573                }
5574            }
5575            if resolved {
5576                continue;
5577            }
5578            // A link: the far store's read columns matter, and so does the owner's value
5579            // column; rewriting it is what moves the row across the link.
5580            if let Some(l) = self
5581                .inner
5582                .links
5583                .borrow()
5584                .iter()
5585                .find(|l| l.owner_table == dep.owner && l.field == dep.field)
5586                .copied()
5587            {
5588                if dep.owner == state.table && !local_extra.contains(&l.local) {
5589                    local_extra.push(l.local);
5590                }
5591                let far = self
5592                    .inner
5593                    .tables
5594                    .borrow()
5595                    .iter()
5596                    .find(|(_, h)| h.table == l.target_table)
5597                    .map(|(id, _)| *id);
5598                if let Some(store) = far {
5599                    let mut columns = dep.columns.clone();
5600                    if !columns.contains(&l.remote) {
5601                        columns.push(l.remote);
5602                    }
5603                    watches.push(RelWatch {
5604                        store,
5605                        join_store: None,
5606                        fields: Vec::new(),
5607                        columns,
5608                    });
5609                }
5610                continue;
5611            }
5612            for j in joins.iter() {
5613                let side = if j.a_table == dep.owner && j.a_field == dep.field {
5614                    Some(j.b_store)
5615                } else if j.b_table == dep.owner && j.b_field.get() == Some(dep.field) {
5616                    Some(j.a_store)
5617                } else {
5618                    None
5619                };
5620                if let Some(other) = side {
5621                    watches.push(RelWatch {
5622                        store: other,
5623                        join_store: Some(j.join_store),
5624                        fields: Vec::new(),
5625                        columns: dep.columns.clone(),
5626                    });
5627                    break;
5628                }
5629            }
5630        }
5631        *state.watches.borrow_mut() = watches;
5632        *state.local_extra.borrow_mut() = local_extra;
5633    }
5634
5635    /// One announced change: decide, per live query, whether it can move the result, and
5636    /// mark stale where it can. No SQL runs here; the requery happens once, after the flush.
5637    fn mark_queries_stale(&self, change: &day_model::Change) {
5638        let Some(&store) = change.components.first() else {
5639            return;
5640        };
5641        let states: Vec<Rc<QueryState>> = {
5642            let mut queries = self.inner.queries.borrow_mut();
5643            queries.retain(|w| w.strong_count() > 0);
5644            queries.iter().filter_map(std::rc::Weak::upgrade).collect()
5645        };
5646        if states.is_empty() {
5647            return;
5648        }
5649        let key = change.components.get(1).copied();
5650        if key == Some(day_model::STRUCTURE) {
5651            return; // the shape path duplicates the row paths
5652        }
5653        // The change log speaks field names; predicates speak column names.
5654        let (column, changed_table): (Option<String>, Option<&'static str>) = {
5655            let tables = self.inner.tables.borrow();
5656            let hooks = tables.get(&store);
5657            let column = if change.components.len() >= 3 {
5658                hooks.map(|h| {
5659                    h.fields
5660                        .iter()
5661                        .position(|f| f == change.label)
5662                        .map_or_else(|| "\u{0}".to_string(), |i| h.columns[i].clone())
5663                })
5664            } else {
5665                None
5666            };
5667            (column, hooks.map(|h| h.table))
5668        };
5669        let column = column.as_deref();
5670
5671        for state in &states {
5672            if state.needs_sql.get() {
5673                continue; // already stale; nothing to learn
5674            }
5675            if let Some(raw) = &state.raw {
5676                if changed_table.is_some_and(|t| raw.tables.iter().any(|r| r == t)) {
5677                    state.needs_sql.set(true);
5678                }
5679                continue;
5680            }
5681            // Both gates apply: a self-referential relation makes the query's own store a
5682            // related store too, so the own-table check must not shadow the watch check.
5683            let own_stale = state.store_id == store
5684                && match (key, change.op) {
5685                    // No key is a wholesale rewrite: anything may have moved.
5686                    (None, _) | (Some(_), Op::Insert | Op::Delete) => true,
5687                    (Some(_), Op::Set) => match column {
5688                        // A row-level merge without a field name: assume the worst.
5689                        None => true,
5690                        Some(c) => {
5691                            let set = state.set.borrow();
5692                            set.deps().touches_local(c) || state.local_extra.borrow().contains(&c)
5693                        }
5694                    },
5695                    (Some(_), Op::Move) => false, // user order is not a query's business
5696                };
5697            let stale = own_stale || {
5698                let watches = state.watches.borrow();
5699                watches.iter().any(|w| {
5700                    if w.join_store == Some(store) {
5701                        return true; // every link/unlink/reposition is a membership change
5702                    }
5703                    if w.store != store {
5704                        return false;
5705                    }
5706                    match (key, change.op) {
5707                        (None, _) | (Some(_), Op::Insert | Op::Delete) => true,
5708                        (Some(_), Op::Set) => {
5709                            w.fields.contains(&change.label)
5710                                || column.is_some_and(|c| w.columns.contains(&c))
5711                        }
5712                        (Some(_), Op::Move) => false,
5713                    }
5714                })
5715            };
5716            if stale {
5717                state.needs_sql.set(true);
5718                state.pending_commit.set(true);
5719            }
5720        }
5721    }
5722
5723    /// After a flush: every stale query re-derives (plus raw queries whose tables the flush
5724    /// touched), each answer diffed into list deltas.
5725    fn run_deferred_requeries(&self, stmts: &[(String, Vec<Value>)]) {
5726        let states: Vec<Rc<QueryState>> = self
5727            .inner
5728            .queries
5729            .borrow()
5730            .iter()
5731            .filter_map(std::rc::Weak::upgrade)
5732            .collect();
5733        for state in states {
5734            let mut due = state.needs_sql.replace(false) | state.pending_commit.replace(false);
5735            if let Some(raw) = &state.raw
5736                && !due
5737            {
5738                due = stmts
5739                    .iter()
5740                    .any(|(sql, _)| raw.tables.iter().any(|t| statement_touches(sql, t)));
5741            }
5742            if due {
5743                self.requery(&state);
5744            }
5745        }
5746    }
5747
5748    /// Mark every query stale and requery now: the external-change and rescan path.
5749    fn requery_all(&self) {
5750        let states: Vec<Rc<QueryState>> = self
5751            .inner
5752            .queries
5753            .borrow()
5754            .iter()
5755            .filter_map(std::rc::Weak::upgrade)
5756            .collect();
5757        for state in states {
5758            state.needs_sql.set(false);
5759            self.requery(&state);
5760        }
5761    }
5762
5763    /// [`ModelContainer::requery_all`] for the live queries that read `tables`: a query over
5764    /// one of them, one crossing into one through a relation (or that relation's join
5765    /// table), or a raw query naming one.
5766    fn requery_tables(&self, tables: &[&'static str]) {
5767        let stores: Vec<u64> = self
5768            .inner
5769            .tables
5770            .borrow()
5771            .iter()
5772            .filter(|(_, h)| tables.contains(&h.table))
5773            .map(|(id, _)| *id)
5774            .collect();
5775        let states: Vec<Rc<QueryState>> = self
5776            .inner
5777            .queries
5778            .borrow()
5779            .iter()
5780            .filter_map(std::rc::Weak::upgrade)
5781            .collect();
5782        for state in states {
5783            let reads = match &state.raw {
5784                Some(raw) => raw.tables.iter().any(|t| tables.contains(&t.as_str())),
5785                None => {
5786                    stores.contains(&state.store_id)
5787                        || state.watches.borrow().iter().any(|w| {
5788                            stores.contains(&w.store)
5789                                || w.join_store.is_some_and(|j| stores.contains(&j))
5790                        })
5791                }
5792            };
5793            if reads {
5794                state.needs_sql.set(false);
5795                self.requery(&state);
5796            }
5797        }
5798    }
5799
5800    /// One requery: answer through the engine, adopt, narrate.
5801    fn requery(&self, state: &Rc<QueryState>) {
5802        if state.count_only {
5803            let n = self.answer_count(state);
5804            if n != state.count.get() {
5805                state.count.set(n);
5806                state
5807                    .version
5808                    .set(state.version.get_untracked().wrapping_add(1));
5809            }
5810            return;
5811        }
5812        let ids = self.answer(state);
5813        let change = state.set.borrow_mut().adopt(ids);
5814        match change {
5815            SetChange::Same => {}
5816            SetChange::Deltas(d) => {
5817                state.pending.borrow_mut().push(&d);
5818                state
5819                    .version
5820                    .set(state.version.get_untracked().wrapping_add(1));
5821            }
5822            SetChange::Reload => {
5823                state.pending.borrow_mut().reload();
5824                state
5825                    .version
5826                    .set(state.version.get_untracked().wrapping_add(1));
5827            }
5828        }
5829    }
5830
5831    /// Answer a COUNT-shaped fetch: one `SELECT COUNT(*)`, capped by the fetch's limit
5832    /// (a limited set's length is `min(count, limit)`). The fallback form counts its
5833    /// re-checked rows the same way.
5834    fn answer_count(&self, state: &Rc<QueryState>) -> usize {
5835        let result = (|| {
5836            if self.table_detached(state.table) {
5837                return Ok(0);
5838            }
5839            if let Some(raw) = &state.raw {
5840                return self
5841                    .select_ids_checked(&raw.sql, &raw.params)
5842                    .map(|ids| ids.len());
5843            }
5844            let fetch = state.set.borrow().fetch().clone();
5845            let cap = fetch.limit.unwrap_or(usize::MAX);
5846            let snapshot = self.sql_snapshot();
5847            match compile_count(state.table, &fetch, &snapshot) {
5848                Ok(q) => {
5849                    let mut n = Ok(0_i64);
5850                    self.conn().query(&q.sql, &q.params, &mut |row| {
5851                        n = row.get(0).as_int();
5852                    })?;
5853                    Ok((n?.max(0) as usize).min(cap))
5854                }
5855                Err(CompileErr::NeedsFold) => {
5856                    let (q, cols) = compile_fallback(state.table, &fetch, &snapshot)
5857                        .map_err(|e| DbError::new(DbErrorKind::Unsupported, e.message()))?;
5858                    let mut n = 0;
5859                    self.conn().query(&q.sql, &q.params, &mut |row| {
5860                        let values = (0..row.len()).map(|i| row.get(i)).collect::<Vec<_>>();
5861                        if let Some(h) = value_to_handle(&values[0])
5862                            && fetch.pred.eval(
5863                                h,
5864                                &FallbackRow {
5865                                    cols: &cols,
5866                                    values: &values,
5867                                },
5868                            )
5869                        {
5870                            n += 1;
5871                        }
5872                    })?;
5873                    Ok(n.min(cap))
5874                }
5875                Err(e) => Err(DbError::new(DbErrorKind::Unsupported, e.message())),
5876            }
5877        })();
5878        match result {
5879            Ok(n) => {
5880                state.error.set(None);
5881                n
5882            }
5883            Err(error) => {
5884                self.inner.error.set(Some(error.to_string()));
5885                state.error.set(Some(error));
5886                state.count.get()
5887            }
5888        }
5889    }
5890
5891    /// Answer a query's fetch through the engine: the compiled form, the fallback form for
5892    /// drivers without `day_fold`, or the raw statement.
5893    fn answer(&self, state: &Rc<QueryState>) -> Vec<u64> {
5894        match self.answer_checked(state) {
5895            Ok(ids) => {
5896                state.error.set(None);
5897                ids
5898            }
5899            Err(error) => {
5900                self.inner.error.set(Some(error.to_string()));
5901                state.error.set(Some(error));
5902                // A failed read is not an empty result. Retain the last successful set.
5903                state.set.borrow().ids().to_vec()
5904            }
5905        }
5906    }
5907    /// Whether `table` belongs to an attached database that is not attached right now. The
5908    /// alias is the model's declared one, not a prefix of the name: a table of the
5909    /// container's own may be called `dot.ted`.
5910    fn table_detached(&self, table: &str) -> bool {
5911        let tables = self.inner.tables.borrow();
5912        let Some(alias) = tables
5913            .values()
5914            .find(|h| h.table == table)
5915            .and_then(|h| h.name.schema)
5916        else {
5917            return false;
5918        };
5919        alias != "main"
5920            && alias != "temp"
5921            && !self.inner.attached.borrow().iter().any(|a| a == alias)
5922    }
5923    fn answer_checked(&self, state: &Rc<QueryState>) -> Result<Vec<u64>, DbError> {
5924        // Explicit detachment is a successful transition to an empty catalog, not an I/O error.
5925        if self.table_detached(state.table) {
5926            return Ok(Vec::new());
5927        }
5928        if let Some(raw) = &state.raw {
5929            return self.select_ids_checked(&raw.sql, &raw.params);
5930        }
5931        let fetch = state.set.borrow().fetch().clone();
5932        let snapshot = self.sql_snapshot();
5933        match compile_fetch(state.table, &fetch, &snapshot) {
5934            Ok(q) => self.select_ids_checked(&q.sql, &q.params),
5935            Err(CompileErr::NeedsFold) => {
5936                let (q, cols) = compile_fallback(state.table, &fetch, &snapshot)
5937                    .map_err(|e| DbError::new(DbErrorKind::Unsupported, e.message()))?;
5938                let mut ids = Vec::new();
5939                self.conn().query(&q.sql, &q.params, &mut |row| {
5940                    let values = (0..row.len()).map(|i| row.get(i)).collect::<Vec<_>>();
5941                    if let Some(h) = value_to_handle(&values[0])
5942                        && fetch.pred.eval(
5943                            h,
5944                            &FallbackRow {
5945                                cols: &cols,
5946                                values: &values,
5947                            },
5948                        )
5949                    {
5950                        ids.push(h);
5951                    }
5952                })?;
5953                if let Some(limit) = fetch.limit {
5954                    ids.truncate(limit);
5955                }
5956                Ok(ids)
5957            }
5958            Err(e) => Err(DbError::new(DbErrorKind::Unsupported, e.message())),
5959        }
5960    }
5961
5962    /// A point-in-time view of names for the SQL compiler: tables, keys, shadows, wired
5963    /// relations, and whether the driver folds case.
5964    fn sql_snapshot(&self) -> SqlSnapshot {
5965        let tables = self.inner.tables.borrow();
5966        let relations = self.inner.relations.borrow();
5967        let joins = self.inner.joins.borrow();
5968        SqlSnapshot {
5969            tables: tables
5970                .values()
5971                .map(|h| TableInfo {
5972                    table: h.table.to_string(),
5973                    sql: h.name.sql(),
5974                    key: h.key_cols[0].clone(),
5975                    fts: h.fts.as_ref().map(|_| FtsSql {
5976                        from: h.name.shadow("_fts"),
5977                        match_col: ident(&format!("{}_fts", h.name.bare)),
5978                    }),
5979                    fts_key: h.fts_key,
5980                    spatial: h
5981                        .spatial
5982                        .as_ref()
5983                        .map(|(lat, lon, _)| (lat.clone(), lon.clone(), h.name.shadow("_geo"))),
5984                })
5985                .collect(),
5986            rels: relations
5987                .iter()
5988                .map(|r| RelInfo {
5989                    parent_table: r.parent_table,
5990                    parent_field: r.parent_field,
5991                    parent_key: r.parent_key_col,
5992                    child_table: r.child_table,
5993                    child_key: r.child_key_col,
5994                    fk_field: r.fk_field,
5995                    fk_col: r.fk_col,
5996                })
5997                .collect(),
5998            joins: joins
5999                .iter()
6000                .map(|j| JoinInfo {
6001                    join_table: j.join_table,
6002                    a_table: j.a_table,
6003                    a_field: j.a_field,
6004                    a_key: j.a_key_col,
6005                    b_table: j.b_table,
6006                    b_field: j.b_field.get(),
6007                    b_key: j.b_key_col,
6008                    parent_col: j.parent_col,
6009                    child_col: j.child_col,
6010                })
6011                .collect(),
6012            links: self.inner.links.borrow().clone(),
6013            fold: self.inner.caps.unicode_fold,
6014        }
6015    }
6016}
6017
6018/// One table as the SQL compiler sees it: `table` is the plain name it is asked about,
6019/// `sql` the quoted form it writes; the shadows come already quoted, the columns plain.
6020struct TableInfo {
6021    table: String,
6022    sql: String,
6023    key: String,
6024    fts: Option<FtsSql>,
6025    fts_key: &'static str,
6026    spatial: Option<(String, String, String)>,
6027}
6028
6029struct RelInfo {
6030    parent_table: &'static str,
6031    parent_field: &'static str,
6032    parent_key: &'static str,
6033    child_table: &'static str,
6034    child_key: &'static str,
6035    fk_field: &'static str,
6036    fk_col: &'static str,
6037}
6038
6039struct JoinInfo {
6040    join_table: &'static str,
6041    a_table: &'static str,
6042    a_field: &'static str,
6043    a_key: &'static str,
6044    b_table: &'static str,
6045    b_field: Option<&'static str>,
6046    b_key: &'static str,
6047    parent_col: &'static str,
6048    child_col: &'static str,
6049}
6050
6051struct SqlSnapshot {
6052    tables: Vec<TableInfo>,
6053    rels: Vec<RelInfo>,
6054    joins: Vec<JoinInfo>,
6055    links: Vec<LinkInfo>,
6056    fold: bool,
6057}
6058
6059impl SqlIndex for SqlSnapshot {
6060    fn relation(&self, owner: &str, field: &str) -> Option<RelSql> {
6061        for r in &self.rels {
6062            if r.parent_table == owner && r.parent_field == field {
6063                return Some(RelSql::Children {
6064                    target_key: r.child_key.to_string(),
6065                    fk_col: r.fk_col.to_string(),
6066                    owner_key: r.parent_key.to_string(),
6067                });
6068            }
6069            if r.child_table == owner && r.fk_field == field {
6070                return Some(RelSql::Referent {
6071                    target_key: r.parent_key.to_string(),
6072                    fk_col: r.fk_col.to_string(),
6073                });
6074            }
6075        }
6076        for j in &self.joins {
6077            if j.a_table == owner && j.a_field == field {
6078                return Some(RelSql::Join {
6079                    join_table: j.join_table.to_string(),
6080                    owner_col: j.parent_col.to_string(),
6081                    target_col: j.child_col.to_string(),
6082                    owner_key: j.a_key.to_string(),
6083                    target_key: j.b_key.to_string(),
6084                });
6085            }
6086            if j.b_table == owner && j.b_field == Some(field) {
6087                return Some(RelSql::Join {
6088                    join_table: j.join_table.to_string(),
6089                    owner_col: j.child_col.to_string(),
6090                    target_col: j.parent_col.to_string(),
6091                    owner_key: j.b_key.to_string(),
6092                    target_key: j.a_key.to_string(),
6093                });
6094            }
6095        }
6096        for l in &self.links {
6097            if l.owner_table == owner && l.field == field {
6098                return Some(RelSql::Linked {
6099                    local_col: l.local.to_string(),
6100                    remote_col: l.remote.to_string(),
6101                    target_key: self.key_of(l.target_table)?,
6102                });
6103            }
6104        }
6105        None
6106    }
6107
6108    fn key_of(&self, table: &str) -> Option<String> {
6109        self.tables
6110            .iter()
6111            .find(|t| t.table == table)
6112            .map(|t| t.key.clone())
6113    }
6114
6115    fn table_sql(&self, table: &str) -> String {
6116        self.tables
6117            .iter()
6118            .find(|t| t.table == table)
6119            .map_or_else(|| ident(table), |t| t.sql.clone())
6120    }
6121
6122    fn fts_of(&self, table: &str) -> Option<FtsSql> {
6123        self.tables
6124            .iter()
6125            .find(|t| t.table == table)
6126            .and_then(|t| t.fts.clone())
6127    }
6128
6129    fn fts_key_of(&self, table: &str) -> &'static str {
6130        self.tables
6131            .iter()
6132            .find(|t| t.table == table)
6133            .map_or("rowid", |t| t.fts_key)
6134    }
6135
6136    fn geo_of(&self, table: &str, lat: &str, lon: &str) -> Option<String> {
6137        self.tables
6138            .iter()
6139            .find(|t| t.table == table)
6140            .and_then(|t| t.spatial.as_ref())
6141            .filter(|(a, o, _)| a == lat && o == lon)
6142            .map(|(_, _, g)| g.clone())
6143    }
6144
6145    fn unicode_fold(&self) -> bool {
6146        self.fold
6147    }
6148}
6149
6150/// Whether a statement's target table is `table`, a plain-text check over the SQL the fold
6151/// itself produced (`INSERT INTO "t" … ON CONFLICT`, `UPDATE "t" SET …`, `DELETE FROM "t" …`).
6152/// The fold quotes its table names, so the name is read back through the quotes; `table` is
6153/// the plain name a raw query declared.
6154fn statement_touches(sql: &str, table: &str) -> bool {
6155    for prefix in [
6156        "INSERT INTO ",
6157        "INSERT OR REPLACE INTO ",
6158        "UPDATE ",
6159        "DELETE FROM ",
6160    ] {
6161        if let Some(rest) = sql.strip_prefix(prefix) {
6162            return quoted_name(rest).as_deref() == Some(table);
6163        }
6164    }
6165    false
6166}
6167
6168/// The identifier a quoted name at the start of `sql` spells (a doubled `"` is one quote),
6169/// `None` when `sql` does not start with one.
6170fn quoted_name(sql: &str) -> Option<String> {
6171    let mut chars = sql.strip_prefix('"')?.chars();
6172    let mut name = String::new();
6173    loop {
6174        match chars.next()? {
6175            '"' => match chars.next() {
6176                Some('"') => name.push('"'),
6177                _ => return Some(name),
6178            },
6179            ch => name.push(ch),
6180        }
6181    }
6182}
6183
6184// ---------------------------------------------------------------------------
6185// Query as a list row source (feature `pieces`)
6186// ---------------------------------------------------------------------------
6187
6188#[cfg(feature = "pieces")]
6189mod pieces_glue {
6190    use std::cell::RefCell;
6191
6192    use day_model::{Elem, Keyed, Store};
6193    use day_pieces::{ModelSlot, RowConn, RowSource};
6194
6195    use crate::{Delta, Model, Query, QueryEvents};
6196
6197    /// How far around a bound row the list source faults in one batch: the price of showing
6198    /// row N is one `SELECT` that also covers the rows about to scroll in.
6199    const FAULT_BEHIND: usize = 16;
6200    const FAULT_AHEAD: usize = 64;
6201
6202    /// `list(query, row)`: the query's ids are the display order, rows fault in as the list
6203    /// binds them (a window at a time), and set changes arrive as row deltas the native list
6204    /// can animate.
6205    impl<M: Model> RowSource for Query<M> {
6206        type Slot = ModelSlot<M>;
6207        type Ref = Elem<M>;
6208        type Conn = QueryConn<M>;
6209        fn connect(self) -> QueryConn<M> {
6210            let store = self.container.cache::<M>();
6211            QueryConn {
6212                query: self,
6213                store,
6214                keys: RefCell::new(Vec::new()),
6215            }
6216        }
6217    }
6218
6219    pub struct QueryConn<M: Model> {
6220        query: Query<M>,
6221        store: Store<Keyed<M>>,
6222        keys: RefCell<Vec<u64>>,
6223    }
6224
6225    impl<M: Model> QueryConn<M> {
6226        /// Make the row at `index` (and its neighborhood) resident before anything binds it.
6227        fn fault_window(&self, index: usize) {
6228            let len = self.keys.borrow().len();
6229            let start = index.saturating_sub(FAULT_BEHIND);
6230            let end = (index + FAULT_AHEAD).min(len);
6231            self.query.materialize(start..end);
6232        }
6233    }
6234
6235    impl<M: Model> RowConn for QueryConn<M> {
6236        type Slot = ModelSlot<M>;
6237        type Ref = Elem<M>;
6238
6239        fn refresh(&self) -> Vec<u64> {
6240            let keys: Vec<u64> = self.query.ids().iter().map(|id| id.handle()).collect();
6241            (*self.keys.borrow_mut()).clone_from(&keys);
6242            keys
6243        }
6244        fn len(&self) -> usize {
6245            self.keys.borrow().len()
6246        }
6247        fn token_at(&self, index: usize) -> u64 {
6248            self.keys.borrow().get(index).copied().unwrap_or(0)
6249        }
6250        fn tokens_now(&self) -> Vec<u64> {
6251            self.keys.borrow().clone()
6252        }
6253        fn slot_at(&self, index: usize) -> Option<ModelSlot<M>> {
6254            let key = self.keys.borrow().get(index).copied()?;
6255            if !self.store.with_untracked(|k| k.get(key).is_some()) {
6256                self.fault_window(index);
6257            }
6258            Some(ModelSlot::for_key(self.store, key))
6259        }
6260        fn rebind(&self, slot: &ModelSlot<M>, index: usize) {
6261            if let Some(key) = self.keys.borrow().get(index).copied() {
6262                if !self.store.with_untracked(|k| k.get(key).is_some()) {
6263                    self.fault_window(index);
6264                }
6265                slot.rebind_key(key);
6266            }
6267        }
6268        fn select_ref(&self, index: usize) -> Option<Elem<M>> {
6269            let key = self.keys.borrow().get(index).copied()?;
6270            if !self.store.with_untracked(|k| k.get(key).is_some()) {
6271                self.fault_window(index);
6272            }
6273            Some(self.store.elem(key))
6274        }
6275        fn values_flow_by_reload(&self) -> bool {
6276            false // row values flow through the store's own per-field notifications
6277        }
6278        fn take_row_events(&self) -> Option<Vec<day_spec::props::RowDelta>> {
6279            match self.query.take_events() {
6280                QueryEvents::Deltas(d) => Some(
6281                    d.into_iter()
6282                        .map(|d| match d {
6283                            Delta::Insert(i, _) => day_spec::props::RowDelta::Insert(i),
6284                            Delta::Remove(i, _) => day_spec::props::RowDelta::Remove(i),
6285                            Delta::Move(from, to, _) => day_spec::props::RowDelta::Move(from, to),
6286                        })
6287                        .collect(),
6288                ),
6289                // Reload means "the whole set moved"; None here means the same to the list.
6290                QueryEvents::Reload | QueryEvents::None => None,
6291            }
6292        }
6293        fn commit_move(&self, from: usize, to: usize) {
6294            // The native drag already moved the row: serve that order until the app's write
6295            // (its on_reorder, typically a rank column the query sorts by) requeries. The list
6296            // expects exactly this order back as the echo of its own move; left unrotated, the
6297            // echo never matched, and the query's own move was replayed on rows already moved.
6298            let mut keys = self.keys.borrow_mut();
6299            if from < keys.len() && to < keys.len() {
6300                let key = keys.remove(from);
6301                keys.insert(to, key);
6302            }
6303        }
6304        fn commit_delete(&self, index: usize) {
6305            // The native delete affordance: drop from the local snapshot; the app's on_delete
6306            // writes the store, and the query's own delta follows as the echo.
6307            let mut keys = self.keys.borrow_mut();
6308            if index < keys.len() {
6309                keys.remove(index);
6310            }
6311        }
6312    }
6313}
6314
6315#[cfg(feature = "pieces")]
6316pub use pieces_glue::QueryConn;
6317
6318// ---------------------------------------------------------------------------
6319// Date and time codecs (feature `datetime`; docs/persistence.md)
6320// ---------------------------------------------------------------------------
6321
6322/// Column codecs for `day::datetime`'s [`DayDate`](day_spec::controls::datetime::DayDate) and
6323/// [`DayTime`](day_spec::controls::datetime::DayTime), for `#[derive(Model)]` fields.
6324///
6325/// Canonical, what a bare field of the type stores: `DayDate` → `INTEGER` epoch days,
6326/// `DayTime` → `INTEGER` seconds-of-day: the same numeric interchange forms the pickers
6327/// already speak across native boundaries, and both sort chronologically as plain integers.
6328///
6329/// Named, picked per field with `#[model(with = …)]`: [`Iso8601`](datetime::Iso8601) stores
6330/// `TEXT` in the form `Display` writes (readable in any sqlite tool, and lexicographic order
6331/// is chronological order, ISO-8601's design goal); [`EpochSeconds`](datetime::EpochSeconds)
6332/// and [`EpochMillis`](datetime::EpochMillis) store the `INTEGER` a server that speaks Unix
6333/// time expects, midnight-of-day for a `DayDate`.
6334///
6335/// The impls live here rather than in the piece because one of the two crates has to own
6336/// them, and this is the one that depends on the other.
6337#[cfg(feature = "datetime")]
6338pub mod datetime {
6339    use crate::{ColumnValue, DbError, DbErrorKind, SqlType, Value, ValueCodec};
6340    use day_spec::controls::datetime::{DayDate, DayTime};
6341
6342    impl ColumnValue for DayDate {
6343        const SQL_TYPE: SqlType = SqlType::Integer;
6344        fn to_sqlite_value(&self) -> Value {
6345            Value::Int(self.to_epoch_days())
6346        }
6347        fn from_sqlite_value(v: Value) -> Result<Self, DbError> {
6348            Ok(DayDate::from_epoch_days(v.as_int()?))
6349        }
6350    }
6351
6352    impl ColumnValue for DayTime {
6353        const SQL_TYPE: SqlType = SqlType::Integer;
6354        fn to_sqlite_value(&self) -> Value {
6355            Value::Int(self.seconds_of_day())
6356        }
6357        fn from_sqlite_value(v: Value) -> Result<Self, DbError> {
6358            Ok(DayTime::from_seconds_of_day(v.as_int()?))
6359        }
6360    }
6361
6362    /// `TEXT` in the `Display` form: `YYYY-MM-DD` for a date, `HH:MM[:SS]` for a time.
6363    pub struct Iso8601;
6364
6365    impl ValueCodec<DayDate> for Iso8601 {
6366        const SQL_TYPE: SqlType = SqlType::Text;
6367        fn to_sqlite_value(v: &DayDate) -> Value {
6368            Value::Text(v.to_string())
6369        }
6370        fn from_sqlite_value(v: Value) -> Result<DayDate, DbError> {
6371            let s = v.as_text()?;
6372            DayDate::parse_iso(s).ok_or_else(|| {
6373                DbError::new(DbErrorKind::Decode, format!("not an ISO-8601 date: {s:?}"))
6374            })
6375        }
6376    }
6377
6378    impl ValueCodec<DayTime> for Iso8601 {
6379        const SQL_TYPE: SqlType = SqlType::Text;
6380        fn to_sqlite_value(v: &DayTime) -> Value {
6381            Value::Text(v.to_string())
6382        }
6383        fn from_sqlite_value(v: Value) -> Result<DayTime, DbError> {
6384            let s = v.as_text()?;
6385            DayTime::parse_iso(s).ok_or_else(|| {
6386                DbError::new(DbErrorKind::Decode, format!("not an ISO-8601 time: {s:?}"))
6387            })
6388        }
6389    }
6390
6391    /// `INTEGER` seconds since 1970-01-01T00:00Z; midnight of the day, for a [`DayDate`].
6392    pub struct EpochSeconds;
6393
6394    impl ValueCodec<DayDate> for EpochSeconds {
6395        const SQL_TYPE: SqlType = SqlType::Integer;
6396        fn to_sqlite_value(v: &DayDate) -> Value {
6397            Value::Int(v.to_epoch_days() * 86_400)
6398        }
6399        fn from_sqlite_value(v: Value) -> Result<DayDate, DbError> {
6400            Ok(DayDate::from_epoch_days(v.as_int()?.div_euclid(86_400)))
6401        }
6402    }
6403
6404    /// `INTEGER` milliseconds since the same epoch, for interop that expects it.
6405    pub struct EpochMillis;
6406
6407    impl ValueCodec<DayDate> for EpochMillis {
6408        const SQL_TYPE: SqlType = SqlType::Integer;
6409        fn to_sqlite_value(v: &DayDate) -> Value {
6410            Value::Int(v.to_epoch_days() * 86_400_000)
6411        }
6412        fn from_sqlite_value(v: Value) -> Result<DayDate, DbError> {
6413            Ok(DayDate::from_epoch_days(v.as_int()?.div_euclid(86_400_000)))
6414        }
6415    }
6416}
6417
6418#[cfg(all(test, feature = "driver-rusqlite"))]
6419mod eviction_tests;