Skip to main content

day_persistence/
lib.rs

1// Copyright © The Daybrite Project
2// SPDX-License-Identifier: MPL-2.0
3
4//! SQLite persistence for the observable model (docs/persistence.md).
5//!
6//! An app depends on this crate directly, beside the `day` facade with its `model` feature on,
7//! and starts a model module with `use day_persistence::prelude::*`. The crate is built on the
8//! framework's public API and nothing in the framework depends on it; `#[derive(Model)]` comes
9//! from day-persistence-macros and the web engine from day-sqlite-worker.
10//!
11//! The engine owns the data; memory holds a working set. A [`ModelContainer`] opens a
12//! database through a pluggable [`SqliteDriver`], creates or migrates each model's table,
13//! and stops: no rows are read at open, so opening a million-row store costs the same as
14//! opening an empty one. Rows enter memory by faulting: a typed [`ModelContainer::get`], a
15//! batch [`ModelContainer::ensure_resident`], or a list binding materializing the rows it is
16//! about to show. Each model's [`Store`] is that working set (an ordinary day-model store,
17//! so every binding and accessor works on it unchanged), bounded by a per-table cache limit;
18//! clean rows nothing observes leave silently and fault back on the next read.
19//!
20//! The write half is day-model's change log, folded. Any write requires its row resident
21//! (editing is what faulted it in); at the end of any turn that touched a store (autosave,
22//! the default) the accumulated changes fold into the smallest statement list that expresses
23//! them: twenty keystrokes into one field is one `UPDATE`; a row inserted and then filled is
24//! one `INSERT`; a thousand-child cascade is a handful of chunked `DELETE`s.
25//!
26//! Typed live queries ([`ModelContainer::query`]) compile entirely to SQL: predicate to
27//! `WHERE` (relation crossings as correlated `EXISTS`, full-text as an FTS5 subquery, spatial
28//! boxes through the R*Tree shadow), sort to `ORDER BY` with the key as tie-break, window to
29//! `LIMIT`, so the engine's indexes answer them at any table size. Live maintenance is
30//! dependency-gated: a change to a column no query mentions costs nothing; a change a query
31//! does depend on marks it stale, and one requery after the turn's flush re-derives its id
32//! set, diffed against the previous answer so a list animates the difference instead of
33//! reloading. Rows behind the ids stay lazy: the query holds ids, and the list faults the
34//! window it shows.
35//!
36//! The driver is a trait so the engine is the app's choice: the built-in [`Sqlite`] driver
37//! (feature `driver-rusqlite`, on by default) compiles a bundled SQLite, links the system one
38//! (`system`), or builds SQLCipher (`cipher`); the [`Recorder`] answers from fixtures and
39//! records every statement, which is what keeps persistence assertable headlessly: a test
40//! can check the SQL a UI action produced without a database on disk.
41//!
42//! Another connection's committed writes arrive through [`ModelContainer::check_external`],
43//! which re-reads only the resident rows and re-runs the live queries: O(working set), never
44//! O(table). Row identity is the model's `#[model(id)]` key; display order is a projection
45//! concern and is not persisted.
46
47use std::any::{Any, TypeId};
48use std::cell::{Cell, RefCell};
49use std::collections::HashMap;
50use std::fmt;
51use std::path::{Path, PathBuf};
52use std::rc::Rc;
53
54use day_model::Source;
55use day_model::{Identified, Key, Keyed, ModelId, Op, Store};
56
57#[cfg(not(target_arch = "wasm32"))]
58mod worker;
59#[cfg(not(target_arch = "wasm32"))]
60pub use worker::{
61    DatabaseWorker, WorkerOptions, WorkerRequest, WorkerSnapshot, WorkerSubscription,
62};
63
64mod queries;
65pub use queries::{
66    Col, Delta, Deps, Fetch, FtsRef, GeoRect, GeoRef, Pred, Quant, RelatedDep, RelationCol,
67    ResultSet, RowView, SetChange, Sort, compare_values, encode_column, rank,
68};
69use queries::{
70    CompileErr, FtsSql, RelSql, SqlIndex, compile_count, compile_fallback, compile_fetch,
71};
72
73mod relations;
74pub use relations::{
75    DeleteRule, Many, One, Registrar, RelationDef, RelationRef, wire_join, wire_to_many,
76};
77
78#[cfg(feature = "driver-rusqlite")]
79mod rusqlite_driver;
80#[cfg(feature = "driver-rusqlite")]
81pub use rusqlite_driver::Sqlite;
82#[cfg(all(
83    feature = "driver-rusqlite",
84    target_family = "wasm",
85    target_os = "unknown"
86))]
87pub use rusqlite_driver::WebStorage;
88
89/// The derive for the [`Model`] trait (the two share a name, so `use day_persistence::Model`
90/// brings both): everything `#[derive(Observable)]` generates plus the schema half.
91/// `#[model(id)]` marks the key; docs/persistence.md lists the field and struct options.
92pub use day_persistence_macros::Model;
93
94// `#[derive(Model)]` expands its observable half to this (day-macros' `observable_impls!`),
95// reached by this crate's name so a model crate needs no day-macros dependency of its own.
96#[doc(hidden)]
97pub use day_macros::observable_impls as __observable_impls;
98
99/// What a model module starts from: `use day_persistence::prelude::*;` beside the app's `use
100/// day::prelude::*;`. The container, the `Model` trait and its derive, the relation markers,
101/// the drivers and `schema!`.
102pub mod prelude {
103    #[cfg(feature = "driver-rusqlite")]
104    pub use crate::Sqlite;
105    pub use crate::{DeleteRule, Many, Model, ModelContainer, One, Recorder, Secret, schema};
106}
107
108// ---------------------------------------------------------------------------
109// Errors
110// ---------------------------------------------------------------------------
111
112/// Why a persistence operation failed. `message` is for the log; `kind` is for the code path
113/// that can do something about it.
114#[derive(Clone, Debug)]
115pub struct DbError {
116    pub kind: DbErrorKind,
117    pub message: String,
118}
119
120#[derive(Clone, Copy, PartialEq, Eq, Debug)]
121pub enum DbErrorKind {
122    /// An asynchronous request was rejected because its worker has closed.
123    Closed,
124    /// The bounded worker queue has no room; the request was not accepted.
125    Busy,
126    /// A queued read was cancelled before it began.
127    Cancelled,
128    /// A worker operation panicked. The worker is closed and must be reopened.
129    Panicked,
130    /// The transaction committed, but refreshing resident models failed. Do not replay the
131    /// write. Recover the cache with `rescan` before editing; workers retry before their next job.
132    CommittedRefreshFailed,
133    /// The engine said no: SQL error, I/O, constraint.
134    Driver,
135    /// The stored schema and the declared one disagree in a way lightweight migration cannot
136    /// close (a type change, a rename); the file is refused, never silently rewritten.
137    Schema,
138    /// The database would not open with the key it was given.
139    BadKey,
140    /// A stored value would not decode as the field's type.
141    Decode,
142    /// The driver cannot do this (encryption on a non-cipher build, …).
143    Unsupported,
144    /// A `DeleteRule::Deny` relation refused the delete: children still reference the row.
145    Deny,
146}
147
148impl DbError {
149    pub fn new(kind: DbErrorKind, message: impl Into<String>) -> Self {
150        DbError {
151            kind,
152            message: message.into(),
153        }
154    }
155    /// Engine-error shorthand for driver implementations (public: external drivers use it too).
156    pub fn driver(message: impl Into<String>) -> Self {
157        Self::new(DbErrorKind::Driver, message)
158    }
159}
160
161impl fmt::Display for DbError {
162    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
163        write!(f, "{:?}: {}", self.kind, self.message)
164    }
165}
166
167impl std::error::Error for DbError {}
168
169// ---------------------------------------------------------------------------
170// Values and rows
171// ---------------------------------------------------------------------------
172
173/// SQLite's five storage classes, and nothing else; no driver type reaches the layer above.
174#[derive(Clone, Debug, PartialEq)]
175pub enum Value {
176    Null,
177    Int(i64),
178    Real(f64),
179    Text(String),
180    Blob(Vec<u8>),
181}
182
183impl Value {
184    pub fn as_int(&self) -> Result<i64, DbError> {
185        match self {
186            Value::Int(i) => Ok(*i),
187            other => Err(DbError::new(
188                DbErrorKind::Decode,
189                format!("expected INTEGER, found {other:?}"),
190            )),
191        }
192    }
193    pub fn as_real(&self) -> Result<f64, DbError> {
194        match self {
195            Value::Real(r) => Ok(*r),
196            Value::Int(i) => Ok(*i as f64),
197            other => Err(DbError::new(
198                DbErrorKind::Decode,
199                format!("expected REAL, found {other:?}"),
200            )),
201        }
202    }
203    pub fn as_text(&self) -> Result<&str, DbError> {
204        match self {
205            Value::Text(t) => Ok(t),
206            other => Err(DbError::new(
207                DbErrorKind::Decode,
208                format!("expected TEXT, found {other:?}"),
209            )),
210        }
211    }
212    pub fn as_blob(&self) -> Result<&[u8], DbError> {
213        match self {
214            Value::Blob(b) => Ok(b),
215            other => Err(DbError::new(
216                DbErrorKind::Decode,
217                format!("expected BLOB, found {other:?}"),
218            )),
219        }
220    }
221}
222
223/// A column's declared type. Part of the schema fingerprint, so changing one is a migration.
224#[derive(Clone, Copy, PartialEq, Eq, Debug)]
225pub enum SqlType {
226    Integer,
227    Real,
228    Text,
229    Blob,
230}
231
232impl SqlType {
233    pub fn ddl(self) -> &'static str {
234        match self {
235            SqlType::Integer => "INTEGER",
236            SqlType::Real => "REAL",
237            SqlType::Text => "TEXT",
238            SqlType::Blob => "BLOB",
239        }
240    }
241}
242
243/// One result row, by position.
244pub trait Row {
245    fn len(&self) -> usize;
246    fn get(&self, i: usize) -> Value;
247    fn is_empty(&self) -> bool {
248        self.len() == 0
249    }
250}
251
252impl Row for Vec<Value> {
253    fn len(&self) -> usize {
254        Vec::len(self)
255    }
256    fn get(&self, i: usize) -> Value {
257        self.as_slice().get(i).cloned().unwrap_or(Value::Null)
258    }
259}
260
261// ---------------------------------------------------------------------------
262// The driver interface
263// ---------------------------------------------------------------------------
264
265/// Where the database lives.
266#[derive(Clone, Debug)]
267pub enum Location {
268    Memory,
269    File(PathBuf),
270}
271
272/// A key for an encrypted database. Redacted in Debug and zeroed on drop; where it lives
273/// between launches is the app's business, never this crate's.
274#[derive(Clone)]
275pub struct Secret(String);
276
277impl Secret {
278    pub fn new(key: impl Into<String>) -> Self {
279        Secret(key.into())
280    }
281    #[cfg(feature = "cipher")]
282    pub(crate) fn reveal(&self) -> &str {
283        &self.0
284    }
285}
286
287impl fmt::Debug for Secret {
288    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
289        f.write_str("Secret(…)")
290    }
291}
292
293impl Drop for Secret {
294    fn drop(&mut self) {
295        // Best-effort scrub; the allocation may have been moved by earlier clones.
296        unsafe {
297            // SAFETY: we own the String and overwrite only its initialized bytes in place.
298            for b in self.0.as_bytes_mut() {
299                *b = 0;
300            }
301        }
302    }
303}
304
305/// How to open the database. Built by the driver's own constructors ([`Sqlite::at`] etc.);
306/// public so alternative drivers speak the same vocabulary.
307#[derive(Clone, Debug)]
308pub struct OpenOptions {
309    pub location: Location,
310    /// SQLCipher key; drivers without encryption reject it loudly.
311    pub key: Option<Secret>,
312    /// Journal in WAL mode (files only; ignored in memory). Default true.
313    pub wal: bool,
314    /// Enforce foreign keys. Default true.
315    pub foreign_keys: bool,
316    pub busy_timeout_ms: u32,
317    /// Upgrade a file written by an older SQLCipher generation (PRAGMA cipher_migrate).
318    pub cipher_migrate: bool,
319}
320
321impl OpenOptions {
322    pub fn new(location: Location) -> Self {
323        OpenOptions {
324            location,
325            key: None,
326            wal: true,
327            foreign_keys: true,
328            busy_timeout_ms: 5_000,
329            cipher_migrate: false,
330        }
331    }
332}
333
334/// What a driver can and cannot do, so the container degrades visibly instead of silently.
335#[derive(Clone, Copy, Debug, Default)]
336pub struct Capabilities {
337    /// Survives a relaunch (false for memory stores and the Recorder).
338    pub durable: bool,
339    pub encryption: bool,
340    pub wal: bool,
341    pub full_text_search: bool,
342    pub rtree: bool,
343    /// Another connection's committed writes are detectable (SQLite's `PRAGMA data_version`
344    /// counter), so [`ModelContainer::check_external`] can merge them. The built-in driver
345    /// claims it for file databases on native targets; memory databases (no second connection
346    /// can reach one), the Recorder, and the web engine (its OPFS access is exclusive) do not.
347    pub external_changes: bool,
348    /// The connection has `day_fold` registered (Rust's full-Unicode `to_lowercase` as a
349    /// scalar SQL function), so case-insensitive text predicates compile to exact SQL. The
350    /// built-in native driver registers it at open; without it those predicates take the
351    /// fallback path (SQL for everything else, the folding test re-checked in memory).
352    pub unicode_fold: bool,
353}
354
355/// What `PRAGMA wal_checkpoint(TRUNCATE)` answered ([`ModelContainer::checkpoint`]): the
356/// three columns of its status row. The engine reads the counts after the checkpoint, so a
357/// completed one answers `0` and `0` (the log was emptied); a busy one describes the log it
358/// had to leave behind. Both counts are `-1` when the database is not in WAL mode (a memory
359/// database, or `wal: false`).
360#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
361pub struct Checkpoint {
362    /// A reader or writer kept the checkpoint from completing within the busy timeout; the
363    /// WAL was folded as far as it could be and the rest waits for the next call.
364    pub busy: bool,
365    /// Frames in the WAL when the checkpoint finished.
366    pub log_frames: i64,
367    /// Frames of that log already carried into the main file.
368    pub checkpointed_frames: i64,
369}
370
371impl Checkpoint {
372    /// The whole log was folded and truncated.
373    pub fn is_complete(&self) -> bool {
374        !self.busy
375    }
376}
377
378/// One row of `PRAGMA foreign_key_check` ([`ModelContainer::foreign_key_check`]): a child
379/// row whose reference names no parent.
380#[derive(Clone, Debug, PartialEq, Eq)]
381pub struct ForeignKeyViolation {
382    /// The child table.
383    pub table: String,
384    /// The child row's rowid; `None` for a `WITHOUT ROWID` table.
385    pub rowid: Option<i64>,
386    /// The parent table the reference names.
387    pub parent: String,
388    /// The constraint's index in the child's `PRAGMA foreign_key_list`.
389    pub constraint: i64,
390}
391
392impl fmt::Display for ForeignKeyViolation {
393    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
394        match self.rowid {
395            Some(rowid) => write!(
396                f,
397                "foreign key: {} row {rowid} references missing {}",
398                self.table, self.parent
399            ),
400            None => write!(
401                f,
402                "foreign key: {} row references missing {}",
403                self.table, self.parent
404            ),
405        }
406    }
407}
408
409/// The engine trait. Object-safe on the connection side, so the container stores
410/// `Box<dyn SqliteConnection>` and never names an engine type.
411pub trait SqliteDriver {
412    type Connection: SqliteConnection;
413    fn open(self) -> Result<Self::Connection, DbError>;
414    fn capabilities(&self) -> Capabilities;
415}
416
417impl SqliteConnection for Box<dyn SqliteConnection> {
418    fn execute(&mut self, sql: &str, params: &[Value]) -> Result<u64, DbError> {
419        (**self).execute(sql, params)
420    }
421    fn query(
422        &mut self,
423        sql: &str,
424        params: &[Value],
425        row: &mut dyn FnMut(&dyn Row),
426    ) -> Result<(), DbError> {
427        (**self).query(sql, params, row)
428    }
429    fn execute_batch(&mut self, sql: &str) -> Result<(), DbError> {
430        (**self).execute_batch(sql)
431    }
432    fn query_named(
433        &mut self,
434        sql: &str,
435        params: &[Value],
436        row: &mut dyn FnMut(&[String], &dyn Row),
437    ) -> Result<(), DbError> {
438        (**self).query_named(sql, params, row)
439    }
440}
441
442/// One open connection. Everything above speaks plain SQL and [`Value`]s through it.
443pub trait SqliteConnection: 'static {
444    fn execute(&mut self, sql: &str, params: &[Value]) -> Result<u64, DbError>;
445    fn query(
446        &mut self,
447        sql: &str,
448        params: &[Value],
449        row: &mut dyn FnMut(&dyn Row),
450    ) -> Result<(), DbError>;
451    fn begin(&mut self) -> Result<(), DbError> {
452        self.execute("BEGIN", &[]).map(|_| ())
453    }
454    fn commit(&mut self) -> Result<(), DbError> {
455        self.execute("COMMIT", &[]).map(|_| ())
456    }
457    fn rollback(&mut self) -> Result<(), DbError> {
458        self.execute("ROLLBACK", &[]).map(|_| ())
459    }
460    /// Several statements in one string, no parameters: a DDL script, a migration step
461    /// (day-lite's storage speaks this). The default runs the string as one statement;
462    /// drivers whose engine executes scripts override it (the built-in native driver does).
463    fn execute_batch(&mut self, sql: &str) -> Result<(), DbError> {
464        self.execute(sql, &[]).map(|_| ())
465    }
466    /// [`SqliteConnection::query`], with the result's column names alongside each row, for
467    /// callers that surface rows as named objects (day-lite's JS bridge). A driver that
468    /// cannot name columns refuses rather than guessing.
469    fn query_named(
470        &mut self,
471        sql: &str,
472        params: &[Value],
473        row: &mut dyn FnMut(&[String], &dyn Row),
474    ) -> Result<(), DbError> {
475        let _ = (sql, params, row);
476        Err(DbError::new(
477            DbErrorKind::Unsupported,
478            "this driver does not report column names",
479        ))
480    }
481}
482
483// ---------------------------------------------------------------------------
484// The Recorder: the headless fake
485// ---------------------------------------------------------------------------
486
487/// A driver that answers from fixtures and records every statement: the piece that keeps
488/// persistence assertable with no database on disk. `let (driver, log) = Recorder::new();`
489///
490/// Fixtures answer by table, not by statement: any `SELECT … FROM <table>` serves every
491/// fixture row registered for it, whatever the `WHERE` clause says. Faulting paths keep only
492/// the keys they asked for, so `get` and `ensure_resident` behave; a query against the
493/// Recorder answers with all fixture keys, so assert the SQL it recorded, and use
494/// `Sqlite::memory()` where predicate results themselves are under test.
495pub struct Recorder {
496    state: Rc<RecorderState>,
497}
498
499struct RecorderState {
500    log: RefCell<Vec<(String, Vec<Value>)>>,
501    /// table name → rows served for a `SELECT … FROM <table>`.
502    fixtures: RefCell<HashMap<String, Vec<Vec<Value>>>>,
503}
504
505/// The recorded statement log, shared with the test that holds it.
506#[derive(Clone)]
507pub struct RecorderLog {
508    state: Rc<RecorderState>,
509}
510
511impl RecorderLog {
512    /// Every statement so far, SQL text only.
513    pub fn sql(&self) -> Vec<String> {
514        self.state
515            .log
516            .borrow()
517            .iter()
518            .map(|(s, _)| s.clone())
519            .collect()
520    }
521    /// Statements with their bound parameters.
522    pub fn entries(&self) -> Vec<(String, Vec<Value>)> {
523        self.state.log.borrow().clone()
524    }
525    pub fn clear(&self) {
526        self.state.log.borrow_mut().clear();
527    }
528}
529
530impl Recorder {
531    pub fn new() -> (Recorder, RecorderLog) {
532        let state = Rc::new(RecorderState {
533            log: RefCell::new(Vec::new()),
534            fixtures: RefCell::new(HashMap::new()),
535        });
536        (
537            Recorder {
538                state: state.clone(),
539            },
540            RecorderLog { state },
541        )
542    }
543
544    /// Rows a table's `SELECT` will answer with.
545    pub fn with_table(self, table: &str, rows: Vec<Vec<Value>>) -> Self {
546        self.state.fixtures.borrow_mut().insert(table.into(), rows);
547        self
548    }
549}
550
551pub struct RecorderConn {
552    state: Rc<RecorderState>,
553}
554
555impl SqliteDriver for Recorder {
556    type Connection = RecorderConn;
557    fn open(self) -> Result<RecorderConn, DbError> {
558        Ok(RecorderConn { state: self.state })
559    }
560    fn capabilities(&self) -> Capabilities {
561        Capabilities {
562            durable: false,
563            wal: false,
564            // The fake answers everything: an FTS/spatial model must open against fixtures
565            // (its reads just come back empty unless a fixture answers them), and folded
566            // predicates compile as if day_fold existed; the SQL is recorded, never run.
567            full_text_search: true,
568            rtree: true,
569            unicode_fold: true,
570            ..Default::default()
571        }
572    }
573}
574
575impl SqliteConnection for RecorderConn {
576    fn execute(&mut self, sql: &str, params: &[Value]) -> Result<u64, DbError> {
577        self.state
578            .log
579            .borrow_mut()
580            .push((sql.to_string(), params.to_vec()));
581        Ok(0)
582    }
583    fn query(
584        &mut self,
585        sql: &str,
586        params: &[Value],
587        row: &mut dyn FnMut(&dyn Row),
588    ) -> Result<(), DbError> {
589        self.state
590            .log
591            .borrow_mut()
592            .push((sql.to_string(), params.to_vec()));
593        // Serve a fixture when the statement reads `FROM` a table one was registered for:
594        // quoted, as the container writes it, or plain, as a raw query might.
595        if let Some(from) = sql.split(" FROM ").nth(1) {
596            let table = quoted_name(from)
597                .unwrap_or_else(|| from.split_whitespace().next().unwrap_or("").to_string());
598            if let Some(rows) = self.state.fixtures.borrow().get(&table) {
599                for r in rows {
600                    row(r);
601                }
602            }
603        }
604        Ok(())
605    }
606}
607
608// ---------------------------------------------------------------------------
609// Column values and codecs (docs/persistence.md)
610// ---------------------------------------------------------------------------
611
612/// A Rust type that knows how to be a column. One impl per type: its canonical form.
613pub trait ColumnValue: Sized + 'static {
614    /// The declared column type. Part of the schema fingerprint, so changing it is a
615    /// migration, never a surprise.
616    const SQL_TYPE: SqlType;
617    fn to_sqlite_value(&self) -> Value;
618    fn from_sqlite_value(v: Value) -> Result<Self, DbError>;
619}
620
621/// A named alternative representation for `T`, serde's `#[serde(with = …)]` idiom.
622/// Implement on a unit struct; select it per field with `#[model(with = …)]`.
623pub trait ValueCodec<T>: 'static {
624    const SQL_TYPE: SqlType;
625    fn to_sqlite_value(v: &T) -> Value;
626    fn from_sqlite_value(v: Value) -> Result<T, DbError>;
627}
628
629macro_rules! int_column {
630    ($($t:ty),*) => {$(
631        impl ColumnValue for $t {
632            const SQL_TYPE: SqlType = SqlType::Integer;
633            fn to_sqlite_value(&self) -> Value {
634                Value::Int(*self as i64)
635            }
636            fn from_sqlite_value(v: Value) -> Result<Self, DbError> {
637                Ok(v.as_int()? as $t)
638            }
639        }
640    )*};
641}
642int_column!(i8, i16, i32, i64, u8, u16, u32, u64, usize, isize);
643
644impl ColumnValue for bool {
645    const SQL_TYPE: SqlType = SqlType::Integer;
646    fn to_sqlite_value(&self) -> Value {
647        Value::Int(*self as i64)
648    }
649    fn from_sqlite_value(v: Value) -> Result<Self, DbError> {
650        Ok(v.as_int()? != 0)
651    }
652}
653
654impl ColumnValue for f64 {
655    const SQL_TYPE: SqlType = SqlType::Real;
656    fn to_sqlite_value(&self) -> Value {
657        Value::Real(*self)
658    }
659    fn from_sqlite_value(v: Value) -> Result<Self, DbError> {
660        v.as_real()
661    }
662}
663
664impl ColumnValue for f32 {
665    const SQL_TYPE: SqlType = SqlType::Real;
666    fn to_sqlite_value(&self) -> Value {
667        Value::Real(*self as f64)
668    }
669    fn from_sqlite_value(v: Value) -> Result<Self, DbError> {
670        Ok(v.as_real()? as f32)
671    }
672}
673
674impl ColumnValue for String {
675    const SQL_TYPE: SqlType = SqlType::Text;
676    fn to_sqlite_value(&self) -> Value {
677        Value::Text(self.clone())
678    }
679    fn from_sqlite_value(v: Value) -> Result<Self, DbError> {
680        Ok(v.as_text()?.to_string())
681    }
682}
683
684impl ColumnValue for Vec<u8> {
685    const SQL_TYPE: SqlType = SqlType::Blob;
686    fn to_sqlite_value(&self) -> Value {
687        Value::Blob(self.clone())
688    }
689    fn from_sqlite_value(v: Value) -> Result<Self, DbError> {
690        Ok(v.as_blob()?.to_vec())
691    }
692}
693
694/// A Uuid is a 16-byte `BLOB`: compact, indexable, and readable by any tool that knows the
695/// convention. (`day_model::Uuid` is `uuid::Uuid`, so no second dependency edge exists.)
696impl ColumnValue for day_model::Uuid {
697    const SQL_TYPE: SqlType = SqlType::Blob;
698    fn to_sqlite_value(&self) -> Value {
699        Value::Blob(self.as_bytes().to_vec())
700    }
701    fn from_sqlite_value(v: Value) -> Result<Self, DbError> {
702        day_model::Uuid::from_slice(v.as_blob()?)
703            .map_err(|e| DbError::new(DbErrorKind::Decode, format!("uuid: {e}")))
704    }
705}
706
707/// A key handle, as the bound parameter its stored form takes: `INTEGER` for integer keys,
708/// a 16-byte `BLOB` for Uuid keys, `TEXT` for string keys. `Pair` keys (join rows) bind
709/// through their own two-column clause, never through this.
710fn key_param(handle: u64) -> Value {
711    match Key::of_handle(handle) {
712        Some(Key::U64(k)) => Value::Int(k as i64),
713        Some(Key::Uuid(u)) => Value::Blob(day_model::Uuid::from_u128(u).as_bytes().to_vec()),
714        Some(Key::Str(s)) => Value::Text(s.to_string()),
715        Some(Key::Pair(..)) | None => Value::Null,
716    }
717}
718
719/// A stored key value, back as its path handle. `None` for shapes no key takes (a negative
720/// integer, a blob that is not 16 bytes); the caller treats those rows as unaddressable.
721fn value_to_handle(v: &Value) -> Option<u64> {
722    match v {
723        Value::Int(i) if *i >= 0 => Some(Key::U64(*i as u64).handle()),
724        Value::Blob(b) if b.len() == 16 => day_model::Uuid::from_slice(b)
725            .ok()
726            .map(|u| Key::Uuid(u.as_u128()).handle()),
727        Value::Text(t) => Some(Key::Str(t.as_str().into()).handle()),
728        _ => None,
729    }
730}
731
732/// `NULL` belongs to the framework: `Option` wraps any column type, encoding is called only on
733/// present values, and a NULL decodes to `None`, so an impl never sees `Null` and cannot
734/// disagree about it.
735impl<T: ColumnValue> ColumnValue for Option<T> {
736    const SQL_TYPE: SqlType = T::SQL_TYPE;
737    fn to_sqlite_value(&self) -> Value {
738        match self {
739            Some(v) => v.to_sqlite_value(),
740            None => Value::Null,
741        }
742    }
743    fn from_sqlite_value(v: Value) -> Result<Self, DbError> {
744        match v {
745            Value::Null => Ok(None),
746            v => Ok(Some(T::from_sqlite_value(v)?)),
747        }
748    }
749}
750
751/// The serde codec: any `Serialize + Deserialize` type as a `TEXT` column SQLite's own
752/// `json_*` functions can reach. `#[model(json)]` is sugar for `#[model(with = Json)]`.
753pub struct Json;
754
755impl<T> ValueCodec<T> for Json
756where
757    T: serde::Serialize + serde::de::DeserializeOwned + 'static,
758{
759    const SQL_TYPE: SqlType = SqlType::Text;
760    fn to_sqlite_value(v: &T) -> Value {
761        match serde_json::to_string(v) {
762            Ok(s) => Value::Text(s),
763            // Serialization of an in-memory value failing is a bug in the type's serde impl;
764            // surface it as a NULL rather than a panic, and the decode side will report it.
765            Err(_) => Value::Null,
766        }
767    }
768    fn from_sqlite_value(v: Value) -> Result<T, DbError> {
769        serde_json::from_str(v.as_text()?)
770            .map_err(|e| DbError::new(DbErrorKind::Decode, format!("json: {e}")))
771    }
772}
773
774// ---------------------------------------------------------------------------
775// The Model trait: what #[derive(Model)] implements
776// ---------------------------------------------------------------------------
777
778/// One column of a model's table, as the derive declares it. `field` is the struct field the
779/// column stores: the change log speaks field names, the SQL speaks column names, and this is
780/// where the two meet (they differ only under `#[model(column = "…")]`).
781#[derive(Clone, Copy, Debug)]
782pub struct ColumnDef {
783    pub name: &'static str,
784    pub field: &'static str,
785    pub sql: SqlType,
786    pub not_null: bool,
787    pub unique: bool,
788    pub indexed: bool,
789}
790
791/// The column pair a `#[model(spatial(lat = …, lon = …))]` declaration names.
792#[derive(Clone, Copy, Debug, PartialEq, Eq)]
793pub struct SpatialCols {
794    pub lat: &'static str,
795    pub lon: &'static str,
796}
797
798/// One `#[model(link(…))]` as the derive emits it: `owner.local = target.remote` is the join.
799#[derive(Clone, Copy, Debug)]
800pub struct LinkDef {
801    /// The marker field the link was declared on: the name `Owner::field()` answers to.
802    pub field: &'static str,
803    /// The owner's column holding the value.
804    pub local: &'static str,
805    /// The target's (qualified) table.
806    pub target_table: &'static str,
807    /// The target's column the value names.
808    pub remote: &'static str,
809}
810
811/// The marker a `#[model(link(…))]` field carries. It has neither a column nor state; the
812/// type is what names the far model for the derive.
813pub struct Linked<M: ?Sized> {
814    _m: std::marker::PhantomData<fn() -> M>,
815}
816
817impl<M: ?Sized> Clone for Linked<M> {
818    fn clone(&self) -> Self {
819        *self
820    }
821}
822impl<M: ?Sized> Copy for Linked<M> {}
823impl<M: ?Sized> Default for Linked<M> {
824    fn default() -> Self {
825        Linked {
826            _m: std::marker::PhantomData,
827        }
828    }
829}
830impl<M: ?Sized> PartialEq for Linked<M> {
831    fn eq(&self, _: &Self) -> bool {
832        true
833    }
834}
835impl<M: ?Sized> std::fmt::Debug for Linked<M> {
836    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
837        f.write_str("Linked")
838    }
839}
840
841/// A wired link, as the SQL compiler and the query watcher see it.
842#[derive(Clone, Copy, Debug)]
843pub(crate) struct LinkInfo {
844    pub(crate) owner_table: &'static str,
845    pub(crate) field: &'static str,
846    pub(crate) local: &'static str,
847    pub(crate) target_table: &'static str,
848    pub(crate) remote: &'static str,
849}
850
851/// A persistable model: an [`Identified`] observable struct with a schema half. Implemented by
852/// `#[derive(Model)]`; the container consumes it.
853pub trait Model: Identified + day_model::ApplyField + Clone + 'static {
854    const TABLE: &'static str;
855    /// The key column's name (the `#[model(id)]` field). Also the first entry of `COLUMNS`.
856    const KEY: &'static str;
857    const COLUMNS: &'static [ColumnDef];
858    /// Composite indexes from struct-level `#[model(index("a", "b"))]`.
859    const COMPOSITE_INDEXES: &'static [&'static [&'static str]] = &[];
860    /// Full-text-indexed columns from struct-level `#[model(fts("a", "b"))]`: an
861    /// external-content FTS5 shadow table plus sync triggers, generated at open.
862    const FTS_COLUMNS: &'static [&'static str] = &[];
863    /// The FTS5 tokenizer, from `#[model(fts(…, tokenize = "…"))]`, e.g.
864    /// `"unicode61 remove_diacritics 2"` for diacritics-insensitive search. `None` keeps
865    /// FTS5's default. Part of the schema fingerprint, so changing it rebuilds the shadow.
866    const FTS_TOKENIZE: Option<&'static str> = None;
867    /// The R*Tree pair from struct-level `#[model(spatial(lat = "…", lon = "…"))]`.
868    const SPATIAL: Option<SpatialCols> = None;
869    /// The key column's SQL shape: what a `One<Self>` foreign-key column stores.
870    const KEY_SQL: SqlType = SqlType::Integer;
871    /// Relations declared on this model's `Many` fields (`#[model(relation(…))]`).
872    const RELATIONS: &'static [RelationDef] = &[];
873    /// Relations by value (`#[model(link(…))]`): rows of another model whose `remote` column
874    /// equals this row's `local` column, with no foreign key: the shape that reaches across an
875    /// attached database (see [`ModelContainer::attach_database`]).
876    const LINKS: &'static [LinkDef] = &[];
877    /// Declared `external = "alias"`: the table belongs to a database attached under that alias
878    /// and managed elsewhere. The container creates, migrates, and fingerprints nothing for
879    /// it, and treats it as read-only.
880    const EXTERNAL: bool = false;
881    /// The alias an `EXTERNAL` table's database is attached under, the part `TABLE` is
882    /// qualified with (`catalog` in `catalog.stations`). Kept apart so SQL can quote each
883    /// half; `None` for a table in the container's own file.
884    const SCHEMA: Option<&'static str> = None;
885    /// One [`Value`] per column, in `COLUMNS` order.
886    fn to_row(&self) -> Vec<Value>;
887    fn from_row(row: &dyn Row) -> Result<Self, DbError>;
888    /// Each column's Rust-default value: what an added column backfills with.
889    fn default_row() -> Vec<Value>;
890    /// Wire this model's declared relations. Generated; the default declares none.
891    fn wire(reg: &mut Registrar<'_>) {
892        let _ = reg;
893    }
894}
895
896/// `name` as a SQL identifier: double-quoted, an embedded `"` doubled. Quoted, any name
897/// reaches the engine as itself: a keyword (`order`, `group`), a space, a quote, an emoji.
898/// Every statement the container writes names its tables and columns this way, and raw SQL
899/// written against them should too.
900pub fn ident(name: &str) -> String {
901    let mut out = String::with_capacity(name.len() + 2);
902    out.push('"');
903    for ch in name.chars() {
904        if ch == '"' {
905            out.push('"');
906        }
907        out.push(ch);
908    }
909    out.push('"');
910    out
911}
912
913/// A table's name in the two halves SQL quotes separately: the attached database's alias,
914/// if any, and the table's own name. [`Model::TABLE`] (`notes`, `catalog.stations`) stays the
915/// plain key every lookup, fingerprint and `_day_schema` row uses; this is what a statement
916/// gets.
917#[derive(Clone, Copy)]
918pub(crate) struct TableName {
919    pub(crate) schema: Option<&'static str>,
920    pub(crate) bare: &'static str,
921}
922
923impl TableName {
924    pub(crate) fn of<M: Model>() -> Self {
925        let bare = match M::SCHEMA {
926            Some(schema) => M::TABLE
927                .strip_prefix(schema)
928                .and_then(|rest| rest.strip_prefix('.'))
929                .unwrap_or(M::TABLE),
930            None => M::TABLE,
931        };
932        TableName {
933            schema: M::SCHEMA,
934            bare,
935        }
936    }
937
938    /// A table in the container's own file, named directly (a join table).
939    pub(crate) fn plain(bare: &'static str) -> Self {
940        TableName { schema: None, bare }
941    }
942
943    /// The quoted SQL that names the table in a statement: `"notes"`, `"catalog"."stations"`.
944    pub(crate) fn sql(&self) -> String {
945        table_sql(self.schema, self.bare)
946    }
947
948    /// The quoted SQL naming a shadow beside the table (`_fts`, `_geo`), in the same schema.
949    pub(crate) fn shadow(&self, suffix: &str) -> String {
950        table_sql(self.schema, &format!("{}{suffix}", self.bare))
951    }
952}
953
954pub(crate) fn table_sql(schema: Option<&str>, bare: &str) -> String {
955    match schema {
956        Some(schema) => format!("{}.{}", ident(schema), ident(bare)),
957        None => ident(bare),
958    }
959}
960
961/// The declared schema's fingerprint: table, columns, types, flags, indexes. Equal fingerprints
962/// open instantly; a difference migrates or refuses, never silently.
963pub fn model_fingerprint<M: Model>() -> u64 {
964    let mut h: u64 = 0xcbf2_9ce4_8422_2325;
965    let mut eat = |bytes: &[u8]| {
966        for b in bytes {
967            h ^= *b as u64;
968            h = h.wrapping_mul(0x1000_0000_01b3);
969        }
970    };
971    // The DDL generation this fingerprint vouches for. Bumped when what the file must look
972    // like changes (constrained column additions, unique indexes), so a fingerprint an older
973    // build recorded no longer matches and the file gets one comparison pass, which rebuilds
974    // what the older path left unconstrained and records the new fingerprint.
975    eat(b"ddl-v2|");
976    eat(M::TABLE.as_bytes());
977    eat(M::KEY.as_bytes());
978    for c in M::COLUMNS {
979        eat(c.name.as_bytes());
980        eat(c.sql.ddl().as_bytes());
981        eat(&[c.not_null as u8, c.unique as u8, c.indexed as u8]);
982    }
983    for idx in M::COMPOSITE_INDEXES {
984        for c in *idx {
985            eat(c.as_bytes());
986        }
987        eat(b"|");
988    }
989    for c in M::FTS_COLUMNS {
990        eat(b"fts:");
991        eat(c.as_bytes());
992    }
993    if !M::FTS_COLUMNS.is_empty()
994        && M::COLUMNS
995            .iter()
996            .any(|c| c.name == M::KEY && c.sql != SqlType::Integer)
997    {
998        eat(b"wide-fts-key-map-v1");
999    }
1000    if let Some(t) = M::FTS_TOKENIZE {
1001        eat(b"tok:");
1002        eat(t.as_bytes());
1003    }
1004    if let Some(s) = M::SPATIAL {
1005        eat(b"geo:");
1006        eat(s.lat.as_bytes());
1007        eat(s.lon.as_bytes());
1008    }
1009    h
1010}
1011
1012// ---------------------------------------------------------------------------
1013// Schema sets and migrations
1014// ---------------------------------------------------------------------------
1015
1016/// The set of models a container manages; build with [`schema!`].
1017#[derive(Default)]
1018pub struct Schema {
1019    installers: Vec<Installer>,
1020    /// Run after every table is attached, because relation wiring needs both ends present.
1021    wirers: Vec<Installer>,
1022    /// Foreign-key clauses each declared relation contributes to its target's table.
1023    fk_specs: Vec<FkSpec>,
1024}
1025
1026type Installer = Box<dyn FnOnce(&ModelContainer) -> Result<(), DbError>>;
1027/// Decode raw rows into the cache silently, answering their handles.
1028type AbsorbFn = Rc<dyn Fn(Vec<Vec<Value>>) -> Result<Vec<u64>, DbError>>;
1029/// Diff raw rows (selected by the resident keys) against the cache: per-field announcements
1030/// authored [`ModelContainer::EXTERNAL_AUTHOR`], resident rows missing from the input deleted
1031/// the same way. Returns whether anything differed.
1032type RefreshFn = Rc<dyn Fn(Vec<Vec<Value>>) -> Result<bool, DbError>>;
1033/// A `WHERE` clause plus its parameters, addressing one row by handle.
1034type KeyWhere = Rc<dyn Fn(u64) -> (String, Vec<Value>)>;
1035/// A `WHERE` clause plus its parameters, addressing a set of rows by handle.
1036type KeysWhere = Rc<dyn Fn(&[u64]) -> (String, Vec<Value>)>;
1037/// Evict up to N clean rows from an ordered candidate list, answering how many left.
1038type EvictFn = Rc<dyn Fn(&[u64], usize) -> usize>;
1039
1040/// One foreign-key clause, resolved from a `RelationDef` at `Schema::with` time: the child's
1041/// column carries `REFERENCES parent(key) ON DELETE …` in its generated DDL.
1042#[derive(Clone, Debug)]
1043struct FkSpec {
1044    child_table: &'static str,
1045    child_field: &'static str,
1046    parent_table: &'static str,
1047    /// `parent_table` as the `REFERENCES` clause names it.
1048    parent_sql: String,
1049    /// `parent_table` as `PRAGMA foreign_key_list` reports it: the bare, unquoted name.
1050    parent_bare: &'static str,
1051    parent_key: &'static str,
1052    rule: DeleteRule,
1053}
1054
1055impl Schema {
1056    pub fn new() -> Self {
1057        Self::default()
1058    }
1059    pub fn with<M: Model>(mut self) -> Self {
1060        for def in M::RELATIONS {
1061            if def.join.is_none() {
1062                self.fk_specs.push(FkSpec {
1063                    child_table: def.target_table,
1064                    child_field: def.inverse,
1065                    parent_table: M::TABLE,
1066                    parent_sql: TableName::of::<M>().sql(),
1067                    parent_bare: TableName::of::<M>().bare,
1068                    parent_key: M::KEY,
1069                    rule: def.delete,
1070                });
1071            }
1072        }
1073        self.installers.push(Box::new(|c| c.attach::<M>()));
1074        self.wirers.push(Box::new(|c| {
1075            let mut reg = Registrar {
1076                container: c,
1077                error: None,
1078            };
1079            M::wire(&mut reg);
1080            match reg.error {
1081                Some(e) => Err(e),
1082                None => Ok(()),
1083            }
1084        }));
1085        self
1086    }
1087}
1088
1089/// `schema![Trip, Lodging]`: the models a container manages.
1090#[macro_export]
1091macro_rules! schema {
1092    ($($m:ty),+ $(,)?) => {{
1093        let s = $crate::Schema::new();
1094        $(let s = s.with::<$m>();)+
1095        s
1096    }};
1097}
1098
1099/// Staged migrations: anything with intent behind it (a rename, a split) gets a stage the app
1100/// writes; each stage runs in its own transaction against the raw connection, and versions only
1101/// go up. Lightweight migration (added/dropped columns and indexes) runs afterward, inferred
1102/// from the schema fingerprint.
1103#[derive(Default)]
1104pub struct MigrationPlan {
1105    stages: Vec<Stage>,
1106}
1107
1108type StageFn = Box<dyn FnOnce(&mut dyn SqliteConnection) -> Result<(), DbError>>;
1109
1110struct Stage {
1111    from: u32,
1112    to: u32,
1113    run: StageFn,
1114}
1115
1116impl MigrationPlan {
1117    pub fn new() -> Self {
1118        Self::default()
1119    }
1120    /// A hand-written stage taking the database from `from` to `to`. Stages run in ascending
1121    /// order from the file's current version; a gap is an error at open.
1122    pub fn custom(
1123        mut self,
1124        from: u32,
1125        to: u32,
1126        f: impl FnOnce(&mut dyn SqliteConnection) -> Result<(), DbError> + 'static,
1127    ) -> Self {
1128        self.stages.push(Stage {
1129            from,
1130            to,
1131            run: Box::new(f),
1132        });
1133        self
1134    }
1135    fn target_version(&self) -> u32 {
1136        self.stages.iter().map(|s| s.to).max().unwrap_or(0)
1137    }
1138}
1139
1140// ---------------------------------------------------------------------------
1141// Dirty tracking: the fold, applied live
1142// ---------------------------------------------------------------------------
1143
1144/// What one row needs at the next flush. The merge rules are the change-log fold: same-row
1145/// changes coalesce, an INSERT absorbs later field writes, a DELETE absorbs everything.
1146#[derive(Clone, Debug, PartialEq)]
1147pub(crate) enum DirtyRow {
1148    Insert,
1149    Update(Vec<&'static str>),
1150    Delete,
1151}
1152
1153#[derive(Clone, Default)]
1154struct DirtyState {
1155    /// (store, key) → pending statement kind, in first-touch order.
1156    rows: HashMap<(u64, u64), DirtyRow>,
1157    order: Vec<(u64, u64)>,
1158    /// Stores whose whole value was rewritten (a wholesale `Store::update`), flushed by
1159    /// upserting every resident row. With a lazy cache no "delete the rest" is possible (the
1160    /// rest was never loaded), so wholesale rewrites are a resync of the working set only.
1161    full: Vec<u64>,
1162    /// (store, key) → the distinct author tags of the changes folded into that row, in
1163    /// first-seen order; the unauthored user edit (`None`) leaves no tag.
1164    authors: HashMap<(u64, u64), Vec<&'static str>>,
1165    /// store → the author tags of its wholesale rewrites.
1166    store_authors: HashMap<u64, Vec<&'static str>>,
1167}
1168
1169/// Record one author tag once, in first-seen order; `None` (the user) is not a tag.
1170fn add_author(tags: &mut Vec<&'static str>, author: Option<&'static str>) {
1171    if let Some(author) = author
1172        && !tags.contains(&author)
1173    {
1174        tags.push(author);
1175    }
1176}
1177
1178impl DirtyState {
1179    fn merge(&mut self, newer: Self) {
1180        for store in newer.full {
1181            if !self.full.contains(&store) {
1182                self.full.push(store);
1183            }
1184        }
1185        for (store, tags) in newer.store_authors {
1186            let mine = self.store_authors.entry(store).or_default();
1187            for tag in tags {
1188                add_author(mine, Some(tag));
1189            }
1190        }
1191        for (id, tags) in newer.authors {
1192            let mine = self.authors.entry(id).or_default();
1193            for tag in tags {
1194                add_author(mine, Some(tag));
1195            }
1196        }
1197        for id in newer.order {
1198            let next = newer.rows[&id].clone();
1199            if !self.rows.contains_key(&id) {
1200                self.order.push(id);
1201            }
1202            match (self.rows.get_mut(&id), next) {
1203                (Some(DirtyRow::Update(old)), DirtyRow::Update(cols)) => {
1204                    for col in cols {
1205                        if !old.contains(&col) {
1206                            old.push(col);
1207                        }
1208                    }
1209                }
1210                (Some(DirtyRow::Insert | DirtyRow::Delete), DirtyRow::Update(_)) => {}
1211                (_, next) => {
1212                    self.rows.insert(id, next);
1213                }
1214            }
1215        }
1216    }
1217
1218    fn is_empty(&self) -> bool {
1219        self.rows.is_empty() && self.full.is_empty()
1220    }
1221
1222    fn note(&mut self, change: &day_model::Change, known_store: impl Fn(u64) -> bool) {
1223        let Some(&store) = change.components.first() else {
1224            return;
1225        };
1226        if !known_store(store) {
1227            return;
1228        }
1229        let Some(&key) = change.components.get(1) else {
1230            // A store-level change: the whole value may have been replaced.
1231            if !self.full.contains(&store) {
1232                self.full.push(store);
1233            }
1234            add_author(self.store_authors.entry(store).or_default(), change.author);
1235            return;
1236        };
1237        if key == day_model::STRUCTURE {
1238            return; // the shape path is for the UI; the row paths carry the work
1239        }
1240        let id = (store, key);
1241        if !self.rows.contains_key(&id) {
1242            self.order.push(id);
1243        }
1244        add_author(self.authors.entry(id).or_default(), change.author);
1245        let slot = self.rows.entry(id).or_insert(DirtyRow::Update(Vec::new()));
1246        match (change.op, &mut *slot) {
1247            (Op::Delete, _) => *slot = DirtyRow::Delete,
1248            (Op::Insert, _) => *slot = DirtyRow::Insert,
1249            (Op::Set, DirtyRow::Insert) | (Op::Set, DirtyRow::Delete) => {}
1250            (Op::Set, DirtyRow::Update(columns)) => {
1251                if change.components.len() >= 3 && !columns.contains(&change.label) {
1252                    columns.push(change.label);
1253                }
1254            }
1255            (Op::Move, _) => {}
1256        }
1257    }
1258}
1259
1260// ---------------------------------------------------------------------------
1261// The container
1262// ---------------------------------------------------------------------------
1263
1264/// Everything the container knows how to do for one attached model type, monomorphized at
1265/// [`Schema::with`] and stored type-erased. Join tables (relations.rs) hand-build one over
1266/// their pair-keyed membership cache, which is why keys are plural and clause-shaped here.
1267pub(crate) struct TableHooks {
1268    pub(crate) table: &'static str,
1269    /// `table` split for SQL: the statements below name it through `name.sql()`.
1270    pub(crate) name: TableName,
1271    /// The key column(s), in order: the key SELECT list, the upsert's conflict target, and
1272    /// what its DO UPDATE leaves alone. One entry everywhere but join tables. Plain names,
1273    /// like `columns`; [`ident`] quotes them where a statement is written.
1274    pub(crate) key_cols: Vec<String>,
1275    /// `WHERE` clause + params addressing one row by its key handle.
1276    pub(crate) key_where: KeyWhere,
1277    pub(crate) columns: Vec<String>,
1278    /// `columns` again, as the model's static names: what a [`RowChange`] reports.
1279    pub(crate) column_names: Vec<&'static str>,
1280    /// Same order as `columns`; what a change's label matches against.
1281    pub(crate) fields: Vec<String>,
1282    /// The FTS5 shadow's table name, when the model declares `fts(…)`.
1283    pub(crate) fts: Option<String>,
1284    pub(crate) fts_key: &'static str,
1285    /// The R*Tree shadow: (lat column, lon column, shadow table name).
1286    pub(crate) spatial: Option<(String, String, String)>,
1287    /// Current row values by key, read from the cache at flush time; the change log carries
1288    /// which rows and columns moved, never their contents.
1289    pub(crate) row_for: Rc<dyn Fn(u64) -> Option<Vec<Value>>>,
1290    /// Every (key, row) currently resident, for wholesale resyncs.
1291    pub(crate) all_rows: Rc<dyn Fn() -> Vec<(u64, Vec<Value>)>>,
1292    /// The resident keys, in cache (≈ fault) order: what eviction walks.
1293    pub(crate) resident_keys: Rc<dyn Fn() -> Vec<u64>>,
1294    /// The resident count, O(1): the fault path's eviction gate.
1295    pub(crate) resident_len: Rc<dyn Fn() -> usize>,
1296    pub(crate) is_resident: Rc<dyn Fn(u64) -> bool>,
1297    /// Decode raw rows into the cache silently (a fault landing).
1298    pub(crate) absorb: AbsorbFn,
1299    /// Diff raw rows against the resident set, announcing as the external author.
1300    pub(crate) refresh: RefreshFn,
1301    /// Silently drop up to `want` clean rows from the cache, taking candidates in order and
1302    /// skipping anything observed; answers how many left. One retain over the store, so an
1303    /// eviction pass costs O(cache), not O(cache × evicted). (Dirtiness is the container's
1304    /// knowledge, checked before this is called.)
1305    pub(crate) evict: EvictFn,
1306    #[cfg(not(target_arch = "wasm32"))]
1307    pub(crate) dispose_cache: Rc<dyn Fn()>,
1308    /// Bring this store under an undo history; captured here because the model type is known
1309    /// only at attach time.
1310    pub(crate) watch_undo: Rc<dyn Fn(&day_model::UndoStack)>,
1311}
1312
1313/// Why a durable transaction is being performed. Cache refreshes use EXTERNAL_AUTHOR and
1314/// do not constitute a write transaction. Remote imports persist but must not echo outbound.
1315#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
1316pub enum WriteOrigin {
1317    #[default]
1318    Local,
1319    RemoteImport,
1320    Maintenance,
1321}
1322
1323/// One row of a transaction's folded change set, as [`TransactionParticipant::before_commit`]
1324/// receives it: every row the ORM's pipeline wrote in the transaction (model edits, relation
1325/// upkeep, in-process cascades), one entry per (table, key). Rows the engine's own `ON
1326/// DELETE` clause removed without the process faulting them are not in the set.
1327#[derive(Clone, Debug, PartialEq)]
1328pub struct RowChange {
1329    /// The model's table, as [`Model::TABLE`] names it (a join table by its generated name).
1330    pub table: &'static str,
1331    /// The key as stored: one value, or the (parent, child) pair of a join row.
1332    pub key: Vec<Value>,
1333    pub kind: RowChangeKind,
1334    /// The distinct author tags ([`day_model::with_author`]) of the changes folded into
1335    /// the row, first seen first. The user's own edits carry no tag; the container's
1336    /// external-merge author never reaches the set.
1337    pub authors: Vec<&'static str>,
1338}
1339
1340/// What happened to a [`RowChange`]'s row over the transaction. Same-row changes coalesce
1341/// the way the autosave fold does: an insert absorbs later updates, a delete absorbs
1342/// everything before it, updates union their columns, and a wholesale write supersedes a
1343/// column list.
1344#[derive(Clone, Debug, PartialEq, Eq)]
1345pub enum RowChangeKind {
1346    Inserted,
1347    /// The changed columns' names, in first-touch order; transient fields never appear.
1348    Updated(Vec<&'static str>),
1349    /// The key only; the removed row's values are not retained.
1350    Deleted,
1351    /// Every column was written: a row-level replacement, or a row of a store rewritten
1352    /// wholesale (`Store::update`), which resyncs the resident rows.
1353    Rewritten,
1354}
1355
1356impl RowChange {
1357    /// Fold a later change to the same row into this one, by the autosave fold's rules.
1358    fn absorb(&mut self, later: RowChange) {
1359        for author in later.authors {
1360            add_author(&mut self.authors, Some(author));
1361        }
1362        self.kind = match (
1363            std::mem::replace(&mut self.kind, RowChangeKind::Deleted),
1364            later.kind,
1365        ) {
1366            (RowChangeKind::Updated(mut cols), RowChangeKind::Updated(more)) => {
1367                for col in more {
1368                    if !cols.contains(&col) {
1369                        cols.push(col);
1370                    }
1371                }
1372                RowChangeKind::Updated(cols)
1373            }
1374            (RowChangeKind::Updated(_), RowChangeKind::Rewritten)
1375            | (RowChangeKind::Rewritten, RowChangeKind::Updated(_) | RowChangeKind::Rewritten) => {
1376                RowChangeKind::Rewritten
1377            }
1378            (
1379                kept @ (RowChangeKind::Inserted | RowChangeKind::Deleted),
1380                RowChangeKind::Updated(_) | RowChangeKind::Rewritten,
1381            ) => kept,
1382            (_, next @ (RowChangeKind::Inserted | RowChangeKind::Deleted)) => next,
1383        };
1384    }
1385}
1386
1387/// The cache refresh a participant asks for after its own SQL may have changed model rows
1388/// that were already flushed from the caches.
1389#[derive(Clone, Debug, PartialEq, Eq, Default)]
1390pub enum Refresh {
1391    /// The participant wrote nothing the caches hold.
1392    #[default]
1393    None,
1394    /// Re-read and diff the resident rows of these tables only, drop the relation memos
1395    /// that involve them, and re-run the live queries that read them.
1396    Tables(Vec<&'static str>),
1397    /// Every resident row, every relation memo, every live query.
1398    All,
1399}
1400
1401/// Queue-confined extension participating in the SAME SQL transaction as model changes.
1402/// Callbacks must use only the supplied connection, never re-enter the container, commit,
1403/// roll back, perform network I/O, or mutate reactive models. Returning an error aborts the
1404/// transaction. `before_commit` runs after worker reactions and intermediate saves settle.
1405/// Persist extension state transactionally; callbacks may run again following a rollback.
1406pub trait TransactionParticipant: 'static {
1407    fn begin(
1408        &self,
1409        connection: &mut dyn SqliteConnection,
1410        origin: WriteOrigin,
1411    ) -> Result<(), DbError>;
1412    /// The transaction's folded change set is `changes`: accumulated across every flush in
1413    /// the transaction (a worker job's intermediate saves included), empty again after a
1414    /// rollback. The answer names the cache refresh to run after a successful COMMIT, before
1415    /// the work is acknowledged; a refresh failure surfaces as
1416    /// [`DbErrorKind::CommittedRefreshFailed`] (the data is durable, it cannot roll back).
1417    fn before_commit(
1418        &self,
1419        connection: &mut dyn SqliteConnection,
1420        origin: WriteOrigin,
1421        changes: &[RowChange],
1422    ) -> Result<Refresh, DbError>;
1423}
1424
1425pub(crate) struct ContainerInner {
1426    conn: RefCell<Box<dyn SqliteConnection>>,
1427    caps: Capabilities,
1428    /// store root id → hooks.
1429    tables: RefCell<HashMap<u64, TableHooks>>,
1430    /// model TypeId → the `Store<Keyed<M>>` handle, boxed.
1431    stores: RefCell<HashMap<TypeId, Box<dyn Any>>>,
1432    dirty: RefCell<DirtyState>,
1433    /// Live query result sets, marked stale from the change sink. Weak: the app's `Query`
1434    /// handles own the state; dead entries are pruned on dispatch.
1435    queries: RefCell<Vec<std::rc::Weak<QueryState>>>,
1436    /// The engine's cross-connection change counter as of the last look (`PRAGMA
1437    /// data_version`; it moves only when another connection commits to the file). `None`
1438    /// where the driver reports no external-change detection.
1439    data_version: Cell<Option<i64>>,
1440    /// Foreign-key clauses relations contribute to their targets' tables, set before attach.
1441    fk_specs: RefCell<Vec<FkSpec>>,
1442    /// The wired relations, maintained from the change sink and read by `RelationRef`s.
1443    pub(crate) relations: RefCell<Vec<Rc<relations::ToOneRel>>>,
1444    /// The wired many-to-manys, each over its own membership cache.
1445    pub(crate) joins: RefCell<Vec<Rc<relations::JoinRel>>>,
1446    /// The declared value links (`#[model(link(…))]`), from every attached model.
1447    pub(crate) links: RefCell<Vec<LinkInfo>>,
1448    /// The databases `attach_database` holds open, by alias.
1449    attached: RefCell<Vec<String>>,
1450    sink: Cell<Option<day_model::ChangeSinkId>>,
1451    autosave: Cell<bool>,
1452    /// An outer worker transaction owns commit/rollback; inner model saves only execute SQL.
1453    worker_transaction: Cell<bool>,
1454    participant: RefCell<Option<Rc<dyn TransactionParticipant>>>,
1455    /// The open transaction's folded change set, kept only while a participant is installed:
1456    /// appended per flush, handed to `before_commit`, cleared at begin, commit and rollback.
1457    transaction_changes: RefCell<Vec<RowChange>>,
1458    /// What `before_commit` asked for, acted on once the COMMIT has succeeded.
1459    refresh_scope: RefCell<Refresh>,
1460    refresh_pending: Cell<bool>,
1461    /// Soft per-table bound on resident rows. Dirty and observed rows never evict, so the
1462    /// working set can exceed it; everything else does not.
1463    cache_limit: Cell<usize>,
1464    /// Retry eviction after growth when dirty/observed rows prevent reaching the limit.
1465    eviction_retry: RefCell<HashMap<u64, usize>>,
1466    /// The last autosave failure, observable by the UI (`when(container.last_error()…)`).
1467    pub(crate) error: day_reactive::Signal<Option<String>>,
1468    /// Guards re-entrant flushes (a turn-end firing during an explicit save).
1469    flushing: Cell<bool>,
1470}
1471
1472/// An open database and the working-set caches over it. Clone is shallow: clones share the
1473/// connection and the dirty state.
1474#[derive(Clone)]
1475pub struct ModelContainer {
1476    pub(crate) inner: Rc<ContainerInner>,
1477}
1478
1479/// The default per-table bound on resident rows ([`ModelContainer::set_cache_limit`]).
1480pub const DEFAULT_CACHE_LIMIT: usize = 8_192;
1481
1482impl ModelContainer {
1483    /// Open through `driver`, migrate, and attach every model in `schema`. Attaching creates
1484    /// or migrates the table and nothing more: no rows load, so open cost does not grow with
1485    /// the file. Autosave is on: any turn that touched a store flushes at its end.
1486    pub fn open<D: SqliteDriver>(driver: D, schema: Schema) -> Result<ModelContainer, DbError>
1487    where
1488        D::Connection: 'static,
1489    {
1490        Self::open_with(driver, schema, MigrationPlan::new())
1491    }
1492
1493    /// [`ModelContainer::open`], with hand-written migration stages that run first.
1494    pub fn open_with<D: SqliteDriver>(
1495        driver: D,
1496        schema: Schema,
1497        plan: MigrationPlan,
1498    ) -> Result<ModelContainer, DbError>
1499    where
1500        D::Connection: 'static,
1501    {
1502        let caps = driver.capabilities();
1503        let conn: Box<dyn SqliteConnection> = Box::new(driver.open()?);
1504        let error = day_reactive::Scope::detached().enter(|| day_reactive::Signal::new(None));
1505        let container = ModelContainer {
1506            inner: Rc::new(ContainerInner {
1507                conn: RefCell::new(conn),
1508                caps,
1509                tables: RefCell::new(HashMap::new()),
1510                stores: RefCell::new(HashMap::new()),
1511                dirty: RefCell::new(DirtyState::default()),
1512                queries: RefCell::new(Vec::new()),
1513                data_version: Cell::new(None),
1514                fk_specs: RefCell::new(Vec::new()),
1515                relations: RefCell::new(Vec::new()),
1516                joins: RefCell::new(Vec::new()),
1517                links: RefCell::new(Vec::new()),
1518                attached: RefCell::new(Vec::new()),
1519                sink: Cell::new(None),
1520                autosave: Cell::new(true),
1521                worker_transaction: Cell::new(false),
1522                refresh_pending: Cell::new(false),
1523                participant: RefCell::new(None),
1524                transaction_changes: RefCell::new(Vec::new()),
1525                refresh_scope: RefCell::new(Refresh::None),
1526                cache_limit: Cell::new(DEFAULT_CACHE_LIMIT),
1527                eviction_retry: RefCell::new(HashMap::new()),
1528                error,
1529                flushing: Cell::new(false),
1530            }),
1531        };
1532
1533        container.ensure_schema_table()?;
1534        container.run_stages(plan)?;
1535        let Schema {
1536            installers,
1537            wirers,
1538            fk_specs,
1539        } = schema;
1540        *container.inner.fk_specs.borrow_mut() = fk_specs;
1541        for install in installers {
1542            install(&container)?;
1543        }
1544        // Relations wire once every table is attached (both ends exist), and only then does
1545        // the sink go live.
1546        for wire in wirers {
1547            wire(&container)?;
1548        }
1549        relations::register_container(&container.inner);
1550        container.install_sink();
1551        container.install_autosave();
1552        if caps.external_changes {
1553            container
1554                .inner
1555                .data_version
1556                .set(container.file_data_version()?);
1557        }
1558        Ok(container)
1559    }
1560
1561    /// The author tag on changes [`ModelContainer::check_external`] merges in: the database's
1562    /// own contents arriving, distinguishable by every change consumer and never persisted
1563    /// back. Reserved: an app writing stores under this tag would have its writes dropped by
1564    /// the autosave fold.
1565    pub const EXTERNAL_AUTHOR: &'static str = "database";
1566
1567    /// The working-set cache for `M`, an ordinary day-model store holding the rows currently
1568    /// resident and not the table: its `keys()` are whatever happens to be faulted in. Bindings
1569    /// and accessors work on it unchanged; enumerate rows through a [`ModelContainer::query`],
1570    /// never through the cache. Panics only if `M` was not in the container's `schema!`,
1571    /// which is a wiring bug worth stopping on.
1572    pub fn cache<M: Model>(&self) -> Store<Keyed<M>> {
1573        let stores = self.inner.stores.borrow();
1574        let any = stores
1575            .get(&TypeId::of::<M>())
1576            .unwrap_or_else(|| panic!("model `{}` is not in this container's schema!", M::TABLE));
1577        *any.downcast_ref::<Store<Keyed<M>>>()
1578            .expect("store map holds the exact type it was keyed by")
1579    }
1580
1581    /// What the driver can and cannot do.
1582    pub fn capabilities(&self) -> Capabilities {
1583        self.inner.caps
1584    }
1585
1586    /// Autosave on/off (default on). Off, changes accumulate until [`ModelContainer::save`],
1587    /// and live queries answer from the last save, since only the file can answer them.
1588    pub fn set_autosave(&self, on: bool) {
1589        self.inner.autosave.set(on);
1590    }
1591
1592    /// The soft per-table bound on resident rows (default [`DEFAULT_CACHE_LIMIT`]). Dirty
1593    /// rows and rows something observes never evict; beyond that, faulting past the limit
1594    /// releases the oldest resident rows. `usize::MAX` disables eviction. Lowering the limit
1595    /// enforces it immediately.
1596    pub fn set_cache_limit(&self, rows: usize) {
1597        self.inner.cache_limit.set(rows.max(1));
1598        let store_ids: Vec<u64> = self.inner.tables.borrow().keys().copied().collect();
1599        for id in store_ids {
1600            self.enforce_cache_limit(id, &[], true);
1601        }
1602    }
1603
1604    /// Install one transaction extension before exposing the container to writers. Registration
1605    /// does not initialize its schema; do that in the worker factory first. No replacement or
1606    /// removal is supported, preventing a sync-enabled connection silently losing journaling.
1607    pub fn install_transaction_participant(
1608        &self,
1609        participant: impl TransactionParticipant,
1610    ) -> Result<(), DbError> {
1611        if self.inner.worker_transaction.get()
1612            || !self.inner.dirty.borrow().is_empty()
1613            || self.inner.participant.borrow().is_some()
1614        {
1615            return Err(DbError::new(
1616                DbErrorKind::Deny,
1617                "transaction participant must be installed on an idle, clean container",
1618            ));
1619        }
1620        *self.inner.participant.borrow_mut() = Some(Rc::new(participant));
1621        Ok(())
1622    }
1623
1624    fn participant_begin(
1625        &self,
1626        conn: &mut dyn SqliteConnection,
1627        origin: WriteOrigin,
1628    ) -> Result<(), DbError> {
1629        let participant = self.inner.participant.borrow().clone();
1630        if let Some(p) = participant {
1631            self.inner.transaction_changes.borrow_mut().clear();
1632            *self.inner.refresh_scope.borrow_mut() = Refresh::None;
1633            p.begin(conn, origin)?;
1634        }
1635        Ok(())
1636    }
1637
1638    /// Hand the participant the transaction's change set and keep its refresh request for
1639    /// [`ModelContainer::participant_refresh`], which runs once the COMMIT has succeeded.
1640    fn participant_commit(
1641        &self,
1642        conn: &mut dyn SqliteConnection,
1643        origin: WriteOrigin,
1644    ) -> Result<(), DbError> {
1645        let participant = self.inner.participant.borrow().clone();
1646        if let Some(p) = participant {
1647            let changes = std::mem::take(&mut *self.inner.transaction_changes.borrow_mut());
1648            let scope = p.before_commit(conn, origin, &changes)?;
1649            *self.inner.refresh_scope.borrow_mut() = scope;
1650        }
1651        Ok(())
1652    }
1653
1654    /// A transaction is ending without a COMMIT: its change set is not anyone's to see.
1655    fn participant_rollback(&self) {
1656        self.inner.transaction_changes.borrow_mut().clear();
1657        *self.inner.refresh_scope.borrow_mut() = Refresh::None;
1658    }
1659
1660    /// Append one flush's folded rows to the open transaction's change set, coalescing with
1661    /// the rows earlier flushes of the same transaction already recorded.
1662    fn record_changes(&self, changes: Vec<RowChange>) {
1663        if changes.is_empty() {
1664            return;
1665        }
1666        let mut set = self.inner.transaction_changes.borrow_mut();
1667        for change in changes {
1668            match set
1669                .iter_mut()
1670                .find(|c| c.table == change.table && c.key == change.key)
1671            {
1672                None => set.push(change),
1673                Some(prior) => prior.absorb(change),
1674            }
1675        }
1676    }
1677
1678    /// The last autosave failure, if any, as a tracked signal the UI can watch.
1679    pub fn last_error(&self) -> day_reactive::Signal<Option<String>> {
1680        self.inner.error
1681    }
1682
1683    /// Flush every pending change now, in one transaction. Autosave calls this at turn end;
1684    /// call it directly where the error matters at a known point.
1685    pub fn save(&self) -> Result<(), DbError> {
1686        self.save_now().map(|_| ())
1687    }
1688
1689    /// Run `f` and return the SQL one flush of everything it changed issues: the headless
1690    /// persistence assert (autosave is suspended for the duration so the statements land
1691    /// here). Twenty keystrokes into one field come back as one `UPDATE`.
1692    pub fn record_sql(&self, f: impl FnOnce()) -> Result<Vec<String>, DbError> {
1693        let prior = self.inner.autosave.replace(false);
1694        f();
1695        let result = self.save_now();
1696        self.inner.autosave.set(prior);
1697        result.map(|stmts| stmts.into_iter().map(|(sql, _)| sql).collect())
1698    }
1699
1700    fn save_now(&self) -> Result<Vec<(String, Vec<Value>)>, DbError> {
1701        if self.inner.flushing.get() {
1702            return Ok(Vec::new());
1703        }
1704        if self.inner.refresh_pending.get() {
1705            self.refresh_committed()?;
1706        }
1707        let dirty = {
1708            let mut d = self.inner.dirty.borrow_mut();
1709            if d.is_empty() {
1710                drop(d);
1711                // Nothing to flush, but a stale query may still be waiting on its deferred
1712                // requery (a fetch swap on a clean container lands here).
1713                self.run_deferred_requeries(&[]);
1714                return Ok(Vec::new());
1715            }
1716            std::mem::take(&mut *d)
1717        };
1718        self.inner.flushing.set(true);
1719        let had_writes = !dirty.is_empty();
1720        let result = self.flush(&dirty);
1721        self.inner.flushing.set(false);
1722        match &result {
1723            Ok(stmts) => {
1724                // Previously dirty rows can now be evicted on the next fault.
1725                self.inner.eviction_retry.borrow_mut().clear();
1726                self.inner.error.set_if_changed(None);
1727                self.run_deferred_requeries(stmts);
1728            }
1729            Err(e) => {
1730                // Keep the optimistic cache and its pending writes together. A retry must
1731                // persist the same edits, including any edits made after the failed save.
1732                {
1733                    let mut pending = self.inner.dirty.borrow_mut();
1734                    let newer = std::mem::take(&mut *pending);
1735                    *pending = dirty;
1736                    pending.merge(newer);
1737                }
1738                self.inner.error.set(Some(e.to_string()));
1739            }
1740        }
1741        if result.is_ok() && !self.inner.worker_transaction.get() && had_writes {
1742            self.participant_refresh()?;
1743        }
1744        result
1745    }
1746
1747    /// After a successful COMMIT: the refresh `before_commit` asked for. A failure here is
1748    /// [`DbErrorKind::CommittedRefreshFailed`]; the refresh stays pending (as a full one) for
1749    /// the next save to retry, since the file already holds the participant's writes.
1750    fn participant_refresh(&self) -> Result<(), DbError> {
1751        let scope = std::mem::take(&mut *self.inner.refresh_scope.borrow_mut());
1752        let tables = match scope {
1753            Refresh::None => return Ok(()),
1754            Refresh::Tables(tables) => Some(tables),
1755            Refresh::All => None,
1756        };
1757        self.inner.refresh_pending.set(true);
1758        self.refresh_committed_in(tables.as_deref()).map_err(|e| {
1759            let error = DbError::new(DbErrorKind::CommittedRefreshFailed, e.to_string());
1760            self.inner.error.set(Some(error.to_string()));
1761            error
1762        })
1763    }
1764
1765    fn refresh_committed(&self) -> Result<(), DbError> {
1766        self.refresh_committed_in(None)
1767    }
1768
1769    /// Re-read the resident rows of `tables` (every table when `None`) and bring the memos
1770    /// and live queries that depend on them up to date.
1771    fn refresh_committed_in(&self, tables: Option<&[&'static str]>) -> Result<(), DbError> {
1772        if !self.inner.dirty.borrow().is_empty() {
1773            return Err(DbError::new(
1774                DbErrorKind::Deny,
1775                "refresh committed state before making further model edits",
1776            ));
1777        }
1778        self.refresh_resident_in(tables)?;
1779        match tables {
1780            None => {
1781                self.invalidate_relation_memos();
1782                self.requery_all();
1783            }
1784            Some(tables) => {
1785                self.invalidate_relation_memos_of(tables);
1786                self.requery_tables(tables);
1787            }
1788        }
1789        if self.inner.caps.external_changes {
1790            self.inner.data_version.set(self.file_data_version()?);
1791        }
1792        self.inner.refresh_pending.set(false);
1793        self.inner.error.set_if_changed(None);
1794        Ok(())
1795    }
1796
1797    // --- faulting ----------------------------------------------------------------------------
1798
1799    /// One row, made resident (faulted from the file if it was not). `None` when no such row
1800    /// exists (or a driver error surfaced, observable through [`ModelContainer::last_error`]).
1801    pub fn get<M: Model>(&self, id: impl Into<ModelId<M>>) -> Option<day_model::Elem<M>> {
1802        match self.try_get::<M>(id) {
1803            Ok(row) => row,
1804            Err(error) => {
1805                self.inner.error.set(Some(error.to_string()));
1806                None
1807            }
1808        }
1809    }
1810
1811    /// Read a row, distinguishing an absent key from a storage or decoding failure.
1812    pub fn try_get<M: Model>(
1813        &self,
1814        id: impl Into<ModelId<M>>,
1815    ) -> Result<Option<day_model::Elem<M>>, DbError> {
1816        let store = self.cache::<M>();
1817        let h = id.into().handle();
1818        if !store.with_untracked(|k| k.get(h).is_some()) {
1819            self.ensure_resident::<M>(&[h])?;
1820        }
1821        Ok(store
1822            .with_untracked(|k| k.get(h).is_some())
1823            .then(|| store.elem(h)))
1824    }
1825
1826    /// Make these rows resident, faulting the missing ones in one chunked `SELECT`. Rows the
1827    /// file does not have are not resident afterwards; rows deleted this turn are not
1828    /// resurrected.
1829    pub fn ensure_resident<M: Model>(&self, keys: &[u64]) -> Result<(), DbError> {
1830        let store = self.cache::<M>();
1831        self.fault_into(store.store_id(), keys)
1832    }
1833
1834    /// [`ModelContainer::ensure_resident`], typed ids in.
1835    pub fn ensure<M: Model>(
1836        &self,
1837        ids: impl IntoIterator<Item = impl Into<ModelId<M>>>,
1838    ) -> Result<(), DbError> {
1839        let keys: Vec<u64> = ids.into_iter().map(|i| i.into().handle()).collect();
1840        self.ensure_resident::<M>(&keys)
1841    }
1842
1843    /// Fault every row of `M`'s table in: the document pattern, said explicitly. A sketch's
1844    /// scene or a settings table is the working set, and the app that draws all of it warms
1845    /// it once at open. Raise the cache limit first (`set_cache_limit(usize::MAX)` for a
1846    /// document container) so the warmed rows are not immediately eligible to leave; rows
1847    /// deleted this turn stay deleted. Returns how many rows are resident afterward.
1848    pub fn warm<M: Model>(&self) -> Result<usize, DbError> {
1849        let store = self.cache::<M>();
1850        let key = ident(M::KEY);
1851        let keys = self.select_id_column_checked(&format!(
1852            "SELECT {key} FROM {} ORDER BY {key}",
1853            TableName::of::<M>().sql()
1854        ))?;
1855        self.fault_into(store.store_id(), &keys)?;
1856        Ok(store.with_untracked(|k| k.len()))
1857    }
1858
1859    /// Insert one row through the front door: it becomes resident and dirty, announces, and
1860    /// folds to one `INSERT` at the next flush.
1861    pub fn insert<M: Model>(&self, row: M) {
1862        let store = self.cache::<M>();
1863        let h = row.handle();
1864        store.restructure("create", Op::Insert, h, move |k| k.push(row));
1865    }
1866
1867    /// `SELECT COUNT(*)`: the table's true size, which the cache cannot know. Settles
1868    /// pending writes first (with autosave), like every other read.
1869    pub fn table_count<M: Model>(&self) -> Result<u64, DbError> {
1870        if self.inner.autosave.get() && !self.inner.dirty.borrow().is_empty() {
1871            self.save()?;
1872        }
1873        let mut n = 0i64;
1874        self.conn().query(
1875            &format!("SELECT COUNT(*) FROM {}", TableName::of::<M>().sql()),
1876            &[],
1877            &mut |row| {
1878                n = row.get(0).as_int().unwrap_or(0);
1879            },
1880        )?;
1881        Ok(n.max(0) as u64)
1882    }
1883
1884    /// The untyped fault: bring `keys` into `store_id`'s cache.
1885    pub(crate) fn fault_into(&self, store_id: u64, keys: &[u64]) -> Result<(), DbError> {
1886        let (is_resident, absorb, columns, table, keys_where) = {
1887            let tables = self.inner.tables.borrow();
1888            let Some(hooks) = tables.get(&store_id) else {
1889                return Ok(());
1890            };
1891            (
1892                hooks.is_resident.clone(),
1893                hooks.absorb.clone(),
1894                column_list(&hooks.columns),
1895                hooks.name.sql(),
1896                keys_where_of(hooks),
1897            )
1898        };
1899        let missing: Vec<u64> = {
1900            let dirty = self.inner.dirty.borrow();
1901            keys.iter()
1902                .copied()
1903                .filter(|k| !is_resident(*k))
1904                // A row deleted this turn still exists in the file until the flush; faulting
1905                // it back would resurrect it.
1906                .filter(|k| dirty.rows.get(&(store_id, *k)) != Some(&DirtyRow::Delete))
1907                .collect()
1908        };
1909        if missing.is_empty() {
1910            return Ok(());
1911        }
1912        let mut faulted: Vec<u64> = Vec::with_capacity(missing.len());
1913        for chunk in missing.chunks(MAX_BOUND_PARAMS / 2) {
1914            let (clause, params) = keys_where(chunk);
1915            let mut rows: Vec<Vec<Value>> = Vec::new();
1916            self.conn().query(
1917                &format!("SELECT {columns} FROM {table} WHERE {clause}"),
1918                &params,
1919                &mut |row| {
1920                    let n = Row::len(row);
1921                    rows.push((0..n).map(|i| row.get(i)).collect());
1922                },
1923            )?;
1924            faulted.extend(absorb(rows)?);
1925        }
1926        // The Recorder serves whole fixture tables whatever the `WHERE` says; keep only what
1927        // was asked for so faulting stays precise there too. (Through a `HashSet`; a linear
1928        // scan here made warming half a million rows quadratic.)
1929        let wanted: std::collections::HashSet<u64> = missing.iter().copied().collect();
1930        faulted.retain(|k| wanted.contains(k));
1931        self.enforce_cache_limit(store_id, &faulted, false);
1932        Ok(())
1933    }
1934
1935    /// Release resident rows beyond the cache limit: oldest first, never a dirty row, never
1936    /// an observed one, never one just faulted for the caller still holding it.
1937    ///
1938    /// The pass walks every resident key, so it runs with hysteresis on the fault path:
1939    /// nothing happens until the cache overshoots the limit by a slack margin, and one pass
1940    /// then brings it back to the limit, amortizing the walk over the faults that filled the
1941    /// slack, instead of paying O(resident) per fault batch. (Measured: a 50k-row relation
1942    /// traversal over a 500k-row table spent 99% of its time in the un-amortized walk.)
1943    /// `strict` skips the slack, so `set_cache_limit` enforces its new bound immediately.
1944    fn enforce_cache_limit(&self, store_id: u64, protect: &[u64], strict: bool) {
1945        let limit = self.inner.cache_limit.get();
1946        if limit == usize::MAX {
1947            return;
1948        }
1949        let (resident_len, evict) = {
1950            let tables = self.inner.tables.borrow();
1951            let Some(hooks) = tables.get(&store_id) else {
1952                return;
1953            };
1954            ((hooks.resident_len)(), hooks.evict.clone())
1955        };
1956        let slack = if strict {
1957            0
1958        } else {
1959            (limit / 8).clamp(64, 4096)
1960        };
1961        let threshold = if strict {
1962            limit
1963        } else {
1964            self.inner
1965                .eviction_retry
1966                .borrow()
1967                .get(&store_id)
1968                .copied()
1969                .unwrap_or_else(|| limit.saturating_add(slack))
1970        };
1971        if resident_len <= threshold {
1972            return;
1973        }
1974        // Back off even when every row is pinned. A fixed limit gate otherwise scans an
1975        // ever-growing dirty transaction on EVERY single-row fault (quadratic work).
1976        self.inner.eviction_retry.borrow_mut().insert(
1977            store_id,
1978            resident_len.saturating_add(slack.max(resident_len / 8).max(1)),
1979        );
1980        let resident = {
1981            let tables = self.inner.tables.borrow();
1982            match tables.get(&store_id) {
1983                Some(hooks) => (hooks.resident_keys)(),
1984                None => return,
1985            }
1986        };
1987        let excess = resident.len().saturating_sub(limit);
1988        let candidates: Vec<u64> = {
1989            let dirty = self.inner.dirty.borrow();
1990            if dirty.full.contains(&store_id) {
1991                return; // a wholesale rewrite is pending; every row is implicitly dirty
1992            }
1993            let protect: std::collections::HashSet<u64> = protect.iter().copied().collect();
1994            resident
1995                .into_iter()
1996                .filter(|key| !protect.contains(key) && !dirty.rows.contains_key(&(store_id, *key)))
1997                .collect()
1998        };
1999        let evicted = evict(&candidates, excess);
2000        let remaining = resident_len.saturating_sub(evicted);
2001        self.inner.eviction_retry.borrow_mut().insert(
2002            store_id,
2003            remaining
2004                .max(limit)
2005                .saturating_add(slack.max(remaining / 8).max(1)),
2006        );
2007    }
2008
2009    // --- internals ---------------------------------------------------------------------------
2010
2011    pub(crate) fn conn(&self) -> std::cell::RefMut<'_, Box<dyn SqliteConnection>> {
2012        self.inner.conn.borrow_mut()
2013    }
2014
2015    /// Run a one-column SELECT and hand back the values as key handles. Errors surface on the
2016    /// error signal (these run on read paths with no caller to give a `Result` to).
2017    /// One-column key SELECT with the error returned, for callers that have a `Result` to
2018    /// give it to (the read paths use [`ModelContainer::select_id_column`] instead).
2019    fn select_id_column_checked(&self, sql: &str) -> Result<Vec<u64>, DbError> {
2020        let mut ids = Vec::new();
2021        self.conn().query(sql, &[], &mut |row| {
2022            if let Some(h) = value_to_handle(&row.get(0)) {
2023                ids.push(h);
2024            }
2025        })?;
2026        Ok(ids)
2027    }
2028
2029    fn select_ids_checked(&self, sql: &str, params: &[Value]) -> Result<Vec<u64>, DbError> {
2030        let mut ids = Vec::new();
2031        self.conn().query(sql, params, &mut |row| {
2032            if let Some(h) = value_to_handle(&row.get(0)) {
2033                ids.push(h);
2034            }
2035        })?;
2036        Ok(ids)
2037    }
2038    pub(crate) fn select_id_column(&self, sql: &str, params: &[Value]) -> Vec<u64> {
2039        match self.select_ids_checked(sql, params) {
2040            Ok(ids) => ids,
2041            Err(e) => {
2042                self.inner.error.set(Some(e.to_string()));
2043                Vec::new()
2044            }
2045        }
2046    }
2047
2048    /// Run a one-column SELECT for a single REAL.
2049    pub(crate) fn select_real(&self, sql: &str, params: &[Value]) -> Option<f64> {
2050        let mut v = None;
2051        if let Err(e) = self.conn().query(sql, params, &mut |row| {
2052            v = row.get(0).as_real().ok();
2053        }) {
2054            self.inner.error.set(Some(e.to_string()));
2055        }
2056        v
2057    }
2058
2059    /// This turn's unflushed rows of one store: what relation reads overlay.
2060    pub(crate) fn dirty_rows_of(&self, store_id: u64) -> Vec<(u64, DirtyRow)> {
2061        self.inner
2062            .dirty
2063            .borrow()
2064            .rows
2065            .iter()
2066            .filter(|((s, _), _)| *s == store_id)
2067            .map(|((_, k), state)| (*k, state.clone()))
2068            .collect()
2069    }
2070
2071    /// One row's pending state, if any.
2072    pub(crate) fn dirty_state_of(&self, store_id: u64, key: u64) -> Option<DirtyRow> {
2073        self.inner
2074            .dirty
2075            .borrow()
2076            .rows
2077            .get(&(store_id, key))
2078            .cloned()
2079    }
2080
2081    fn ensure_schema_table(&self) -> Result<(), DbError> {
2082        self.conn().execute(
2083            "CREATE TABLE IF NOT EXISTS _day_schema (\
2084             table_name TEXT PRIMARY KEY, fingerprint TEXT NOT NULL, version INTEGER NOT NULL)",
2085            &[],
2086        )?;
2087        Ok(())
2088    }
2089
2090    fn db_version(&self) -> Result<u32, DbError> {
2091        let mut version = 0u32;
2092        self.conn().query(
2093            "SELECT version FROM _day_schema WHERE table_name = '_db'",
2094            &[],
2095            &mut |row| {
2096                version = row.get(0).as_int().unwrap_or(0) as u32;
2097            },
2098        )?;
2099        Ok(version)
2100    }
2101
2102    /// The `_db` version row, written on the caller's connection so it can ride inside the
2103    /// transaction whose effects it labels.
2104    fn write_db_version(conn: &mut dyn SqliteConnection, v: u32) -> Result<(), DbError> {
2105        conn.execute(
2106            "INSERT INTO _day_schema (table_name, fingerprint, version) VALUES ('_db', '', ?) \
2107             ON CONFLICT(table_name) DO UPDATE SET version = excluded.version",
2108            &[Value::Int(v as i64)],
2109        )?;
2110        Ok(())
2111    }
2112
2113    /// `f` and the version row it leads to, in one transaction: the stage's effects and its
2114    /// label commit together or not at all. A failure in the stage, in the version write, or
2115    /// in the commit itself takes the one rollback path.
2116    fn versioned_transaction(
2117        &self,
2118        version: u32,
2119        f: impl FnOnce(&mut dyn SqliteConnection) -> Result<(), DbError>,
2120    ) -> Result<(), DbError> {
2121        let mut conn = self.conn();
2122        conn.begin()?;
2123        let result = f(conn.as_mut())
2124            .and_then(|()| Self::write_db_version(conn.as_mut(), version))
2125            .and_then(|()| conn.commit());
2126        if result.is_err() {
2127            let _ = conn.rollback();
2128        }
2129        result
2130    }
2131
2132    fn run_stages(&self, plan: MigrationPlan) -> Result<(), DbError> {
2133        let target = plan.target_version();
2134        let mut current = self.db_version()?;
2135        if current > target && target > 0 {
2136            return Err(DbError::new(
2137                DbErrorKind::Schema,
2138                format!(
2139                    "database is at version {current}, newer than this build's {target} — \
2140                     refusing to open (an old app writing a new schema is how data rots)"
2141                ),
2142            ));
2143        }
2144        let mut stages = plan.stages;
2145        stages.sort_by_key(|s| s.from);
2146        for stage in stages {
2147            if stage.from != current {
2148                if stage.to <= current {
2149                    continue; // already applied
2150                }
2151                return Err(DbError::new(
2152                    DbErrorKind::Schema,
2153                    format!(
2154                        "migration stage {}→{} does not start at the database's version {current}",
2155                        stage.from, stage.to
2156                    ),
2157                ));
2158            }
2159            self.versioned_transaction(stage.to, stage.run)?;
2160            current = stage.to;
2161        }
2162        if target > 0 && current != target {
2163            self.versioned_transaction(target, |_| Ok(()))?;
2164        }
2165        Ok(())
2166    }
2167
2168    /// Create (or lightweight-migrate) `M`'s table and register its hooks. The cache starts
2169    /// empty: this is the line where the old engine read the whole table, and the lazy one
2170    /// exists so that nothing does.
2171    fn attach<M: Model>(&self) -> Result<(), DbError> {
2172        // FTS5's external-content table and the R*Tree address rows by ROWID (an i64), so
2173        // both need the key column to be the integer rowid. A wide-keyed model declaring one
2174        // is refused at open, naming the constraint, rather than silently mis-indexing.
2175        let key_sql = M::COLUMNS
2176            .iter()
2177            .find(|c| c.name == M::KEY)
2178            .map(|c| c.sql)
2179            .unwrap_or(SqlType::Integer);
2180        if key_sql != SqlType::Integer && M::SPATIAL.is_some() {
2181            return Err(DbError::new(
2182                DbErrorKind::Unsupported,
2183                format!(
2184                    "`{}` declares spatial(…), which addresses rows by ROWID — those \
2185                     need an integer `#[model(id)]`, not a Uuid or String key",
2186                    M::TABLE
2187                ),
2188            ));
2189        }
2190        // A second attach of the same model (a re-attached database after a catalog swap)
2191        // keeps its store and hooks; only the resident rows are stale, and they are dropped
2192        // so the next fault reads the new file.
2193        if self.inner.stores.borrow().contains_key(&TypeId::of::<M>()) {
2194            let store = self.cache::<M>();
2195            let keys: Vec<u64> = store.with_untracked(|k| k.keys());
2196            store.depopulate_many(&keys);
2197            return Ok(());
2198        }
2199        if !M::EXTERNAL {
2200            self.ensure_table::<M>()?;
2201        }
2202        for l in M::LINKS {
2203            self.inner.links.borrow_mut().push(LinkInfo {
2204                owner_table: M::TABLE,
2205                field: l.field,
2206                local: l.local,
2207                target_table: l.target_table,
2208                remote: l.remote,
2209            });
2210        }
2211
2212        let store: Store<Keyed<M>> = Store::new(Keyed::new(Vec::new()));
2213        let store_id = store.store_id();
2214
2215        let columns: Vec<String> = M::COLUMNS.iter().map(|c| c.name.to_string()).collect();
2216        let fields: Vec<String> = M::COLUMNS.iter().map(|c| c.field.to_string()).collect();
2217        let row_for = {
2218            Rc::new(move |key: u64| store.with_untracked(|k| k.get(key).map(|m| m.to_row())))
2219                as Rc<dyn Fn(u64) -> Option<Vec<Value>>>
2220        };
2221        let all_rows = {
2222            Rc::new(move || {
2223                store.with_untracked(|k| {
2224                    k.items().iter().map(|m| (m.handle(), m.to_row())).collect()
2225                })
2226            }) as Rc<dyn Fn() -> Vec<(u64, Vec<Value>)>>
2227        };
2228        let resident_keys =
2229            Rc::new(move || store.with_untracked(|k| k.keys())) as Rc<dyn Fn() -> Vec<u64>>;
2230        let resident_len =
2231            Rc::new(move || store.with_untracked(|k| k.len())) as Rc<dyn Fn() -> usize>;
2232        let is_resident = Rc::new(move |h: u64| store.with_untracked(|k| k.get(h).is_some()))
2233            as Rc<dyn Fn(u64) -> bool>;
2234
2235        let absorb = {
2236            Rc::new(
2237                move |raw_rows: Vec<Vec<Value>>| -> Result<Vec<u64>, DbError> {
2238                    let mut decoded: Vec<M> = Vec::with_capacity(raw_rows.len());
2239                    for r in raw_rows {
2240                        decoded.push(M::from_row(&r)?);
2241                    }
2242                    let keys: Vec<u64> = decoded.iter().map(|m| m.handle()).collect();
2243                    store.populate(decoded);
2244                    Ok(keys)
2245                },
2246            ) as AbsorbFn
2247        };
2248        let refresh = {
2249            Rc::new(move |raw_rows: Vec<Vec<Value>>| -> Result<bool, DbError> {
2250                let mut fresh: Vec<M> = Vec::with_capacity(raw_rows.len());
2251                for r in raw_rows {
2252                    fresh.push(M::from_row(&r)?);
2253                }
2254                let existing: Vec<u64> = store.with_untracked(|k| k.keys());
2255                let fresh_keys: std::collections::HashSet<u64> =
2256                    fresh.iter().map(|m| m.handle()).collect();
2257                let mut changed = false;
2258                day_model::with_author(ModelContainer::EXTERNAL_AUTHOR, || {
2259                    for m in fresh {
2260                        let key = m.handle();
2261                        match store.with_untracked(|k| k.get(key).map(|old| old.to_row())) {
2262                            // Selected by resident keys, so an unknown row means the cache
2263                            // moved underneath us; treat it as an arrival.
2264                            None => {
2265                                changed = true;
2266                                store.restructure("external", Op::Insert, key, |k| k.push(m));
2267                            }
2268                            Some(old) => {
2269                                let new_row = m.to_row();
2270                                let labels: Vec<&'static str> = M::COLUMNS
2271                                    .iter()
2272                                    .enumerate()
2273                                    .filter(|(i, _)| Row::get(&old, *i) != Row::get(&new_row, *i))
2274                                    .map(|(_, c)| c.field)
2275                                    .collect();
2276                                if !labels.is_empty() {
2277                                    changed = true;
2278                                    store.merge_row(key, m, &labels);
2279                                }
2280                            }
2281                        }
2282                    }
2283                    for gone in existing.into_iter().filter(|k| !fresh_keys.contains(k)) {
2284                        changed = true;
2285                        store.restructure("external", Op::Delete, gone, |k| {
2286                            k.remove(gone);
2287                        });
2288                    }
2289                });
2290                Ok(changed)
2291            }) as RefreshFn
2292        };
2293        let evict = Rc::new(move |candidates: &[u64], want: usize| {
2294            let doomed: Vec<u64> = candidates
2295                .iter()
2296                .copied()
2297                .filter(|h| !store.is_observed(*h))
2298                .take(want)
2299                .collect();
2300            store.depopulate_many(&doomed)
2301        }) as EvictFn;
2302        let watch_undo = Rc::new(move |stack: &day_model::UndoStack| stack.watch(store))
2303            as Rc<dyn Fn(&day_model::UndoStack)>;
2304
2305        self.inner.tables.borrow_mut().insert(
2306            store_id,
2307            TableHooks {
2308                table: M::TABLE,
2309                name: TableName::of::<M>(),
2310                key_cols: vec![M::KEY.to_string()],
2311                key_where: Rc::new(|h| (format!("{} = ?", ident(M::KEY)), vec![key_param(h)])),
2312                columns,
2313                column_names: M::COLUMNS.iter().map(|c| c.name).collect(),
2314                fields,
2315                resident_len,
2316                fts: (!M::FTS_COLUMNS.is_empty()).then(|| format!("{}_fts", M::TABLE)),
2317                fts_key: if key_sql == SqlType::Integer {
2318                    "rowid"
2319                } else {
2320                    "day_key"
2321                },
2322                spatial: M::SPATIAL.map(|s| {
2323                    (
2324                        s.lat.to_string(),
2325                        s.lon.to_string(),
2326                        format!("{}_geo", M::TABLE),
2327                    )
2328                }),
2329                row_for,
2330                all_rows,
2331                resident_keys,
2332                is_resident,
2333                absorb,
2334                refresh,
2335                evict,
2336                #[cfg(not(target_arch = "wasm32"))]
2337                dispose_cache: Rc::new(move || store.dispose_contents()),
2338                watch_undo,
2339            },
2340        );
2341        self.inner
2342            .stores
2343            .borrow_mut()
2344            .insert(TypeId::of::<M>(), Box::new(store));
2345        Ok(())
2346    }
2347
2348    fn ensure_table<M: Model>(&self) -> Result<(), DbError> {
2349        // Relation-contributed clauses fold into the fingerprint: adopting or changing a
2350        // delete rule re-runs this path. An existing table cannot gain the SQL-level clause
2351        // in place (SQLite has no `ALTER ADD CONSTRAINT`), so the comparison below routes
2352        // that difference through the table rebuild, which writes the declared DDL whole.
2353        let mut fp_val = model_fingerprint::<M>();
2354        for s in self
2355            .inner
2356            .fk_specs
2357            .borrow()
2358            .iter()
2359            .filter(|s| s.child_table == M::TABLE)
2360        {
2361            for b in s
2362                .child_field
2363                .bytes()
2364                .chain(s.parent_table.bytes())
2365                .chain(s.rule.sql().bytes())
2366            {
2367                fp_val ^= b as u64;
2368                fp_val = fp_val.wrapping_mul(0x1000_0000_01b3);
2369            }
2370        }
2371        let fp = format!("{fp_val:016x}");
2372        let stored = self.stored_fingerprint(M::TABLE)?;
2373        if stored.as_deref() == Some(fp.as_str()) {
2374            // The layout is known to match; the generated triggers are still reconciled, so
2375            // updated trigger logic reaches an existing file (an unchanged trigger is a read
2376            // and no write).
2377            self.conn().begin()?;
2378            let result = self
2379                .create_shadow_tables::<M>()
2380                .and_then(|()| self.conn().commit());
2381            if result.is_err() {
2382                let _ = self.conn().rollback();
2383            }
2384            return result;
2385        }
2386
2387        let name = TableName::of::<M>();
2388        // A missing table is created whole; an existing one is compared column by column,
2389        // index by index, and the comparison decides between in-place DDL and a rebuild,
2390        // or refuses before anything is touched.
2391        let delta = if self.table_exists(M::TABLE)? {
2392            let file = self.file_schema(name)?;
2393            Some((self.compare_schema::<M>(&file)?, file))
2394        } else {
2395            None
2396        };
2397        // A rebuild drops and recreates the table, which `PRAGMA foreign_keys` would treat
2398        // as a delete of every row (cascading into children). The pragma is a no-op inside
2399        // a transaction, so it toggles around one, and the rebuild's own
2400        // `foreign_key_check` is what guarantees nothing dangles afterwards.
2401        let rebuild = matches!(delta, Some((SchemaDelta::Rebuild, _)));
2402        let fk_was_on = rebuild && self.foreign_keys_enabled()?;
2403        if fk_was_on {
2404            self.conn().execute("PRAGMA foreign_keys = OFF", &[])?;
2405        }
2406
2407        // DDL, copy and backfill are one transaction: a failed migration leaves the old
2408        // table, index and fingerprint usable. Remove triggers before dropping indexed
2409        // columns.
2410        // Bound first: the `RefMut` the `begin` call holds must be gone before the closure
2411        // borrows the connection again.
2412        let began = self.conn().begin();
2413        let result = began.and_then(|()| {
2414            let result = (|| {
2415                let t = name.bare;
2416                if self.table_exists(&format!("{t}_fts"))? {
2417                    for suffix in ["ai", "ad", "au"] {
2418                        self.conn().execute(
2419                            &format!(
2420                                "DROP TRIGGER IF EXISTS {}",
2421                                ident(&format!("{t}_fts_{suffix}"))
2422                            ),
2423                            &[],
2424                        )?;
2425                    }
2426                    self.conn()
2427                        .execute(&format!("DROP TABLE {}", name.shadow("_fts")), &[])?;
2428                    self.conn().execute(
2429                        &format!("DROP TABLE IF EXISTS {}", name.shadow("_fts_keys")),
2430                        &[],
2431                    )?;
2432                }
2433                let mut unique_indexes: Vec<&'static str> = Vec::new();
2434                match delta {
2435                    None => self.create_table_named::<M>(name.sql())?,
2436                    Some((SchemaDelta::Lightweight(plan), _)) => {
2437                        unique_indexes = plan.unique_indexes.clone();
2438                        self.lightweight_migrate::<M>(&plan)?;
2439                    }
2440                    Some((SchemaDelta::Rebuild, file)) => {
2441                        self.rebuild_table::<M>(&file)?;
2442                    }
2443                }
2444                self.create_indexes::<M>(&unique_indexes)?;
2445                if rebuild {
2446                    let dangling = Self::foreign_key_check_on(self.conn().as_mut())?;
2447                    if let Some(first) = dangling.first() {
2448                        return Err(DbError::new(
2449                            DbErrorKind::Schema,
2450                            format!(
2451                                "rebuilding `{}` leaves {} row(s) with a dangling foreign \
2452                                 key ({first}); the file is unchanged",
2453                                M::TABLE,
2454                                dangling.len()
2455                            ),
2456                        ));
2457                    }
2458                }
2459                self.create_shadow_tables::<M>()?;
2460                // The fingerprint is the last write: it records a layout that has been
2461                // brought to the declaration and verified, never one still in flight.
2462                self.store_fingerprint(M::TABLE, &fp)?;
2463                self.conn().commit()
2464            })();
2465            if result.is_err() {
2466                let _ = self.conn().rollback();
2467            }
2468            result
2469        });
2470        if fk_was_on {
2471            let restored = self
2472                .conn()
2473                .execute("PRAGMA foreign_keys = ON", &[])
2474                .map(|_| ());
2475            return result.and(restored);
2476        }
2477        result
2478    }
2479
2480    /// Whether this connection enforces foreign keys right now (`PRAGMA foreign_keys`). A
2481    /// driver that answers nothing (the Recorder) counts as not enforcing.
2482    fn foreign_keys_enabled(&self) -> Result<bool, DbError> {
2483        let mut on = false;
2484        self.conn().query("PRAGMA foreign_keys", &[], &mut |row| {
2485            on = row.get(0).as_int().unwrap_or(0) != 0;
2486        })?;
2487        Ok(on)
2488    }
2489
2490    /// The FTS5 and R*Tree shadows a model declares: virtual tables plus `AFTER` triggers, so
2491    /// the indexes stay true inside the same transaction as every write, even a write made by
2492    /// another tool straight into the file. Backfilled when first created.
2493    fn create_shadow_tables<M: Model>(&self) -> Result<(), DbError> {
2494        let name = TableName::of::<M>();
2495        // The plain names, for messages and `sqlite_master` lookups; the quoted ones, for
2496        // every statement. A shadow's trigger is named after the shadow, quoted as one name.
2497        let bare = name.bare;
2498        let t = name.sql();
2499        let key = ident(M::KEY);
2500        let trigger = |suffix: &str| ident(&format!("{bare}{suffix}"));
2501        if !M::FTS_COLUMNS.is_empty() {
2502            if !self.inner.caps.full_text_search {
2503                return Err(DbError::new(
2504                    DbErrorKind::Unsupported,
2505                    format!(
2506                        "`{bare}` declares fts(…) but this driver's engine has no FTS5 — the \
2507                         bundled and cipher engines compile it in"
2508                    ),
2509                ));
2510            }
2511            let fts = name.shadow("_fts");
2512            // The hidden column FTS5 takes its commands through carries the index's own name.
2513            let fts_command = ident(&format!("{bare}_fts"));
2514            let cols = column_list(M::FTS_COLUMNS);
2515            let fresh = !self.table_exists(&format!("{bare}_fts"))?;
2516            // The tokenizer rides in the DDL; its value is a compile-time literal, quoted
2517            // through SQL's own escape so an apostrophe cannot break the statement.
2518            let tokenize = match M::FTS_TOKENIZE {
2519                Some(tok) => format!(", tokenize='{}'", tok.replace('\'', "''")),
2520                None => String::new(),
2521            };
2522            let new_cols = prefixed_list("new", M::FTS_COLUMNS);
2523            let wide = M::COLUMNS
2524                .iter()
2525                .find(|c| c.name == M::KEY)
2526                .is_some_and(|c| c.sql != SqlType::Integer);
2527            if wide {
2528                if M::FTS_COLUMNS.contains(&"day_key") {
2529                    return Err(DbError::new(
2530                        DbErrorKind::Unsupported,
2531                        "day_key is reserved in a non-integer-key FTS index",
2532                    ));
2533                }
2534                let key_type = M::COLUMNS
2535                    .iter()
2536                    .find(|c| c.name == M::KEY)
2537                    .expect("key column")
2538                    .sql
2539                    .ddl();
2540                let fts_keys = name.shadow("_fts_keys");
2541                // FTS rowids stay private. An indexed mapping makes update/delete O(log n)
2542                // rather than scanning the unindexed public key in every FTS row.
2543                self.conn().execute(&format!("CREATE TABLE IF NOT EXISTS {fts_keys}(day_rowid INTEGER PRIMARY KEY, day_key {key_type} NOT NULL UNIQUE)"), &[])?;
2544                self.conn().execute(&format!("CREATE VIRTUAL TABLE IF NOT EXISTS {fts} USING fts5(day_key UNINDEXED, {cols}{tokenize})"), &[])?;
2545                let insert = format!(
2546                    "INSERT INTO {fts_keys}(day_key) VALUES (new.{key}); INSERT INTO {fts}(rowid, day_key, {cols}) VALUES ((SELECT day_rowid FROM {fts_keys} WHERE day_key = new.{key}), new.{key}, {new_cols});"
2547                );
2548                let delete = format!(
2549                    "DELETE FROM {fts} WHERE rowid = (SELECT day_rowid FROM {fts_keys} WHERE day_key = old.{key}); DELETE FROM {fts_keys} WHERE day_key = old.{key};"
2550                );
2551                let (update_of, changed) = fts_update_guard(M::KEY, M::FTS_COLUMNS);
2552                for (suffix, event, body) in [
2553                    ("_fts_ai", "INSERT".to_string(), insert.clone()),
2554                    ("_fts_ad", "DELETE".to_string(), delete.clone()),
2555                    (
2556                        "_fts_au",
2557                        format!("UPDATE OF {update_of}"),
2558                        format!("{delete} {insert}"),
2559                    ),
2560                ] {
2561                    let when = if suffix == "_fts_au" {
2562                        format!(" WHEN {changed}")
2563                    } else {
2564                        String::new()
2565                    };
2566                    self.ensure_trigger(
2567                        &format!("{bare}{suffix}"),
2568                        name.schema,
2569                        &format!(
2570                            "CREATE TRIGGER {} AFTER {event} ON {t}{when} BEGIN {body} END",
2571                            trigger(suffix)
2572                        ),
2573                    )?;
2574                }
2575                if fresh {
2576                    self.conn().execute(
2577                        &format!("INSERT INTO {fts_keys}(day_key) SELECT {key} FROM {t}"),
2578                        &[],
2579                    )?;
2580                    let qualified = prefixed_list("b", M::FTS_COLUMNS);
2581                    self.conn().execute(&format!("INSERT INTO {fts}(rowid, day_key, {cols}) SELECT k.day_rowid, b.{key}, {qualified} FROM {t} b JOIN {fts_keys} k ON k.day_key = b.{key}"), &[])?;
2582                }
2583            } else {
2584                // FTS5 reads its options as SQL string literals; the content table and its
2585                // rowid column are plain names inside them.
2586                self.conn().execute(
2587                    &format!(
2588                        "CREATE VIRTUAL TABLE IF NOT EXISTS {fts} USING fts5({cols}, \
2589                     content='{}', content_rowid='{}'{tokenize})",
2590                        bare.replace('\'', "''"),
2591                        M::KEY.replace('\'', "''")
2592                    ),
2593                    &[],
2594                )?;
2595                let old_cols = prefixed_list("old", M::FTS_COLUMNS);
2596                self.ensure_trigger(
2597                    &format!("{bare}_fts_ai"),
2598                    name.schema,
2599                    &format!(
2600                        "CREATE TRIGGER {} AFTER INSERT ON {t} BEGIN \
2601                     INSERT INTO {fts}(rowid, {cols}) VALUES (new.{key}, {new_cols}); END",
2602                        trigger("_fts_ai")
2603                    ),
2604                )?;
2605                self.ensure_trigger(
2606                    &format!("{bare}_fts_ad"),
2607                    name.schema,
2608                    &format!(
2609                        "CREATE TRIGGER {} AFTER DELETE ON {t} BEGIN \
2610                     INSERT INTO {fts}({fts_command}, rowid, {cols}) VALUES ('delete', old.{key}, {old_cols}); END",
2611                        trigger("_fts_ad")
2612                    ),
2613                )?;
2614                // Only a change to indexed content or to the row's identity re-indexes it: an
2615                // update that touches other columns, or assigns an indexed column its own
2616                // value, leaves the index alone.
2617                let (update_of, changed) = fts_update_guard(M::KEY, M::FTS_COLUMNS);
2618                self.ensure_trigger(
2619                    &format!("{bare}_fts_au"),
2620                    name.schema,
2621                    &format!(
2622                        "CREATE TRIGGER {} AFTER UPDATE OF {update_of} ON {t} WHEN {changed} BEGIN \
2623                     INSERT INTO {fts}({fts_command}, rowid, {cols}) VALUES ('delete', old.{key}, {old_cols}); \
2624                     INSERT INTO {fts}(rowid, {cols}) VALUES (new.{key}, {new_cols}); END",
2625                        trigger("_fts_au")
2626                    ),
2627                )?;
2628                if fresh {
2629                    // External-content rebuild: index whatever rows predate the declaration.
2630                    self.conn().execute(
2631                        &format!("INSERT INTO {fts}({fts_command}) VALUES ('rebuild')"),
2632                        &[],
2633                    )?;
2634                }
2635            }
2636        }
2637        if let Some(s) = M::SPATIAL {
2638            if !self.inner.caps.rtree {
2639                return Err(DbError::new(
2640                    DbErrorKind::Unsupported,
2641                    format!(
2642                        "`{bare}` declares spatial(…) but this driver's engine has no R*Tree — \
2643                         the bundled and cipher engines compile it in"
2644                    ),
2645                ));
2646            }
2647            let (lat, lon) = (ident(s.lat), ident(s.lon));
2648            let geo = name.shadow("_geo");
2649            let fresh = !self.table_exists(&format!("{bare}_geo"))?;
2650            self.conn().execute(
2651                &format!(
2652                    "CREATE VIRTUAL TABLE IF NOT EXISTS {geo} USING rtree({key}, \
2653                     min_lat, max_lat, min_lon, max_lon)"
2654                ),
2655                &[],
2656            )?;
2657            self.ensure_trigger(
2658                &format!("{bare}_geo_ai"),
2659                name.schema,
2660                &format!(
2661                    "CREATE TRIGGER {} AFTER INSERT ON {t} BEGIN \
2662                     INSERT OR REPLACE INTO {geo} VALUES (new.{key}, new.{lat}, new.{lat}, new.{lon}, new.{lon}); END",
2663                    trigger("_geo_ai")
2664                ),
2665            )?;
2666            self.ensure_trigger(
2667                &format!("{bare}_geo_au"),
2668                name.schema,
2669                &format!(
2670                    "CREATE TRIGGER {} AFTER UPDATE OF {lat}, {lon} ON {t} BEGIN \
2671                     INSERT OR REPLACE INTO {geo} VALUES (new.{key}, new.{lat}, new.{lat}, new.{lon}, new.{lon}); END",
2672                    trigger("_geo_au")
2673                ),
2674            )?;
2675            self.ensure_trigger(
2676                &format!("{bare}_geo_ad"),
2677                name.schema,
2678                &format!(
2679                    "CREATE TRIGGER {} AFTER DELETE ON {t} BEGIN \
2680                     DELETE FROM {geo} WHERE {key} = old.{key}; END",
2681                    trigger("_geo_ad")
2682                ),
2683            )?;
2684            if fresh {
2685                self.conn().execute(
2686                    &format!(
2687                        "INSERT OR REPLACE INTO {geo} \
2688                         SELECT {key}, {lat}, {lat}, {lon}, {lon} FROM {t}"
2689                    ),
2690                    &[],
2691                )?;
2692            }
2693        }
2694        Ok(())
2695    }
2696
2697    /// Install a generated trigger, replacing one whose stored text differs: updated trigger
2698    /// logic reaches an existing file on the open that carries it, and a trigger that is
2699    /// already right costs one `sqlite_master` read and no write. `create_sql` is the plain
2700    /// `CREATE TRIGGER` statement; the engine stores that text verbatim, which is what the
2701    /// comparison reads back.
2702    fn ensure_trigger(
2703        &self,
2704        bare_name: &str,
2705        schema: Option<&str>,
2706        create_sql: &str,
2707    ) -> Result<(), DbError> {
2708        let master = table_sql(schema, "sqlite_master");
2709        let mut stored: Option<String> = None;
2710        self.conn().query(
2711            &format!("SELECT sql FROM {master} WHERE type = 'trigger' AND name = ?"),
2712            &[Value::Text(bare_name.into())],
2713            &mut |row| {
2714                if let Ok(sql) = row.get(0).as_text() {
2715                    stored = Some(sql.to_string());
2716                }
2717            },
2718        )?;
2719        match stored {
2720            Some(sql) if sql == create_sql => return Ok(()),
2721            Some(_) => {
2722                self.conn().execute(
2723                    &format!("DROP TRIGGER IF EXISTS {}", table_sql(schema, bare_name)),
2724                    &[],
2725                )?;
2726            }
2727            None => {}
2728        }
2729        self.conn().execute(create_sql, &[])?;
2730        Ok(())
2731    }
2732
2733    fn table_exists(&self, table: &str) -> Result<bool, DbError> {
2734        let mut found = false;
2735        self.conn().query(
2736            "SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?",
2737            &[Value::Text(table.into())],
2738            &mut |_| found = true,
2739        )?;
2740        Ok(found)
2741    }
2742
2743    /// `M`'s declared DDL under `table_sql` (its own quoted name, or the `_new` name a
2744    /// rebuild copies into). The fresh form carries every constraint inline; the upgrade
2745    /// paths produce the same enforced constraints by other means (a `NOT NULL DEFAULT`
2746    /// addition, a `day_uidx_` unique index), and the comparison treats them as equal.
2747    fn create_table_named<M: Model>(&self, table_sql: String) -> Result<(), DbError> {
2748        let cols: Vec<String> = self
2749            .declared_columns::<M>()
2750            .iter()
2751            .map(|d| {
2752                let c = &d.def;
2753                let mut s = format!("{} {}", ident(c.name), c.sql.ddl());
2754                if d.pk {
2755                    s.push_str(" PRIMARY KEY");
2756                } else if c.not_null {
2757                    s.push_str(" NOT NULL");
2758                }
2759                if c.unique && !d.pk {
2760                    s.push_str(" UNIQUE");
2761                }
2762                // A declared relation's foreign key: enforced by the engine too, so another
2763                // process honors the same delete rule, and so a cascade's reach into rows
2764                // this process never faulted still lands in the file. Deferred, so
2765                // within-transaction statement order (a cascade's children, an undo's
2766                // re-inserts) never trips it.
2767                if let Some(fk) = &d.fk {
2768                    s.push_str(&fk.clause());
2769                }
2770                s
2771            })
2772            .collect();
2773        let body = format!("CREATE TABLE {table_sql} ({})", cols.join(", "));
2774        // `STRICT` where the engine allows it (always, for the bundled build); older system
2775        // SQLites reject the keyword, and the plain form is the fallback.
2776        if self.conn().execute(&format!("{body} STRICT"), &[]).is_err() {
2777            self.conn().execute(&body, &[])?;
2778        }
2779        Ok(())
2780    }
2781
2782    /// The indexes `M` owns in the file, by name: one per `indexed` column, one per declared
2783    /// relation's foreign-key column (what `children_of` selects on, what relation predicates
2784    /// correlate on, and what the engine scans to enforce `ON DELETE`; unindexed, each is a
2785    /// full scan of the child table), one per composite declaration, and a unique index for
2786    /// each column in `unique_indexes`, the declared-unique columns whose table carries no
2787    /// inline `UNIQUE` because they arrived through `ADD COLUMN`.
2788    fn declared_indexes<M: Model>(&self, unique_indexes: &[&'static str]) -> Vec<DeclaredIndex> {
2789        let fk_cols: Vec<&str> = self
2790            .inner
2791            .fk_specs
2792            .borrow()
2793            .iter()
2794            .filter(|s| s.child_table == M::TABLE)
2795            .filter_map(|s| {
2796                M::COLUMNS
2797                    .iter()
2798                    .find(|c| c.field == s.child_field)
2799                    .map(|c| c.name)
2800            })
2801            .collect();
2802        let bare = TableName::of::<M>().bare;
2803        let mut out: Vec<DeclaredIndex> = M::COLUMNS
2804            .iter()
2805            .filter(|c| c.indexed || (fk_cols.contains(&c.name) && c.name != M::KEY))
2806            .map(|c| DeclaredIndex {
2807                name: format!("day_idx_{bare}_{}", c.name),
2808                columns: vec![c.name.to_string()],
2809                unique: false,
2810            })
2811            .collect();
2812        out.extend(M::COMPOSITE_INDEXES.iter().map(|idx| DeclaredIndex {
2813            name: format!("day_idx_{bare}_{}", idx.join("_")),
2814            columns: idx.iter().map(|c| c.to_string()).collect(),
2815            unique: false,
2816        }));
2817        out.extend(unique_indexes.iter().map(|c| DeclaredIndex {
2818            name: format!("day_uidx_{bare}_{c}"),
2819            columns: vec![c.to_string()],
2820            unique: true,
2821        }));
2822        out
2823    }
2824
2825    fn create_indexes<M: Model>(&self, unique_indexes: &[&'static str]) -> Result<(), DbError> {
2826        let name = TableName::of::<M>();
2827        let t = name.sql();
2828        for idx in self.declared_indexes::<M>(unique_indexes) {
2829            self.conn().execute(
2830                &format!(
2831                    "CREATE {}INDEX IF NOT EXISTS {} ON {t}({})",
2832                    if idx.unique { "UNIQUE " } else { "" },
2833                    table_sql(name.schema, &idx.name),
2834                    column_list(&idx.columns)
2835                ),
2836                &[],
2837            )?;
2838        }
2839        Ok(())
2840    }
2841
2842    /// `M`'s columns as declared, with the relation-contributed foreign key resolved: the
2843    /// declared side of [`ModelContainer::compare_schema`], and what the DDL is written from.
2844    fn declared_columns<M: Model>(&self) -> Vec<DeclaredColumn> {
2845        let specs = self.inner.fk_specs.borrow();
2846        M::COLUMNS
2847            .iter()
2848            .enumerate()
2849            .map(|(index, c)| DeclaredColumn {
2850                def: *c,
2851                index,
2852                pk: c.name == M::KEY,
2853                fk: specs
2854                    .iter()
2855                    .find(|f| f.child_table == M::TABLE && f.child_field == c.field)
2856                    .map(|s| DeclaredFk {
2857                        parent_sql: s.parent_sql.clone(),
2858                        parent_bare: s.parent_bare,
2859                        parent_key: s.parent_key,
2860                        rule: s.rule,
2861                    }),
2862            })
2863            .collect()
2864    }
2865
2866    /// The table as the file has it: `PRAGMA table_info`, `foreign_key_list`, `index_list`
2867    /// and `index_info`, read into one value the comparison and the rebuild both consume.
2868    fn file_schema(&self, name: TableName) -> Result<FileSchema, DbError> {
2869        let mut columns: Vec<FileColumn> = Vec::new();
2870        self.conn().query(
2871            &pragma_sql(name.schema, "table_info", name.bare),
2872            &[],
2873            &mut |row| {
2874                columns.push(FileColumn {
2875                    name: row.get(1).as_text().unwrap_or("").to_string(),
2876                    decl_type: row.get(2).as_text().unwrap_or("").to_ascii_uppercase(),
2877                    not_null: row.get(3).as_int().unwrap_or(0) != 0,
2878                    pk: row.get(5).as_int().unwrap_or(0) != 0,
2879                    fk: None,
2880                });
2881            },
2882        )?;
2883        self.conn().query(
2884            &pragma_sql(name.schema, "foreign_key_list", name.bare),
2885            &[],
2886            &mut |row| {
2887                let from = row.get(3).as_text().unwrap_or("").to_string();
2888                if let Some(col) = columns.iter_mut().find(|c| c.name == from) {
2889                    col.fk = Some(FileFk {
2890                        parent: row.get(2).as_text().unwrap_or("").to_string(),
2891                        parent_key: row.get(4).as_text().unwrap_or("").to_string(),
2892                        on_delete: row.get(6).as_text().unwrap_or("").to_ascii_uppercase(),
2893                    });
2894                }
2895            },
2896        )?;
2897        let mut indexes: Vec<FileIndex> = Vec::new();
2898        self.conn().query(
2899            &pragma_sql(name.schema, "index_list", name.bare),
2900            &[],
2901            &mut |row| {
2902                indexes.push(FileIndex {
2903                    name: row.get(1).as_text().unwrap_or("").to_string(),
2904                    unique: row.get(2).as_int().unwrap_or(0) != 0,
2905                    origin: row.get(3).as_text().unwrap_or("").to_string(),
2906                    columns: Vec::new(),
2907                });
2908            },
2909        )?;
2910        for idx in &mut indexes {
2911            let mut cols: Vec<(i64, String)> = Vec::new();
2912            self.conn().query(
2913                &pragma_sql(name.schema, "index_info", &idx.name),
2914                &[],
2915                &mut |row| {
2916                    cols.push((
2917                        row.get(0).as_int().unwrap_or(0),
2918                        row.get(2).as_text().unwrap_or("").to_string(),
2919                    ));
2920                },
2921            )?;
2922            cols.sort_by_key(|(seq, _)| *seq);
2923            idx.columns = cols.into_iter().map(|(_, c)| c).collect();
2924        }
2925        Ok(FileSchema { columns, indexes })
2926    }
2927
2928    /// Decide how the file's table reaches the declaration. Column and index additions and
2929    /// drops are closed in place; any other difference (a type, a nullability, a unique
2930    /// constraint, a foreign key or its delete action) is a rebuild; a change the rebuild
2931    /// could not fill truthfully (the key column, a required column with no usable default)
2932    /// is refused here, before anything is touched.
2933    fn compare_schema<M: Model>(&self, file: &FileSchema) -> Result<SchemaDelta, DbError> {
2934        let declared = self.declared_columns::<M>();
2935        let defaults = M::default_row();
2936        let bare = TableName::of::<M>().bare;
2937        let refuse = |column: &str, what: String| {
2938            Err(DbError::new(
2939                DbErrorKind::Schema,
2940                format!(
2941                    "`{}`.`{column}`: {what}; this is not a change the container can make to \
2942                     an existing table, and it refuses to open rather than guess (a staged \
2943                     migration can make it)",
2944                    M::TABLE
2945                ),
2946            ))
2947        };
2948        let mut plan = LightweightPlan::default();
2949        // Any one of these differences needs the copy; which one is immaterial to the work.
2950        let mut rebuild = false;
2951        let mut note = |_why: String| rebuild = true;
2952        let inline_unique = |col: &str| {
2953            file.indexes
2954                .iter()
2955                .any(|i| i.origin == "u" && i.columns.len() == 1 && i.columns[0] == col)
2956        };
2957        let unique_index_name = |col: &str| format!("day_uidx_{bare}_{col}");
2958        let has_unique_index = |col: &str| {
2959            let name = unique_index_name(col);
2960            file.indexes
2961                .iter()
2962                .any(|i| i.name == name && i.unique && i.columns.len() == 1 && i.columns[0] == col)
2963        };
2964
2965        for d in &declared {
2966            let c = &d.def;
2967            let default = Row::get(&defaults, d.index);
2968            let required = c.not_null || d.pk;
2969            match file.columns.iter().find(|f| f.name == c.name) {
2970                None => {
2971                    if d.pk {
2972                        return refuse(c.name, "the declared key column is not in the file".into());
2973                    }
2974                    if required && default == Value::Null {
2975                        return refuse(
2976                            c.name,
2977                            "a required column whose Rust default stores NULL cannot be \
2978                             added to existing rows"
2979                                .into(),
2980                        );
2981                    }
2982                    if d.fk.is_some() && required {
2983                        // `ADD COLUMN … REFERENCES` must default to NULL while foreign keys
2984                        // are enforced; a required reference needs the copy.
2985                        note(format!("required foreign-key column `{}` added", c.name));
2986                    } else {
2987                        plan.add.push(d.index);
2988                        if c.unique {
2989                            plan.unique_indexes.push(c.name);
2990                        }
2991                    }
2992                }
2993                Some(f) => {
2994                    if f.decl_type != c.sql.ddl() {
2995                        if d.pk {
2996                            return refuse(
2997                                c.name,
2998                                format!(
2999                                    "the key column is {} in the file and {} in the declaration",
3000                                    f.decl_type,
3001                                    c.sql.ddl()
3002                                ),
3003                            );
3004                        }
3005                        note(format!(
3006                            "`{}` is {} in the file and {} in the declaration",
3007                            c.name,
3008                            f.decl_type,
3009                            c.sql.ddl()
3010                        ));
3011                    }
3012                    if f.pk != d.pk {
3013                        return refuse(
3014                            c.name,
3015                            if d.pk {
3016                                "declared as the key but not the file's primary key".into()
3017                            } else {
3018                                "the file's primary key but not the declared key".into()
3019                            },
3020                        );
3021                    }
3022                    // The key's NOT NULL is implied by PRIMARY KEY (and reported either way
3023                    // by the engine, depending on its type), so only the other columns compare.
3024                    if !d.pk && f.not_null != c.not_null {
3025                        if c.not_null && default == Value::Null {
3026                            return refuse(
3027                                c.name,
3028                                "becomes required, but its Rust default stores NULL, so \
3029                                 existing NULLs cannot be filled"
3030                                    .into(),
3031                            );
3032                        }
3033                        note(format!(
3034                            "`{}` is {} in the file and {} in the declaration",
3035                            c.name,
3036                            if f.not_null { "NOT NULL" } else { "nullable" },
3037                            if c.not_null { "NOT NULL" } else { "nullable" }
3038                        ));
3039                    }
3040                    let want_unique = c.unique && !d.pk;
3041                    // A declared unique without the inline constraint owns its index,
3042                    // whether that index exists yet or not (`IF NOT EXISTS` keeps one).
3043                    match (want_unique, inline_unique(c.name), has_unique_index(c.name)) {
3044                        (true, false, _) => plan.unique_indexes.push(c.name),
3045                        (false, true, _) => note(format!(
3046                            "`{}` carries UNIQUE but is not declared unique",
3047                            c.name
3048                        )),
3049                        (false, false, true) => plan.drop_indexes.push(unique_index_name(c.name)),
3050                        _ => {}
3051                    }
3052                    let fk_matches = match (&d.fk, &f.fk) {
3053                        (None, None) => true,
3054                        (Some(want), Some(have)) => {
3055                            have.parent == want.parent_bare
3056                                && have.parent_key == want.parent_key
3057                                && have.on_delete == want.rule.sql()
3058                        }
3059                        _ => false,
3060                    };
3061                    if !fk_matches {
3062                        note(format!("`{}`'s foreign key differs", c.name));
3063                    }
3064                }
3065            }
3066        }
3067        for f in file
3068            .columns
3069            .iter()
3070            .filter(|f| !declared.iter().any(|d| d.def.name == f.name))
3071        {
3072            if f.pk {
3073                return refuse(&f.name, "the file's primary key is not declared".into());
3074            }
3075            // `DROP COLUMN` refuses a column under a constraint or a foreign index; those go
3076            // through the copy, which simply leaves them out.
3077            let covered_by_foreign_index = file.indexes.iter().any(|i| {
3078                i.origin == "c" && !is_day_index(&i.name, bare) && i.columns.contains(&f.name)
3079            });
3080            if f.fk.is_some() || inline_unique(&f.name) || covered_by_foreign_index {
3081                note(format!("constrained column `{}` dropped", f.name));
3082            } else {
3083                plan.drop.push(f.name.clone());
3084            }
3085        }
3086        if let Some(extra) = file
3087            .indexes
3088            .iter()
3089            .find(|i| i.origin == "u" && i.columns.len() > 1)
3090        {
3091            note(format!(
3092                "undeclared multi-column UNIQUE constraint over ({})",
3093                extra.columns.join(", ")
3094            ));
3095        }
3096        // Day's own indexes: one that no longer matches a declaration is dropped in place;
3097        // a missing one is created after the column work.
3098        let want = self.declared_indexes::<M>(&plan.unique_indexes);
3099        for i in file
3100            .indexes
3101            .iter()
3102            .filter(|i| i.origin == "c" && is_day_index(&i.name, bare))
3103        {
3104            let matches = want
3105                .iter()
3106                .any(|w| w.name == i.name && w.columns == i.columns && w.unique == i.unique);
3107            if !matches && !plan.drop_indexes.contains(&i.name) {
3108                plan.drop_indexes.push(i.name.clone());
3109            }
3110        }
3111        Ok(if rebuild {
3112            SchemaDelta::Rebuild
3113        } else {
3114            SchemaDelta::Lightweight(plan)
3115        })
3116    }
3117
3118    /// Close the gap in place: indexes that no longer match are dropped, missing columns are
3119    /// added with their declared constraints (`NOT NULL DEFAULT <Rust default>`; a nullable
3120    /// foreign key with its `REFERENCES`), and columns the declaration lost are dropped.
3121    /// Declared-unique additions get their unique index from `create_indexes`, since
3122    /// `ADD COLUMN` cannot carry `UNIQUE`.
3123    fn lightweight_migrate<M: Model>(&self, plan: &LightweightPlan) -> Result<(), DbError> {
3124        let name = TableName::of::<M>();
3125        let t = name.sql();
3126        let declared = self.declared_columns::<M>();
3127        let defaults = M::default_row();
3128        for idx in &plan.drop_indexes {
3129            self.conn().execute(
3130                &format!("DROP INDEX IF EXISTS {}", table_sql(name.schema, idx)),
3131                &[],
3132            )?;
3133        }
3134        for &i in &plan.add {
3135            let d = &declared[i];
3136            let c = &d.def;
3137            let default = Row::get(&defaults, i);
3138            let mut ddl = format!(
3139                "ALTER TABLE {t} ADD COLUMN {} {}",
3140                ident(c.name),
3141                c.sql.ddl()
3142            );
3143            if c.not_null {
3144                ddl.push_str(&format!(" NOT NULL DEFAULT {}", sql_literal(&default)?));
3145            }
3146            if let Some(fk) = &d.fk {
3147                ddl.push_str(&fk.clause());
3148            }
3149            self.conn().execute(&ddl, &[])?;
3150            // A nullable column whose Rust default is a value still backfills, as before.
3151            if !c.not_null && default != Value::Null {
3152                self.conn()
3153                    .execute(&format!("UPDATE {t} SET {} = ?", ident(c.name)), &[default])?;
3154            }
3155        }
3156        for gone in &plan.drop {
3157            self.conn()
3158                .execute(&format!("ALTER TABLE {t} DROP COLUMN {}", ident(gone)), &[])?;
3159        }
3160        Ok(())
3161    }
3162
3163    /// SQLite's documented procedure for every other change
3164    /// (<https://www.sqlite.org/lang_altertable.html#otheralter>): the declared table is
3165    /// created under a temporary name, the rows copied across (shared columns as they are,
3166    /// a required column that was nullable through `COALESCE` with its Rust default, a
3167    /// missing one from that default), the old table dropped, the new one renamed into
3168    /// place, and the application's own indexes that still apply re-created. The caller
3169    /// holds the transaction, has foreign keys off, and runs `foreign_key_check` after.
3170    /// Application-owned triggers on the table go with it; their owner re-creates them.
3171    fn rebuild_table<M: Model>(&self, file: &FileSchema) -> Result<(), DbError> {
3172        let name = TableName::of::<M>();
3173        let t = name.sql();
3174        let bare = name.bare;
3175        let tmp_bare = format!("{bare}_new");
3176        let tmp = table_sql(name.schema, &tmp_bare);
3177        let declared = self.declared_columns::<M>();
3178        let defaults = M::default_row();
3179        let master = table_sql(name.schema, "sqlite_master");
3180
3181        // The application's own indexes, to re-create when every column they name survives.
3182        let mut foreign_indexes: Vec<String> = Vec::new();
3183        self.conn().query(
3184            &format!(
3185                "SELECT name, sql FROM {master} WHERE type = 'index' AND tbl_name = ? \
3186                 AND sql IS NOT NULL"
3187            ),
3188            &[Value::Text(bare.into())],
3189            &mut |row| {
3190                let idx_name = row.get(0).as_text().unwrap_or("").to_string();
3191                if is_day_index(&idx_name, bare) {
3192                    return;
3193                }
3194                let survives = file
3195                    .indexes
3196                    .iter()
3197                    .find(|i| i.name == idx_name)
3198                    .is_some_and(|i| {
3199                        i.columns
3200                            .iter()
3201                            .all(|c| declared.iter().any(|d| d.def.name == c))
3202                    });
3203                if survives && let Ok(sql) = row.get(1).as_text() {
3204                    foreign_indexes.push(sql.to_string());
3205                }
3206            },
3207        )?;
3208
3209        // Triggers on the table go with `DROP TABLE`; the application's come back verbatim
3210        // after the rename when every column they name survives (Day's generated ones are
3211        // reconciled afterwards anyway). Checked by a dry parse against the new table.
3212        let mut triggers: Vec<String> = Vec::new();
3213        self.conn().query(
3214            &format!(
3215                "SELECT sql FROM {master} WHERE type = 'trigger' AND tbl_name = ? \
3216                 AND sql IS NOT NULL ORDER BY rowid"
3217            ),
3218            &[Value::Text(bare.into())],
3219            &mut |row| {
3220                if let Ok(sql) = row.get(0).as_text() {
3221                    triggers.push(sql.to_string());
3222                }
3223            },
3224        )?;
3225
3226        let wrap =
3227            |e: DbError| DbError::new(e.kind, format!("rebuilding `{}`: {}", M::TABLE, e.message));
3228        self.conn()
3229            .execute(&format!("DROP TABLE IF EXISTS {tmp}"), &[])
3230            .map_err(wrap)?;
3231        self.create_table_named::<M>(tmp.clone()).map_err(wrap)?;
3232        let targets = column_list(&declared.iter().map(|d| d.def.name).collect::<Vec<_>>());
3233        let mut sources: Vec<String> = Vec::with_capacity(declared.len());
3234        for d in &declared {
3235            let c = &d.def;
3236            let default = Row::get(&defaults, d.index);
3237            let required = c.not_null || d.pk;
3238            let shared = file.columns.iter().find(|f| f.name == c.name);
3239            sources.push(match shared {
3240                Some(f) if !required || f.not_null => ident(c.name),
3241                Some(_) => format!(
3242                    "COALESCE({}, {})",
3243                    ident(c.name),
3244                    sql_literal(&default).map_err(wrap)?
3245                ),
3246                None => sql_literal(&default).map_err(wrap)?,
3247            });
3248        }
3249        self.conn()
3250            .execute(
3251                &format!(
3252                    "INSERT INTO {tmp} ({targets}) SELECT {} FROM {t}",
3253                    sources.join(", ")
3254                ),
3255                &[],
3256            )
3257            .map_err(wrap)?;
3258        self.conn()
3259            .execute(&format!("DROP TABLE {t}"), &[])
3260            .map_err(wrap)?;
3261        // Other tables' triggers and views may name this table (an adapter's capture trigger
3262        // on a child table reads the parent), and a rename checks every such reference
3263        // against a schema the table is momentarily absent from. The legacy rename leaves
3264        // those references as written, which is what a rebuild under the same name wants.
3265        self.conn()
3266            .execute_batch("PRAGMA legacy_alter_table = ON")
3267            .map_err(wrap)?;
3268        let renamed = self
3269            .conn()
3270            .execute(&format!("ALTER TABLE {tmp} RENAME TO {}", ident(bare)), &[]);
3271        self.conn()
3272            .execute_batch("PRAGMA legacy_alter_table = OFF")
3273            .map_err(wrap)?;
3274        renamed.map_err(wrap)?;
3275        for sql in foreign_indexes {
3276            self.conn().execute(&sql, &[]).map_err(wrap)?;
3277        }
3278        for sql in triggers {
3279            // A trigger naming a dropped column cannot come back; its owner recreates it.
3280            let _ = self.conn().execute(&sql, &[]);
3281        }
3282        Ok(())
3283    }
3284
3285    fn stored_fingerprint(&self, table: &str) -> Result<Option<String>, DbError> {
3286        let mut fp = None;
3287        self.conn().query(
3288            "SELECT fingerprint FROM _day_schema WHERE table_name = ?",
3289            &[Value::Text(table.into())],
3290            &mut |row| {
3291                if let Ok(t) = row.get(0).as_text() {
3292                    fp = Some(t.to_string());
3293                }
3294            },
3295        )?;
3296        Ok(fp)
3297    }
3298
3299    fn store_fingerprint(&self, table: &str, fp: &str) -> Result<(), DbError> {
3300        self.conn().execute(
3301            "INSERT INTO _day_schema (table_name, fingerprint, version) VALUES (?, ?, 0) \
3302             ON CONFLICT(table_name) DO UPDATE SET fingerprint = excluded.fingerprint",
3303            &[Value::Text(table.into()), Value::Text(fp.into())],
3304        )?;
3305        Ok(())
3306    }
3307
3308    fn install_sink(&self) {
3309        let inner = Rc::downgrade(&self.inner);
3310        let sink = day_model::install_change_sink(move |change| {
3311            if let Some(inner) = inner.upgrade() {
3312                // An external merge's changes are the database's own contents arriving:
3313                // queries go stale on them like any edit, but nothing goes back to the file.
3314                if change.author != Some(ModelContainer::EXTERNAL_AUTHOR) {
3315                    let tables = inner.tables.borrow();
3316                    inner
3317                        .dirty
3318                        .borrow_mut()
3319                        .note(change, |store| tables.contains_key(&store));
3320                }
3321                let container = ModelContainer { inner };
3322                container.mark_queries_stale(change);
3323                container.relations_on_change(change);
3324            }
3325        });
3326        self.inner.sink.set(Some(sink));
3327    }
3328
3329    fn install_autosave(&self) {
3330        // Weak: `on_turn_end` registrations live for the process, and a strong clone here
3331        // would keep the container (and its connection) alive forever.
3332        let inner = Rc::downgrade(&self.inner);
3333        day_reactive::on_turn_end(move || {
3334            let Some(inner) = inner.upgrade() else { return };
3335            let this = ModelContainer { inner };
3336            if this.inner.autosave.get() && !this.inner.dirty.borrow().is_empty() {
3337                // The error lands in the signal; a turn end has no caller to give a Result to.
3338                let _ = this.save();
3339            }
3340        });
3341    }
3342
3343    fn flush(&self, dirty: &DirtyState) -> Result<Vec<(String, Vec<Value>)>, DbError> {
3344        let capture = self.inner.participant.borrow().is_some();
3345        let (stmts, changes) = self.fold(dirty, capture)?;
3346        if stmts.is_empty() {
3347            return Ok(stmts);
3348        }
3349        let mut conn = self.conn();
3350        let outer = self.inner.worker_transaction.get();
3351        if !outer {
3352            conn.begin()?;
3353            if let Err(error) = self.participant_begin(conn.as_mut(), WriteOrigin::Local) {
3354                let _ = conn.rollback();
3355                return Err(error);
3356            }
3357        }
3358        // After begin (which clears the set) and before the statements: an outer worker
3359        // transaction accumulates every inner flush, and rolls the set back with the rest.
3360        self.record_changes(changes);
3361        for (sql, params) in &stmts {
3362            if let Err(e) = conn.execute(sql, params) {
3363                if !outer {
3364                    let _ = conn.rollback();
3365                    self.participant_rollback();
3366                }
3367                return Err(e);
3368            }
3369        }
3370        if !outer
3371            && let Err(error) = self
3372                .participant_commit(conn.as_mut(), WriteOrigin::Local)
3373                .and_then(|_| conn.commit())
3374        {
3375            let _ = conn.rollback();
3376            self.participant_rollback();
3377            return Err(error);
3378        }
3379        Ok(stmts)
3380    }
3381
3382    #[cfg(not(target_arch = "wasm32"))]
3383    fn worker_dispose_caches(&self) {
3384        for hooks in self.inner.tables.borrow().values() {
3385            (hooks.dispose_cache)();
3386        }
3387        // Reactive callbacks can retain a container until thread teardown. Release transaction
3388        // resources (including replication's writer lease) before acknowledging worker close.
3389        self.inner.participant.borrow_mut().take();
3390    }
3391
3392    #[cfg(not(target_arch = "wasm32"))]
3393    fn worker_write<R>(
3394        &self,
3395        write: impl FnOnce(&Self) -> Result<R, DbError>,
3396    ) -> Result<R, DbError> {
3397        self.worker_write_origin(WriteOrigin::Local, write)
3398    }
3399
3400    #[cfg(not(target_arch = "wasm32"))]
3401    fn worker_write_origin<R>(
3402        &self,
3403        origin: WriteOrigin,
3404        write: impl FnOnce(&Self) -> Result<R, DbError>,
3405    ) -> Result<R, DbError> {
3406        self.save()?;
3407        self.conn().begin()?;
3408        let admission = self.participant_begin(self.conn().as_mut(), origin);
3409        if let Err(error) = admission {
3410            self.conn().rollback().map_err(|rollback| {
3411                DbError::new(
3412                    DbErrorKind::Closed,
3413                    format!("transaction admission rollback failed: {rollback}"),
3414                )
3415            })?;
3416            return Err(error);
3417        }
3418        self.inner.worker_transaction.set(true);
3419        let result = day_reactive::batch(|| {
3420            let value = write(self)?;
3421            self.save()?;
3422            Ok(value)
3423        })
3424        .and_then(|value| {
3425            // Reactions may have made additional writes while the batch drained. Commit only
3426            // after those writes settle, never before turn-end autosave has run.
3427            self.save()?;
3428            self.participant_commit(self.conn().as_mut(), origin)?;
3429            self.conn().commit()?;
3430            Ok(value)
3431        });
3432        self.inner.worker_transaction.set(false);
3433        if result.is_err() {
3434            self.participant_rollback();
3435            self.conn().rollback().map_err(|error| {
3436                DbError::new(
3437                    DbErrorKind::Closed,
3438                    format!("transaction rollback failed: {error}"),
3439                )
3440            })?;
3441            self.worker_discard_changes()?;
3442        }
3443        if result.is_ok() {
3444            self.participant_refresh()?;
3445        }
3446        result
3447    }
3448
3449    #[cfg(not(target_arch = "wasm32"))]
3450    fn worker_read<R>(&self, read: impl FnOnce(&Self) -> Result<R, DbError>) -> Result<R, DbError> {
3451        self.save()?;
3452        self.conn().execute_batch("PRAGMA query_only = ON")?;
3453        self.conn().begin().map_err(|error| {
3454            DbError::new(
3455                DbErrorKind::Closed,
3456                format!("could not begin read snapshot: {error}"),
3457            )
3458        })?;
3459        let autosave = self.inner.autosave.replace(false);
3460        let mut result =
3461            day_model::with_author("worker-read", || day_reactive::batch(|| read(self)));
3462        let mutated = !self.inner.dirty.borrow().is_empty();
3463        self.conn().rollback().map_err(|error| {
3464            DbError::new(
3465                DbErrorKind::Closed,
3466                format!("could not end read snapshot: {error}"),
3467            )
3468        })?;
3469        self.conn()
3470            .execute_batch("PRAGMA query_only = OFF")
3471            .map_err(|error| {
3472                DbError::new(
3473                    DbErrorKind::Closed,
3474                    format!("could not restore write mode: {error}"),
3475                )
3476            })?;
3477        if mutated {
3478            self.worker_discard_changes()?;
3479            result = Err(DbError::driver(
3480                "a worker read attempted to mutate model state",
3481            ));
3482        }
3483        self.inner.autosave.set(autosave);
3484        result
3485    }
3486
3487    #[cfg(not(target_arch = "wasm32"))]
3488    fn worker_discard_changes(&self) -> Result<(), DbError> {
3489        let autosave = self.inner.autosave.replace(false);
3490        *self.inner.dirty.borrow_mut() = DirtyState::default();
3491        self.participant_rollback();
3492        let result = self.refresh_resident();
3493        self.invalidate_relation_memos();
3494        if result.is_ok() {
3495            self.requery_all();
3496        }
3497        self.inner.autosave.set(autosave);
3498        result.map(|_| ()).map_err(|error| {
3499            DbError::new(
3500                DbErrorKind::Closed,
3501                format!("transaction cache recovery failed: {error}"),
3502            )
3503        })
3504    }
3505
3506    /// The fold, materialized: the smallest statement list that expresses `dirty`, in
3507    /// first-touch order. Row values come from the caches now; the change log carried which
3508    /// rows and columns moved, never their contents. Reads only; [`ModelContainer::flush`]
3509    /// executes the list in one transaction.
3510    ///
3511    /// With `capture`, the second answer is the same work as a change set, one entry per row
3512    /// a statement writes (the participant's view of the flush); without it, empty.
3513    #[allow(clippy::type_complexity)]
3514    fn fold(
3515        &self,
3516        dirty: &DirtyState,
3517        capture: bool,
3518    ) -> Result<(Vec<(String, Vec<Value>)>, Vec<RowChange>), DbError> {
3519        let tables = self.inner.tables.borrow();
3520        let mut stmts: Vec<(String, Vec<Value>)> = Vec::new();
3521        let mut changes: Vec<RowChange> = Vec::new();
3522        let authors_of = |store_id: u64, key: u64| -> Vec<&'static str> {
3523            dirty
3524                .authors
3525                .get(&(store_id, key))
3526                .cloned()
3527                .unwrap_or_default()
3528        };
3529
3530        // Wholesale rewrites first: upsert every resident row. (The cache is a working set;
3531        // rows outside it were never part of the rewrite, and deleting "the rest" would
3532        // delete data the rewrite never saw.)
3533        for store_id in &dirty.full {
3534            let Some(hooks) = tables.get(store_id) else {
3535                continue;
3536            };
3537            for (key, row) in (hooks.all_rows)() {
3538                stmts.push(upsert_stmt(hooks, &row));
3539                if capture {
3540                    let mut authors = dirty
3541                        .store_authors
3542                        .get(store_id)
3543                        .cloned()
3544                        .unwrap_or_default();
3545                    for tag in authors_of(*store_id, key) {
3546                        add_author(&mut authors, Some(tag));
3547                    }
3548                    changes.push(RowChange {
3549                        table: hooks.table,
3550                        key: (hooks.key_where)(key).1,
3551                        kind: RowChangeKind::Rewritten,
3552                        authors,
3553                    });
3554                }
3555            }
3556        }
3557
3558        // Per-row work, into batches that can merge. Deletes of one table merge into one
3559        // `IN`; updates merge when they write the same columns to the same values, which is
3560        // what a multi-selection edit produces ("set fill on twelve shapes").
3561        //
3562        // Deletes are held apart and emitted last. Dirty order interleaves rows by when each
3563        // first changed, which can put a parent row's delete between its children's detach
3564        // updates (the first detach dirties the parent through relation upkeep), and a
3565        // relation column's `ON DELETE CASCADE` fires per statement, `DEFERRABLE` or not, so
3566        // the mid-batch delete would take every still-attached child with it (found by Day
3567        // Sketch's ungroup: two members detached, one row left). With inserts and updates
3568        // first, a row still referencing a deleted parent at delete time is one the batch
3569        // never detached: an orphan the cascade rule is meant to take.
3570        let mut batches: Vec<Batch> = Vec::new();
3571        let mut deletes: Vec<Batch> = Vec::new();
3572        // (store, statement shape, a fingerprint of the `SET` values) → the batch it joins.
3573        let mut open: HashMap<(u64, String, u64), usize> = HashMap::new();
3574        let mut open_deletes: HashMap<(u64, String, u64), usize> = HashMap::new();
3575
3576        for id in &dirty.order {
3577            let Some(state) = dirty.rows.get(id) else {
3578                continue;
3579            };
3580            let (store_id, key) = *id;
3581            if dirty.full.contains(&store_id) {
3582                continue; // the resync above already covered it
3583            }
3584            let Some(hooks) = tables.get(&store_id) else {
3585                continue;
3586            };
3587            // A join row is addressed by a pair of columns, which no single-column `IN` can
3588            // express, so those keep one statement each.
3589            let batchable = hooks.key_cols.len() == 1;
3590            let mut capture_as = |kind: RowChangeKind| {
3591                if capture {
3592                    changes.push(RowChange {
3593                        table: hooks.table,
3594                        key: (hooks.key_where)(key).1,
3595                        kind,
3596                        authors: authors_of(store_id, key),
3597                    });
3598                }
3599            };
3600            match state {
3601                DirtyRow::Delete => {
3602                    capture_as(RowChangeKind::Deleted);
3603                    if !batchable {
3604                        let (clause, params) = (hooks.key_where)(key);
3605                        deletes.push(Batch::Single(
3606                            format!("DELETE FROM {} WHERE {clause}", hooks.name.sql()),
3607                            params,
3608                        ));
3609                        continue;
3610                    }
3611                    join_batch(&mut deletes, &mut open_deletes, store_id, None, key);
3612                }
3613                DirtyRow::Insert => {
3614                    if let Some(row) = (hooks.row_for)(key) {
3615                        let (sql, params) = upsert_stmt(hooks, &row);
3616                        batches.push(Batch::Single(sql, params));
3617                        capture_as(RowChangeKind::Inserted);
3618                    }
3619                }
3620                DirtyRow::Update(cols) => {
3621                    let Some(row) = (hooks.row_for)(key) else {
3622                        continue; // deleted since; its Delete entry handles it
3623                    };
3624                    if cols.is_empty() {
3625                        // A row-level replacement named no columns; write them all.
3626                        let (sql, params) = upsert_stmt(hooks, &row);
3627                        batches.push(Batch::Single(sql, params));
3628                        capture_as(RowChangeKind::Rewritten);
3629                        continue;
3630                    }
3631                    let mut sets: Vec<String> = Vec::with_capacity(cols.len());
3632                    let mut set_params: Vec<Value> = Vec::with_capacity(cols.len());
3633                    for c in cols {
3634                        let Some(i) = hooks.fields.iter().position(|n| n == c) else {
3635                            continue; // a transient field's label, never a column
3636                        };
3637                        sets.push(format!("{} = ?", ident(&hooks.columns[i])));
3638                        set_params.push(row.get(i));
3639                    }
3640                    if set_params.is_empty() {
3641                        continue; // only transient fields changed
3642                    }
3643                    if capture {
3644                        let written: Vec<&'static str> = cols
3645                            .iter()
3646                            .filter_map(|c| hooks.fields.iter().position(|n| n == c))
3647                            .map(|i| hooks.column_names[i])
3648                            .collect();
3649                        capture_as(RowChangeKind::Updated(written));
3650                    }
3651                    let clause = sets.join(", ");
3652                    if !batchable {
3653                        let (where_clause, mut where_params) = (hooks.key_where)(key);
3654                        let mut params = set_params;
3655                        params.append(&mut where_params);
3656                        batches.push(Batch::Single(
3657                            format!(
3658                                "UPDATE {} SET {clause} WHERE {where_clause}",
3659                                hooks.name.sql()
3660                            ),
3661                            params,
3662                        ));
3663                        continue;
3664                    }
3665                    join_batch(
3666                        &mut batches,
3667                        &mut open,
3668                        store_id,
3669                        Some((clause, set_params)),
3670                        key,
3671                    );
3672                }
3673            }
3674        }
3675
3676        for batch in batches.into_iter().chain(deletes) {
3677            match batch {
3678                Batch::Single(sql, params) => stmts.push((sql, params)),
3679                Batch::Keyed { store, set, keys } => {
3680                    let Some(hooks) = tables.get(&store) else {
3681                        continue;
3682                    };
3683                    materialize(hooks, set, &keys, &mut stmts);
3684                }
3685            }
3686        }
3687        Ok((stmts, changes))
3688    }
3689
3690    // --- maintenance (docs/persistence.md) ---------------------------------------------------
3691
3692    /// `VACUUM INTO`: a transactionally consistent, already-compacted snapshot, taken while
3693    /// the app keeps writing. Restore is an open, not a verb.
3694    pub fn backup_to(&self, path: &Path) -> Result<(), DbError> {
3695        self.save()?;
3696        self.conn().execute(
3697            "VACUUM INTO ?",
3698            &[Value::Text(path.to_string_lossy().into_owned())],
3699        )?;
3700        Ok(())
3701    }
3702
3703    /// `PRAGMA integrity_check` and `PRAGMA foreign_key_check`, parsed: an empty list means
3704    /// the database is sound. A foreign-key violation reads
3705    /// `foreign key: <child> row <rowid> references missing <parent>`; the structured form is
3706    /// [`ModelContainer::foreign_key_check`].
3707    pub fn integrity_check(&self) -> Result<Vec<String>, DbError> {
3708        let mut findings = Vec::new();
3709        self.conn()
3710            .query("PRAGMA integrity_check", &[], &mut |row| {
3711                if let Ok(t) = row.get(0).as_text()
3712                    && t != "ok"
3713                {
3714                    findings.push(t.to_string());
3715                }
3716            })?;
3717        for v in self.foreign_key_check()? {
3718            findings.push(v.to_string());
3719        }
3720        Ok(findings)
3721    }
3722
3723    /// `PRAGMA foreign_key_check`, parsed: every row whose foreign key names a parent that
3724    /// does not exist. Empty when every reference resolves. Runs on the whole file, which is
3725    /// what a migration's post-check and a settings-page validation both want.
3726    pub fn foreign_key_check(&self) -> Result<Vec<ForeignKeyViolation>, DbError> {
3727        Self::foreign_key_check_on(self.conn().as_mut())
3728    }
3729
3730    fn foreign_key_check_on(
3731        conn: &mut dyn SqliteConnection,
3732    ) -> Result<Vec<ForeignKeyViolation>, DbError> {
3733        let mut violations = Vec::new();
3734        conn.query("PRAGMA foreign_key_check", &[], &mut |row| {
3735            violations.push(ForeignKeyViolation {
3736                table: row.get(0).as_text().unwrap_or("").to_string(),
3737                rowid: row.get(1).as_int().ok(),
3738                parent: row.get(2).as_text().unwrap_or("").to_string(),
3739                constraint: row.get(3).as_int().unwrap_or(0),
3740            });
3741        })?;
3742        Ok(violations)
3743    }
3744
3745    /// Fold the WAL into the main file (`PRAGMA wal_checkpoint(TRUNCATE)`), before the OS
3746    /// copies it whole (device backup). The engine's own answer comes back: whether a reader
3747    /// kept the checkpoint from completing, and how much of the log it folded.
3748    pub fn checkpoint(&self) -> Result<Checkpoint, DbError> {
3749        let mut result = Checkpoint::default();
3750        self.conn()
3751            .query("PRAGMA wal_checkpoint(TRUNCATE)", &[], &mut |row| {
3752                result = Checkpoint {
3753                    busy: row.get(0).as_int().unwrap_or(0) != 0,
3754                    log_frames: row.get(1).as_int().unwrap_or(-1),
3755                    checkpointed_frames: row.get(2).as_int().unwrap_or(-1),
3756                };
3757            })?;
3758        Ok(result)
3759    }
3760
3761    pub fn vacuum(&self) -> Result<(), DbError> {
3762        self.save()?;
3763        self.conn().execute("VACUUM", &[]).map(|_| ())
3764    }
3765
3766    /// Re-encrypt in place under a new key (`PRAGMA rekey`; feature `cipher`).
3767    #[cfg(feature = "cipher")]
3768    pub fn rekey(&self, new: Secret) -> Result<(), DbError> {
3769        self.save()?;
3770        let quoted = new.reveal().replace('\'', "''");
3771        // The PRAGMA answers with a status row; consume it as a query.
3772        self.conn()
3773            .query(&format!("PRAGMA rekey = '{quoted}'"), &[], &mut |_| {})
3774    }
3775
3776    /// Write an encrypted copy at `path` through SQLCipher's conversion path (`ATTACH` +
3777    /// `sqlcipher_export`); plaintext↔encrypted cannot happen in place.
3778    #[cfg(feature = "cipher")]
3779    pub fn encrypt_to(&self, path: &Path, key: Secret) -> Result<(), DbError> {
3780        self.export_to(path, Some(key))
3781    }
3782
3783    /// Write a plaintext copy at `path`, same mechanism with an empty key.
3784    #[cfg(feature = "cipher")]
3785    pub fn decrypt_to(&self, path: &Path) -> Result<(), DbError> {
3786        self.export_to(path, None)
3787    }
3788
3789    #[cfg(feature = "cipher")]
3790    fn export_to(&self, path: &Path, key: Option<Secret>) -> Result<(), DbError> {
3791        self.save()?;
3792        let quoted_path = path.to_string_lossy().replace('\'', "''");
3793        let quoted_key = key
3794            .as_ref()
3795            .map(|k| k.reveal().replace('\'', "''"))
3796            .unwrap_or_default();
3797        let mut conn = self.conn();
3798        conn.execute(
3799            &format!("ATTACH DATABASE '{quoted_path}' AS day_export KEY '{quoted_key}'"),
3800            &[],
3801        )?;
3802        conn.query("SELECT sqlcipher_export('day_export')", &[], &mut |_| {})
3803            .and_then(|_| conn.execute("DETACH DATABASE day_export", &[]).map(|_| ()))
3804    }
3805
3806    /// Total size in bytes (page_count × page_size).
3807    pub fn size_bytes(&self) -> Result<u64, DbError> {
3808        let mut pages = 0i64;
3809        let mut page_size = 0i64;
3810        self.conn().query("PRAGMA page_count", &[], &mut |row| {
3811            pages = row.get(0).as_int().unwrap_or(0);
3812        })?;
3813        self.conn().query("PRAGMA page_size", &[], &mut |row| {
3814            page_size = row.get(0).as_int().unwrap_or(0);
3815        })?;
3816        Ok((pages * page_size).max(0) as u64)
3817    }
3818}
3819
3820/// The multi-key `WHERE` builder for a hooks entry: `key IN (…)` for single-column keys, an OR
3821/// list of pair clauses for join tables.
3822/// `a, b, c` as a quoted SELECT list or insert column list.
3823pub(crate) fn column_list<S: AsRef<str>>(columns: &[S]) -> String {
3824    columns
3825        .iter()
3826        .map(|c| ident(c.as_ref()))
3827        .collect::<Vec<_>>()
3828        .join(", ")
3829}
3830
3831/// `prefix.a, prefix.b`: the same list through a table alias or a trigger's `new`/`old`.
3832pub(crate) fn prefixed_list<S: AsRef<str>>(prefix: &str, columns: &[S]) -> String {
3833    columns
3834        .iter()
3835        .map(|c| format!("{prefix}.{}", ident(c.as_ref())))
3836        .collect::<Vec<_>>()
3837        .join(", ")
3838}
3839
3840/// The two halves of an FTS update trigger's gate: the `UPDATE OF` column list (the key and
3841/// every indexed column, once each) and the `WHEN` expression that is true only if one of
3842/// them actually changed (`IS NOT`, so a NULL on either side compares like any value).
3843fn fts_update_guard(key: &str, fts_columns: &[&str]) -> (String, String) {
3844    let mut cols: Vec<&str> = vec![key];
3845    cols.extend(fts_columns.iter().copied().filter(|c| *c != key));
3846    let update_of = column_list(&cols);
3847    let changed = cols
3848        .iter()
3849        .map(|c| {
3850            let c = ident(c);
3851            format!("new.{c} IS NOT old.{c}")
3852        })
3853        .collect::<Vec<_>>()
3854        .join(" OR ");
3855    (update_of, changed)
3856}
3857
3858// ---------------------------------------------------------------------------
3859// Schema comparison: the declaration against the file
3860// ---------------------------------------------------------------------------
3861
3862/// One column as the model declares it, with the relation-contributed foreign key resolved.
3863struct DeclaredColumn {
3864    def: ColumnDef,
3865    /// Position in `Model::COLUMNS`, which is also the position in `Model::default_row()`.
3866    index: usize,
3867    pk: bool,
3868    fk: Option<DeclaredFk>,
3869}
3870
3871/// A declared relation's foreign key on a child column.
3872struct DeclaredFk {
3873    parent_sql: String,
3874    parent_bare: &'static str,
3875    parent_key: &'static str,
3876    rule: DeleteRule,
3877}
3878
3879impl DeclaredFk {
3880    /// The column constraint, deferred so within-transaction statement order (a cascade's
3881    /// children, an undo's re-inserts) never trips it.
3882    fn clause(&self) -> String {
3883        format!(
3884            " REFERENCES {}({}) ON DELETE {} DEFERRABLE INITIALLY DEFERRED",
3885            self.parent_sql,
3886            ident(self.parent_key),
3887            self.rule.sql()
3888        )
3889    }
3890}
3891
3892/// An index the container owns, as it is declared.
3893struct DeclaredIndex {
3894    name: String,
3895    columns: Vec<String>,
3896    unique: bool,
3897}
3898
3899/// A column as `PRAGMA table_info` and `PRAGMA foreign_key_list` report it.
3900struct FileColumn {
3901    name: String,
3902    /// Upper-cased declared type (`INTEGER`, `TEXT`, …).
3903    decl_type: String,
3904    not_null: bool,
3905    pk: bool,
3906    fk: Option<FileFk>,
3907}
3908
3909struct FileFk {
3910    parent: String,
3911    parent_key: String,
3912    on_delete: String,
3913}
3914
3915/// An index as `PRAGMA index_list` and `index_info` report it. `origin` is `c` for a
3916/// `CREATE INDEX`, `u` for an inline `UNIQUE`, `pk` for the primary key's.
3917struct FileIndex {
3918    name: String,
3919    unique: bool,
3920    origin: String,
3921    columns: Vec<String>,
3922}
3923
3924struct FileSchema {
3925    columns: Vec<FileColumn>,
3926    indexes: Vec<FileIndex>,
3927}
3928
3929/// What `lightweight_migrate` does in place: positions in `COLUMNS` to add, file columns to
3930/// drop, Day-owned index names to drop, and declared-unique columns that need a unique index.
3931#[derive(Default, Clone)]
3932struct LightweightPlan {
3933    add: Vec<usize>,
3934    drop: Vec<String>,
3935    drop_indexes: Vec<String>,
3936    unique_indexes: Vec<&'static str>,
3937}
3938
3939/// The comparison's verdict for an existing table.
3940enum SchemaDelta {
3941    Lightweight(LightweightPlan),
3942    Rebuild,
3943}
3944
3945/// Whether an index name is one the container writes for `table` (`day_idx_`, `day_uidx_`).
3946fn is_day_index(name: &str, table: &str) -> bool {
3947    name.starts_with(&format!("day_idx_{table}_"))
3948        || name.starts_with(&format!("day_uidx_{table}_"))
3949}
3950
3951/// `PRAGMA [schema.]name("arg")`.
3952fn pragma_sql(schema: Option<&str>, name: &str, arg: &str) -> String {
3953    match schema {
3954        Some(s) => format!("PRAGMA {}.{name}({})", ident(s), ident(arg)),
3955        None => format!("PRAGMA {name}({})", ident(arg)),
3956    }
3957}
3958
3959/// A [`Value`] as an SQL literal, for the places a bound parameter cannot go (a column's
3960/// `DEFAULT`, a copy's `SELECT` list): text single-quoted with `'` doubled, a blob as
3961/// `X'…'`, numbers as themselves. A non-finite real has no literal and is refused.
3962fn sql_literal(v: &Value) -> Result<String, DbError> {
3963    Ok(match v {
3964        Value::Null => "NULL".into(),
3965        Value::Int(i) => i.to_string(),
3966        Value::Real(r) if r.is_finite() => format!("{r:?}"),
3967        Value::Real(r) => {
3968            return Err(DbError::new(
3969                DbErrorKind::Schema,
3970                format!("{r} has no SQL literal to use as a column default"),
3971            ));
3972        }
3973        Value::Text(t) => format!("'{}'", t.replace('\'', "''")),
3974        Value::Blob(b) => {
3975            let mut s = String::with_capacity(b.len() * 2 + 3);
3976            s.push_str("X'");
3977            for byte in b {
3978                s.push_str(&format!("{byte:02x}"));
3979            }
3980            s.push('\'');
3981            s
3982        }
3983    })
3984}
3985
3986fn keys_where_of(hooks: &TableHooks) -> KeysWhere {
3987    if hooks.key_cols.len() == 1 {
3988        let col = ident(&hooks.key_cols[0]);
3989        Rc::new(move |keys: &[u64]| {
3990            let marks = vec!["?"; keys.len()].join(", ");
3991            (
3992                format!("{col} IN ({marks})"),
3993                keys.iter().map(|k| key_param(*k)).collect(),
3994            )
3995        })
3996    } else {
3997        let key_where = hooks.key_where.clone();
3998        Rc::new(move |keys: &[u64]| {
3999            let mut clauses = Vec::with_capacity(keys.len());
4000            let mut params = Vec::with_capacity(keys.len() * 2);
4001            for k in keys {
4002                let (c, mut p) = key_where(*k);
4003                clauses.push(format!("({c})"));
4004                params.append(&mut p);
4005            }
4006            (clauses.join(" OR "), params)
4007        })
4008    }
4009}
4010
4011/// One statement the fold will write, before the mergeable ones have merged.
4012enum Batch {
4013    /// Stands alone: an upsert (its values are the row's own), or a row addressed by a
4014    /// composite key, which no single-column `IN` can express.
4015    Single(String, Vec<Value>),
4016    /// Same table, same operation, same values: one statement per chunk of keys.
4017    Keyed {
4018        store: u64,
4019        /// The `SET` clause and its parameters for an update; `None` for a delete.
4020        set: Option<(String, Vec<Value>)>,
4021        keys: Vec<u64>,
4022    },
4023}
4024
4025/// SQLite's conservative bound-parameter ceiling. Modern builds allow far more, but a
4026/// `system` engine may be an older one, and chunking costs nothing to write.
4027const MAX_BOUND_PARAMS: usize = 900;
4028
4029/// Add `key` to the batch sharing this table, clause and values, or open a new one.
4030///
4031/// The slot key separates a delete from an update (their clauses differ), and separates
4032/// updates writing different values, which must stay different statements. Grouping is by
4033/// hash so a large flush stays linear; the values are compared before merging, so a hash
4034/// collision costs a second statement rather than a wrong one.
4035fn join_batch(
4036    batches: &mut Vec<Batch>,
4037    open: &mut HashMap<(u64, String, u64), usize>,
4038    store: u64,
4039    set: Option<(String, Vec<Value>)>,
4040    key: u64,
4041) {
4042    let clause = set.as_ref().map(|(c, _)| c.clone()).unwrap_or_default();
4043    let print = set.as_ref().map(|(_, p)| fingerprint(p)).unwrap_or(0);
4044    let slot = (store, clause, print);
4045    if let Some(&i) = open.get(&slot)
4046        && let Batch::Keyed {
4047            set: existing,
4048            keys,
4049            ..
4050        } = &mut batches[i]
4051        && existing.as_ref().map(|(_, p)| p.as_slice()) == set.as_ref().map(|(_, p)| p.as_slice())
4052    {
4053        keys.push(key);
4054        return;
4055    }
4056    open.insert(slot, batches.len());
4057    batches.push(Batch::Keyed {
4058        store,
4059        set,
4060        keys: vec![key],
4061    });
4062}
4063
4064/// A cheap order-sensitive hash of bound values, for grouping identical updates. Equality is
4065/// still checked before two rows share a statement; this only decides who to compare with.
4066fn fingerprint(values: &[Value]) -> u64 {
4067    let mut h: u64 = 0xcbf2_9ce4_8422_2325;
4068    let mut eat = |bytes: &[u8]| {
4069        for b in bytes {
4070            h ^= *b as u64;
4071            h = h.wrapping_mul(0x1000_0000_01b3);
4072        }
4073    };
4074    for v in values {
4075        match v {
4076            Value::Null => eat(&[0]),
4077            Value::Int(i) => {
4078                eat(&[1]);
4079                eat(&i.to_le_bytes());
4080            }
4081            // Bit pattern, not numeric value: two values that hash apart are only compared
4082            // separately, never merged wrongly.
4083            Value::Real(r) => {
4084                eat(&[2]);
4085                eat(&r.to_bits().to_le_bytes());
4086            }
4087            Value::Text(t) => {
4088                eat(&[3]);
4089                eat(t.as_bytes());
4090            }
4091            Value::Blob(b) => {
4092                eat(&[4]);
4093                eat(b);
4094            }
4095        }
4096    }
4097    h
4098}
4099
4100/// Write a batch out: the familiar `= ?` form for one key, chunked `IN` statements for more.
4101fn materialize(
4102    hooks: &TableHooks,
4103    set: Option<(String, Vec<Value>)>,
4104    keys: &[u64],
4105    out: &mut Vec<(String, Vec<Value>)>,
4106) {
4107    // One key keeps the `= ?` form: it is the common case by far, it reads plainly in a
4108    // trace, and it is one more statement shape the cache can hold.
4109    let table = hooks.name.sql();
4110    if keys.len() == 1 {
4111        let (where_clause, where_params) = (hooks.key_where)(keys[0]);
4112        match set {
4113            Some((clause, mut params)) => {
4114                params.extend(where_params);
4115                out.push((
4116                    format!("UPDATE {table} SET {clause} WHERE {where_clause}"),
4117                    params,
4118                ));
4119            }
4120            None => out.push((
4121                format!("DELETE FROM {table} WHERE {where_clause}"),
4122                where_params,
4123            )),
4124        }
4125        return;
4126    }
4127
4128    let key_col = ident(&hooks.key_cols[0]);
4129    let set_len = set.as_ref().map(|(_, p)| p.len()).unwrap_or(0);
4130    let cap = MAX_BOUND_PARAMS.saturating_sub(set_len).max(1);
4131    for chunk in keys.chunks(cap) {
4132        let mut params: Vec<Value> = Vec::with_capacity(set_len + chunk.len());
4133        if let Some((_, set_params)) = &set {
4134            params.extend(set_params.iter().cloned());
4135        }
4136        params.extend(chunk.iter().map(|k| key_param(*k)));
4137        let marks = vec!["?"; chunk.len()].join(", ");
4138        let sql = match &set {
4139            Some((clause, _)) => {
4140                format!("UPDATE {table} SET {clause} WHERE {key_col} IN ({marks})")
4141            }
4142            None => format!("DELETE FROM {table} WHERE {key_col} IN ({marks})"),
4143        };
4144        out.push((sql, params));
4145    }
4146}
4147
4148fn upsert_stmt(hooks: &TableHooks, row: &[Value]) -> (String, Vec<Value>) {
4149    let cols = column_list(&hooks.columns);
4150    let marks = vec!["?"; hooks.columns.len()].join(", ");
4151    // A true upsert, not `INSERT OR REPLACE`: the conflict path fires `UPDATE` triggers, which
4152    // is what keeps the generated FTS/R*Tree shadows in sync (`OR REPLACE`'s implicit delete
4153    // skips `AFTER DELETE` triggers unless recursive_triggers is on).
4154    let sets = hooks
4155        .columns
4156        .iter()
4157        .filter(|c| !hooks.key_cols.contains(c))
4158        .map(|c| format!("{0} = excluded.{0}", ident(c)))
4159        .collect::<Vec<_>>()
4160        .join(", ");
4161    let conflict = if sets.is_empty() {
4162        "DO NOTHING".to_string()
4163    } else {
4164        format!("DO UPDATE SET {sets}")
4165    };
4166    (
4167        format!(
4168            "INSERT INTO {} ({cols}) VALUES ({marks}) ON CONFLICT({}) {conflict}",
4169            hooks.name.sql(),
4170            column_list(&hooks.key_cols)
4171        ),
4172        row.to_vec(),
4173    )
4174}
4175
4176impl Drop for ContainerInner {
4177    fn drop(&mut self) {
4178        if let Some(sink) = self.sink.take() {
4179            day_model::remove_change_sink(sink);
4180        }
4181    }
4182}
4183
4184// ---------------------------------------------------------------------------
4185// Live queries (docs/persistence.md; queries.rs holds the compiler and the diff)
4186// ---------------------------------------------------------------------------
4187
4188/// What a query's consumers have not yet seen. `Deltas` can animate a list row by row;
4189/// `Reload` means the whole set moved (a fetch swap, a tangled requery) and a reload is
4190/// the accurate answer.
4191#[derive(Clone, Debug, PartialEq)]
4192pub enum QueryEvents {
4193    None,
4194    Deltas(Vec<Delta>),
4195    Reload,
4196}
4197
4198impl QueryEvents {
4199    fn push(&mut self, deltas: &[Delta]) {
4200        match self {
4201            QueryEvents::None => *self = QueryEvents::Deltas(deltas.to_vec()),
4202            QueryEvents::Deltas(d) => d.extend_from_slice(deltas),
4203            QueryEvents::Reload => {}
4204        }
4205    }
4206    fn reload(&mut self) {
4207        *self = QueryEvents::Reload;
4208    }
4209}
4210
4211/// What one relation crossing subscribes a query to, precomputed at install (and at every
4212/// fetch swap) so the change sink's staleness check is a few comparisons.
4213struct RelWatch {
4214    /// The store on the far side of the crossing (the related table's cache).
4215    store: u64,
4216    /// The join membership store, when the crossing is a many-to-many; every link, unlink
4217    /// and reposition there is a membership change.
4218    join_store: Option<u64>,
4219    /// Field names on the far store whose write is a membership change (the foreign key, the
4220    /// order field) even though no predicate reads them.
4221    fields: Vec<&'static str>,
4222    /// Column names of the far table the inner predicate reads.
4223    columns: Vec<&'static str>,
4224}
4225
4226struct QueryState {
4227    store_id: u64,
4228    table: &'static str,
4229    /// The relation subscriptions, recomputed when the fetch is swapped.
4230    watches: RefCell<Vec<RelWatch>>,
4231    /// Extra local columns whose write is a membership change (a `One` column crossed by the
4232    /// query's own table).
4233    local_extra: RefCell<Vec<&'static str>>,
4234    set: RefCell<ResultSet>,
4235    /// Bumped whenever the result set changes; `ids()`/`count()` track it.
4236    version: day_reactive::Signal<u64>,
4237    pending: RefCell<QueryEvents>,
4238    /// A dependency-touching change arrived; the answer re-derives after the flush.
4239    needs_sql: Cell<bool>,
4240    // Reading the last committed result must not consume the post-commit invalidation.
4241    pending_commit: Cell<bool>,
4242    error: day_reactive::Signal<Option<DbError>>,
4243    /// `query_raw` only: the statement and the tables whose flush re-runs it.
4244    raw: Option<RawQuery>,
4245    /// A count-shaped query: `count` is the answer, the id set stays empty, and requeries
4246    /// run `SELECT COUNT(*)`, the badge query, O(1) memory at any result size.
4247    count_only: bool,
4248    count: Cell<usize>,
4249}
4250
4251struct RawQuery {
4252    sql: String,
4253    params: Vec<Value>,
4254    tables: Vec<String>,
4255}
4256
4257/// A live, typed result set over one model's table: ids only, answered by the engine and
4258/// kept current by dependency-gated requeries. Rows stay lazy: fault the ones you show
4259/// ([`Query::materialize`], or a list source doing it for you). Clone shares the same set.
4260pub struct Query<M: Model> {
4261    state: Rc<QueryState>,
4262    container: ModelContainer,
4263    _p: std::marker::PhantomData<fn() -> M>,
4264}
4265
4266impl<M: Model> Clone for Query<M> {
4267    fn clone(&self) -> Self {
4268        Query {
4269            state: self.state.clone(),
4270            container: self.container.clone(),
4271            _p: std::marker::PhantomData,
4272        }
4273    }
4274}
4275
4276impl<M: Model> Query<M> {
4277    /// This query's last read failure. A failed read preserves its previous result set.
4278    pub fn error(&self) -> day_reactive::Signal<Option<DbError>> {
4279        self.state.error
4280    }
4281
4282    /// Explicitly retry a query after a transient read failure (also useful after external SQL).
4283    pub fn refresh(&self) {
4284        self.state.needs_sql.set(true);
4285        self.refresh_if_stale();
4286    }
4287
4288    /// Checked form of `ids`, for imports and commands that must not mistake failure for absence.
4289    pub fn try_ids(&self) -> Result<Vec<ModelId<M>>, DbError> {
4290        let ids = self.ids();
4291        match self.state.error.get() {
4292            Some(e) => Err(e),
4293            None => Ok(ids),
4294        }
4295    }
4296
4297    /// A checked snapshot for non-UI work, fetched in batches. UI lists should use the query
4298    /// itself as their row source so they only materialize the visible working set.
4299    pub fn try_collect(&self) -> Result<Vec<M>, DbError> {
4300        let ids = self.try_ids()?;
4301        let mut rows = Vec::with_capacity(ids.len());
4302        for chunk in ids.chunks(64) {
4303            self.container
4304                .ensure_resident::<M>(&chunk.iter().map(|id| id.handle()).collect::<Vec<_>>())?;
4305            for id in chunk {
4306                if let Some(row) = self.container.try_get::<M>(*id)?
4307                    && let Some(value) = row.with_value_untracked(|v| v.cloned())
4308                {
4309                    rows.push(value);
4310                }
4311            }
4312        }
4313        Ok(rows)
4314    }
4315
4316    /// The result ids, typed and in query order, as a tracked read: the caller re-runs when
4317    /// the set changes, and only then.
4318    pub fn ids(&self) -> Vec<ModelId<M>> {
4319        self.refresh_if_stale();
4320        let _ = self.state.version.get();
4321        self.state
4322            .set
4323            .borrow()
4324            .ids()
4325            .iter()
4326            .map(|h| ModelId::from_handle(*h))
4327            .collect()
4328    }
4329
4330    /// The result count, tracked like [`Query::ids`].
4331    pub fn count(&self) -> usize {
4332        self.refresh_if_stale();
4333        let _ = self.state.version.get();
4334        self.state.set.borrow().ids().len()
4335    }
4336
4337    /// The first result, tracked.
4338    pub fn first(&self) -> Option<ModelId<M>> {
4339        self.refresh_if_stale();
4340        let _ = self.state.version.get();
4341        self.state
4342            .set
4343            .borrow()
4344            .ids()
4345            .first()
4346            .map(|h| ModelId::from_handle(*h))
4347    }
4348
4349    /// Tracked membership test.
4350    pub fn contains(&self, id: impl Into<ModelId<M>>) -> bool {
4351        self.refresh_if_stale();
4352        let _ = self.state.version.get();
4353        self.state.set.borrow().ids().contains(&id.into().handle())
4354    }
4355
4356    /// Untracked snapshot: reactively silent, but still current (pending staleness settles
4357    /// first, like every other read).
4358    pub fn ids_untracked(&self) -> Vec<ModelId<M>> {
4359        self.refresh_if_stale();
4360        self.state
4361            .set
4362            .borrow()
4363            .ids()
4364            .iter()
4365            .map(|h| ModelId::from_handle(*h))
4366            .collect()
4367    }
4368
4369    /// Bring this window of the result into the cache: what a list binding calls for the
4370    /// rows it is about to show, batched into one `SELECT`.
4371    pub fn materialize(&self, range: std::ops::Range<usize>) {
4372        let keys: Vec<u64> = {
4373            let set = self.state.set.borrow();
4374            let ids = set.ids();
4375            let end = range.end.min(ids.len());
4376            let start = range.start.min(end);
4377            ids[start..end].to_vec()
4378        };
4379        if let Err(e) = self.container.ensure_resident::<M>(&keys) {
4380            self.container.inner.error.set(Some(e.to_string()));
4381        }
4382    }
4383
4384    /// Swap the fetch (a changed filter or sort): requeries and reloads consumers. No-op when
4385    /// equal to the current one, so a `query_fn` closure can re-run cheaply.
4386    pub fn set_fetch(&self, fetch: Fetch) {
4387        if *self.state.set.borrow().fetch() == fetch {
4388            return;
4389        }
4390        self.container.rewire_watches(&self.state, &fetch);
4391        {
4392            let mut set = self.state.set.borrow_mut();
4393            *set = ResultSet::new(fetch);
4394        }
4395        // Resolve once pending statements land; mid-turn the flush has not run yet.
4396        self.state.needs_sql.set(true);
4397        self.state
4398            .pending_commit
4399            .set(!self.container.inner.dirty.borrow().is_empty());
4400        self.refresh_if_stale();
4401        self.state.pending.borrow_mut().reload();
4402        self.bump();
4403    }
4404
4405    /// Drain what changed since the last call: the list source's feed. Pending staleness
4406    /// settles first, so a drain right after an edit already narrates it.
4407    pub fn take_events(&self) -> QueryEvents {
4408        self.refresh_if_stale();
4409        std::mem::replace(&mut *self.state.pending.borrow_mut(), QueryEvents::None)
4410    }
4411
4412    /// A dependency-touching change arrived this turn and the flush has not run yet: settle
4413    /// it now. With autosave the pending statements flush (which requeries everything stale);
4414    /// with autosave off the query re-derives against the last save; the file is the only
4415    /// thing that can answer it.
4416    fn refresh_if_stale(&self) {
4417        if !self.state.needs_sql.get() {
4418            return;
4419        }
4420        if self.container.inner.autosave.get() {
4421            let _ = self.container.save();
4422        }
4423        if self.state.needs_sql.replace(false) {
4424            self.container.requery(&self.state);
4425        }
4426    }
4427
4428    fn bump(&self) {
4429        self.state
4430            .version
4431            .set(self.state.version.get_untracked().wrapping_add(1));
4432    }
4433}
4434
4435/// Builder for a [`Query`]: `container.query::<Trip>().filter(…).sort(…).live()`.
4436pub struct QueryBuilder<'c, M: Model> {
4437    container: &'c ModelContainer,
4438    fetch: Fetch,
4439    _p: std::marker::PhantomData<fn() -> M>,
4440}
4441
4442impl<M: Model> QueryBuilder<'_, M> {
4443    pub fn filter(mut self, p: Pred) -> Self {
4444        self.fetch = self.fetch.filter(p);
4445        self
4446    }
4447    pub fn sort(mut self, s: Sort) -> Self {
4448        self.fetch = self.fetch.sort(s);
4449        self
4450    }
4451    pub fn limit(mut self, n: usize) -> Self {
4452        self.fetch = self.fetch.limit(n);
4453        self
4454    }
4455    /// Run the fetch and keep it live against the change log.
4456    pub fn live(self) -> Query<M> {
4457        self.container.install_query::<M>(self.fetch, None)
4458    }
4459
4460    /// Keep only the count live, the badge form: no id vector, one `SELECT COUNT(*)` per
4461    /// requery, the same dependency gating.
4462    pub fn live_count(self) -> CountQuery<M> {
4463        CountQuery {
4464            state: self.container.install_state::<M>(self.fetch, None, true),
4465            container: self.container.clone(),
4466            _p: std::marker::PhantomData,
4467        }
4468    }
4469}
4470
4471/// A live count over one model's table, the badge query: `SELECT COUNT(*)` behind the same
4472/// dependency-gated staleness as [`Query`], holding no id vector, so a count over a
4473/// million-row result costs O(1) memory. Clone shares the same state.
4474pub struct CountQuery<M: Model> {
4475    state: Rc<QueryState>,
4476    container: ModelContainer,
4477    _p: std::marker::PhantomData<fn() -> M>,
4478}
4479
4480impl<M: Model> Clone for CountQuery<M> {
4481    fn clone(&self) -> Self {
4482        CountQuery {
4483            state: self.state.clone(),
4484            container: self.container.clone(),
4485            _p: std::marker::PhantomData,
4486        }
4487    }
4488}
4489
4490impl<M: Model> CountQuery<M> {
4491    /// The last read error, independently of the last successful count.
4492    pub fn error(&self) -> day_reactive::Signal<Option<DbError>> {
4493        self.state.error
4494    }
4495    /// Checked count for operations that require a successful read.
4496    pub fn try_get(&self) -> Result<usize, DbError> {
4497        let n = self.get();
4498        match self.state.error.get() {
4499            Some(e) => Err(e),
4500            None => Ok(n),
4501        }
4502    }
4503    /// Retry after a transient read error.
4504    pub fn refresh(&self) {
4505        self.state.needs_sql.set(true);
4506        self.refresh_if_stale();
4507    }
4508
4509    /// The count, as a tracked read: the caller re-runs when it changes, and only then.
4510    pub fn get(&self) -> usize {
4511        self.refresh_if_stale();
4512        let _ = self.state.version.get();
4513        self.state.count.get()
4514    }
4515
4516    /// Untracked snapshot: reactively silent, still current.
4517    pub fn get_untracked(&self) -> usize {
4518        self.refresh_if_stale();
4519        self.state.count.get()
4520    }
4521
4522    /// Swap the fetch (a changed filter): re-counts. No-op when equal, so a `count_fn`
4523    /// closure can re-run cheaply.
4524    pub fn set_fetch(&self, fetch: Fetch) {
4525        if *self.state.set.borrow().fetch() == fetch {
4526            return;
4527        }
4528        self.container.rewire_watches(&self.state, &fetch);
4529        *self.state.set.borrow_mut() = ResultSet::new(fetch);
4530        self.state.needs_sql.set(true);
4531        self.state
4532            .pending_commit
4533            .set(!self.container.inner.dirty.borrow().is_empty());
4534        self.refresh_if_stale();
4535    }
4536
4537    fn refresh_if_stale(&self) {
4538        if !self.state.needs_sql.get() {
4539            return;
4540        }
4541        if self.container.inner.autosave.get() {
4542            let _ = self.container.save();
4543        }
4544        if self.state.needs_sql.replace(false) {
4545            self.container.requery(&self.state);
4546        }
4547    }
4548}
4549
4550/// A fallback row: the key plus the dependency columns the fallback `SELECT` carried.
4551struct FallbackRow<'a> {
4552    cols: &'a [&'static str],
4553    /// values[0] is the key; values[1..] align with `cols`.
4554    values: &'a [Value],
4555}
4556
4557impl RowView for FallbackRow<'_> {
4558    fn col(&self, c: &str) -> Option<Value> {
4559        self.cols
4560            .iter()
4561            .position(|n| *n == c)
4562            .and_then(|i| self.values.get(i + 1).cloned())
4563    }
4564}
4565
4566impl ModelContainer {
4567    /// Start a typed query over `M`'s rows. Panics (like [`ModelContainer::cache`]) if `M` is
4568    /// not in this container's `schema!`, a wiring bug worth stopping on.
4569    pub fn query<M: Model>(&self) -> QueryBuilder<'_, M> {
4570        let _ = self.cache::<M>();
4571        QueryBuilder {
4572            container: self,
4573            fetch: Fetch::new(),
4574            _p: std::marker::PhantomData,
4575        }
4576    }
4577
4578    /// The reactive-fetch form: `f` is a computation, so a query whose fetch depends on signals
4579    /// (a search term, a filter toggle) re-derives itself when they change.
4580    pub fn query_fn<M: Model>(&self, f: impl Fn() -> Fetch + 'static) -> Query<M> {
4581        let q = self.install_query::<M>(f(), None);
4582        let q2 = q.clone();
4583        day_reactive::bind(f, move |fetch| q2.set_fetch(fetch.clone()));
4584        q
4585    }
4586
4587    /// The reactive-fetch count: a badge whose fetch depends on signals (the "Today" cutoff,
4588    /// a filter toggle) re-counts itself when they change.
4589    pub fn count_fn<M: Model>(&self, f: impl Fn() -> Fetch + 'static) -> CountQuery<M> {
4590        let q = CountQuery::<M> {
4591            state: self.install_state::<M>(f(), None, true),
4592            container: self.clone(),
4593            _p: std::marker::PhantomData,
4594        };
4595        let q2 = q.clone();
4596        day_reactive::bind(f, move |fetch| q2.set_fetch(fetch.clone()));
4597        q
4598    }
4599
4600    /// Raw SQL, declared: a read-only SELECT returning ids, re-run whenever a flush touches
4601    /// one of the named tables. The price of the escape hatch is whole-query invalidation.
4602    pub fn query_raw<M: Model>(
4603        &self,
4604        sql: impl Into<String>,
4605        params: Vec<Value>,
4606        tables: &[&str],
4607    ) -> Query<M> {
4608        self.install_query::<M>(
4609            Fetch::new().filter(Pred::Raw(String::new(), Vec::new())),
4610            Some(RawQuery {
4611                sql: sql.into(),
4612                params,
4613                tables: tables.iter().map(|s| s.to_string()).collect(),
4614            }),
4615        )
4616    }
4617
4618    /// Opt into undo: One history over every store this container manages, `levels` deep.
4619    /// Undo/redo replay flows through the same change pipeline as the user's edits, so
4620    /// autosave writes the inverse statements and live queries hear rows come back. Call it
4621    /// once, after open; clear it on migration (`UndoStack::clear`). While a stack is
4622    /// installed, deletes materialize the rows they remove (a cascade included) so the
4623    /// history can restore them.
4624    pub fn undo(&self, levels: usize) -> day_model::UndoStack {
4625        let stack = day_model::UndoStack::new(levels);
4626        for hooks in self.inner.tables.borrow().values() {
4627            (hooks.watch_undo)(&stack);
4628        }
4629        stack
4630    }
4631
4632    /// The driver's connection, directly, for maintenance, imports, or an extension's
4633    /// statements. Pending changes flush first; writes made here bypass the change log, so
4634    /// call [`ModelContainer::rescan`] afterward if they touched Day's tables.
4635    ///
4636    /// # Panics
4637    /// Panics if the pending save fails. Use [`ModelContainer::try_with_connection`] when
4638    /// the caller needs to handle storage errors.
4639    pub fn with_connection<R>(&self, f: impl FnOnce(&mut dyn SqliteConnection) -> R) -> R {
4640        self.save().expect(
4641            "pending persistence save failed; use try_with_connection to handle storage errors",
4642        );
4643        f(self.conn().as_mut())
4644    }
4645
4646    /// Flush pending edits before accessing the connection. The closure is never run if
4647    /// that save fails; its own error is returned unchanged.
4648    pub fn try_with_connection<R>(
4649        &self,
4650        f: impl FnOnce(&mut dyn SqliteConnection) -> Result<R, DbError>,
4651    ) -> Result<R, DbError> {
4652        self.save()?;
4653        f(self.conn().as_mut())
4654    }
4655
4656    /// Re-read the resident rows from the file and re-run every query: the recovery from
4657    /// writes that bypassed the change log ([`ModelContainer::with_connection`], another
4658    /// process). O(working set + query results), never O(table).
4659    pub fn rescan(&self) -> Result<(), DbError> {
4660        self.save()?;
4661        self.refresh_resident()?;
4662        self.invalidate_relation_memos();
4663        self.requery_all();
4664        // The cache now equals the file; a later check_external should not re-merge for
4665        // whatever external commits this refresh already picked up.
4666        if self.inner.caps.external_changes {
4667            self.inner.data_version.set(self.file_data_version()?);
4668        }
4669        Ok(())
4670    }
4671
4672    /// Attach another SQLite file under `alias` and bring `schema`'s models (every one
4673    /// declared `external = "alias"`) into this container: their tables are read as they
4674    /// are, with no DDL, migration, or fingerprint, through the same caches, queries, and
4675    /// lists as the container's own. Writes to them fail at flush (the file is attached
4676    /// read-only where the engine honors `mode=ro`), which is the contract: the other
4677    /// database has its own owner.
4678    ///
4679    /// Calling it again with the same alias swaps the file: the old one is detached, every
4680    /// resident row of its models dropped, and every live query re-run: the shape a
4681    /// downloaded catalog update takes.
4682    pub fn attach_database(&self, alias: &str, path: &str, schema: Schema) -> Result<(), DbError> {
4683        if alias.is_empty() || !alias.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') {
4684            return Err(DbError::new(
4685                DbErrorKind::Unsupported,
4686                format!("attach_database: {alias:?} is not a plain SQL schema name"),
4687            ));
4688        }
4689        let held = self.inner.attached.borrow().iter().any(|a| a == alias);
4690        if held {
4691            self.detach_database(alias)?;
4692        }
4693        // Flush first: a DETACH/ATTACH pair inside an open write transaction is refused.
4694        if !self.inner.dirty.borrow().is_empty() {
4695            self.save()?;
4696        }
4697        // `file:` URIs carry the read-only mode; a driver without URI filenames (the web
4698        // worker) takes the bare path and relies on nobody writing.
4699        let uri = if path.starts_with("file:") || path.contains("://") {
4700            path.to_string()
4701        } else {
4702            format!("file:{}?mode=ro", path.replace('?', "%3F"))
4703        };
4704        let attach = |target: &str| -> Result<(), DbError> {
4705            self.conn().execute(
4706                &format!("ATTACH DATABASE ? AS {}", ident(alias)),
4707                &[Value::Text(target.to_string())],
4708            )?;
4709            Ok(())
4710        };
4711        if let Err(uri_err) = attach(&uri) {
4712            attach(path).map_err(|_| uri_err)?;
4713        }
4714        self.inner.attached.borrow_mut().push(alias.to_string());
4715        let Schema {
4716            installers, wirers, ..
4717        } = schema;
4718        for install in installers {
4719            install(self)?;
4720        }
4721        for wire in wirers {
4722            wire(self)?;
4723        }
4724        self.rescan()
4725    }
4726
4727    /// Detach a database [`ModelContainer::attach_database`] holds, dropping every resident
4728    /// row of its models. Their stores stay registered (empty), so a query over them answers
4729    /// nothing until the alias is attached again.
4730    pub fn detach_database(&self, alias: &str) -> Result<(), DbError> {
4731        let position = self.inner.attached.borrow().iter().position(|a| a == alias);
4732        let Some(position) = position else {
4733            return Ok(());
4734        };
4735        if !self.inner.dirty.borrow().is_empty() {
4736            self.save()?;
4737        }
4738        type Doomed = Vec<(Rc<dyn Fn() -> Vec<u64>>, EvictFn)>;
4739        let doomed: Doomed = self
4740            .inner
4741            .tables
4742            .borrow()
4743            .values()
4744            .filter(|h| h.name.schema == Some(alias))
4745            .map(|h| (h.resident_keys.clone(), h.evict.clone()))
4746            .collect();
4747        for (keys, evict) in &doomed {
4748            let resident = keys();
4749            let _ = evict(&resident, resident.len());
4750        }
4751        self.conn()
4752            .execute(&format!("DETACH DATABASE {}", ident(alias)), &[])?;
4753        self.inner.attached.borrow_mut().remove(position);
4754        self.rescan()
4755    }
4756
4757    /// Look for other connections' committed writes (another process, a sync engine, a CLI)
4758    /// and merge what changed. Detection is one `PRAGMA data_version` (the counter moves only
4759    /// when another connection commits, never for this one's own writes), so this is cheap
4760    /// enough to wire to app foreground, window focus, or a timer.
4761    ///
4762    /// When the counter moved, pending local edits flush first; then the resident rows are
4763    /// re-read and diffed: changed fields announce per column, disappearances take the
4764    /// structural path, every live query re-derives, all authored
4765    /// [`ModelContainer::EXTERNAL_AUTHOR`], so the autosave fold declines the echo and an
4766    /// installed undo stack skips them (another author's writes are not the user's history).
4767    /// Rows that arrived elsewhere become visible through the re-run queries and fault in
4768    /// like any other row. A row another connection rewrote arrives whole, so its
4769    /// `#[model(transient)]` fields reset to their defaults, exactly as at fault.
4770    ///
4771    /// Returns whether anything arrived. On a driver without detection
4772    /// ([`Capabilities::external_changes`]) this is `Ok(false)`; writes made
4773    /// through [`ModelContainer::with_connection`] are this connection's own and stay
4774    /// [`ModelContainer::rescan`]'s job.
4775    pub fn check_external(&self) -> Result<bool, DbError> {
4776        if !self.inner.caps.external_changes {
4777            return Ok(false);
4778        }
4779        let Some(current) = self.file_data_version()? else {
4780            return Ok(false);
4781        };
4782        if self.inner.data_version.get() == Some(current) {
4783            return Ok(false);
4784        }
4785        // Local edits flush first, so the diff compares the file against a cache with nothing
4786        // pending: an unflushed local edit must not read as the other side's deletion.
4787        self.save()?;
4788        self.refresh_resident()?;
4789        self.invalidate_relation_memos();
4790        self.requery_all();
4791        // A commit matters even when it inserted only nonresident rows: count/projection
4792        // observers still need invalidation. Advance only after a successful merge so errors
4793        // remain retryable.
4794        self.inner.data_version.set(Some(current));
4795        Ok(true)
4796    }
4797
4798    /// `PRAGMA data_version`, or `None` where the engine did not answer (the Recorder).
4799    fn file_data_version(&self) -> Result<Option<i64>, DbError> {
4800        let mut v = None;
4801        self.conn().query("PRAGMA data_version", &[], &mut |row| {
4802            if let Ok(i) = row.get(0).as_int() {
4803                v = Some(i);
4804            }
4805        })?;
4806        Ok(v)
4807    }
4808
4809    /// Re-select every resident row and feed the differences through the caches: per-field
4810    /// announcements, external deletions, all authored [`ModelContainer::EXTERNAL_AUTHOR`].
4811    fn refresh_resident(&self) -> Result<bool, DbError> {
4812        self.refresh_resident_in(None)
4813    }
4814
4815    /// [`ModelContainer::refresh_resident`] over the tables named (all of them for `None`).
4816    fn refresh_resident_in(&self, only: Option<&[&'static str]>) -> Result<bool, DbError> {
4817        struct RefreshJob {
4818            resident_keys: Rc<dyn Fn() -> Vec<u64>>,
4819            refresh: RefreshFn,
4820            columns: String,
4821            table: String,
4822            keys_where: KeysWhere,
4823        }
4824        let jobs: Vec<RefreshJob> = {
4825            let tables = self.inner.tables.borrow();
4826            tables
4827                .values()
4828                .filter(|h| only.is_none_or(|t| t.contains(&h.table)))
4829                .map(|h| RefreshJob {
4830                    resident_keys: h.resident_keys.clone(),
4831                    refresh: h.refresh.clone(),
4832                    columns: column_list(&h.columns),
4833                    table: h.name.sql(),
4834                    keys_where: keys_where_of(h),
4835                })
4836                .collect()
4837        };
4838        let mut changed = false;
4839        for RefreshJob {
4840            resident_keys,
4841            refresh,
4842            columns,
4843            table,
4844            keys_where,
4845        } in jobs
4846        {
4847            let resident = resident_keys();
4848            if resident.is_empty() {
4849                continue;
4850            }
4851            let mut rows: Vec<Vec<Value>> = Vec::new();
4852            for chunk in resident.chunks(MAX_BOUND_PARAMS / 2) {
4853                let (clause, params) = keys_where(chunk);
4854                self.conn().query(
4855                    &format!("SELECT {columns} FROM {table} WHERE {clause}"),
4856                    &params,
4857                    &mut |row| {
4858                        let n = Row::len(row);
4859                        rows.push((0..n).map(|i| row.get(i)).collect());
4860                    },
4861                )?;
4862            }
4863            // The connection borrow is released before refresh announces; a sink hearing
4864            // these changes may read back through SQL.
4865            if refresh(rows)? {
4866                changed = true;
4867            }
4868        }
4869        Ok(changed)
4870    }
4871
4872    fn install_state<M: Model>(
4873        &self,
4874        fetch: Fetch,
4875        raw: Option<RawQuery>,
4876        count_only: bool,
4877    ) -> Rc<QueryState> {
4878        let store = self.cache::<M>();
4879        let state = Rc::new(QueryState {
4880            store_id: store.store_id(),
4881            table: M::TABLE,
4882            watches: RefCell::new(Vec::new()),
4883            local_extra: RefCell::new(Vec::new()),
4884            set: RefCell::new(ResultSet::new(fetch.clone())),
4885            version: day_reactive::Scope::detached().enter(|| day_reactive::Signal::new(0)),
4886            pending: RefCell::new(QueryEvents::None),
4887            needs_sql: Cell::new(false),
4888            pending_commit: Cell::new(!self.inner.dirty.borrow().is_empty()),
4889            error: day_reactive::Scope::detached().enter(|| day_reactive::Signal::new(None)),
4890            raw,
4891            count_only,
4892            count: Cell::new(0),
4893        });
4894        self.rewire_watches(&state, &fetch);
4895        self.inner.queries.borrow_mut().push(Rc::downgrade(&state));
4896        // The seed: flush anything pending (with autosave; without it the query answers from
4897        // the last save, documented), then one SQL round trip.
4898        if self.inner.autosave.get() && !self.inner.dirty.borrow().is_empty() {
4899            let _ = self.save();
4900        }
4901        if count_only {
4902            state.count.set(self.answer_count(&state));
4903        } else {
4904            let ids = self.answer(&state);
4905            state.set.borrow_mut().reset(ids);
4906        }
4907        // The seed is the list's starting snapshot, not a stream of inserts to replay.
4908        *state.pending.borrow_mut() = QueryEvents::None;
4909        state
4910    }
4911
4912    fn install_query<M: Model>(&self, fetch: Fetch, raw: Option<RawQuery>) -> Query<M> {
4913        Query {
4914            state: self.install_state::<M>(fetch, raw, false),
4915            container: self.clone(),
4916            _p: std::marker::PhantomData,
4917        }
4918    }
4919
4920    /// (Re)compute a query's relation subscriptions from its fetch.
4921    fn rewire_watches(&self, state: &Rc<QueryState>, fetch: &Fetch) {
4922        let deps = fetch.dependencies();
4923        let mut watches: Vec<RelWatch> = Vec::new();
4924        let mut local_extra: Vec<&'static str> = Vec::new();
4925        let relations = self.inner.relations.borrow();
4926        let joins = self.inner.joins.borrow();
4927        for dep in &deps.related {
4928            let mut resolved = false;
4929            for r in relations.iter() {
4930                if r.parent_table == dep.owner && r.parent_field == dep.field {
4931                    watches.push(RelWatch {
4932                        store: r.child_store,
4933                        join_store: None,
4934                        fields: {
4935                            let mut f = vec![r.fk_field];
4936                            if let Some(o) = r.ordered {
4937                                f.push(o);
4938                            }
4939                            f
4940                        },
4941                        columns: dep.columns.clone(),
4942                    });
4943                    resolved = true;
4944                    break;
4945                }
4946                if r.child_table == dep.owner && r.fk_field == dep.field {
4947                    // The membership column lives on the query's table: a rewrite of the
4948                    // reference must mark it stale even though no predicate reads it.
4949                    if dep.owner == state.table && !local_extra.contains(&r.fk_col) {
4950                        local_extra.push(r.fk_col);
4951                    }
4952                    watches.push(RelWatch {
4953                        store: r.parent_store,
4954                        join_store: None,
4955                        fields: Vec::new(),
4956                        columns: dep.columns.clone(),
4957                    });
4958                    resolved = true;
4959                    break;
4960                }
4961            }
4962            if resolved {
4963                continue;
4964            }
4965            // A link: the far store's read columns matter, and so does the owner's value
4966            // column; rewriting it is what moves the row across the link.
4967            if let Some(l) = self
4968                .inner
4969                .links
4970                .borrow()
4971                .iter()
4972                .find(|l| l.owner_table == dep.owner && l.field == dep.field)
4973                .copied()
4974            {
4975                if dep.owner == state.table && !local_extra.contains(&l.local) {
4976                    local_extra.push(l.local);
4977                }
4978                let far = self
4979                    .inner
4980                    .tables
4981                    .borrow()
4982                    .iter()
4983                    .find(|(_, h)| h.table == l.target_table)
4984                    .map(|(id, _)| *id);
4985                if let Some(store) = far {
4986                    let mut columns = dep.columns.clone();
4987                    if !columns.contains(&l.remote) {
4988                        columns.push(l.remote);
4989                    }
4990                    watches.push(RelWatch {
4991                        store,
4992                        join_store: None,
4993                        fields: Vec::new(),
4994                        columns,
4995                    });
4996                }
4997                continue;
4998            }
4999            for j in joins.iter() {
5000                let side = if j.a_table == dep.owner && j.a_field == dep.field {
5001                    Some(j.b_store)
5002                } else if j.b_table == dep.owner && j.b_field.get() == Some(dep.field) {
5003                    Some(j.a_store)
5004                } else {
5005                    None
5006                };
5007                if let Some(other) = side {
5008                    watches.push(RelWatch {
5009                        store: other,
5010                        join_store: Some(j.join_store),
5011                        fields: Vec::new(),
5012                        columns: dep.columns.clone(),
5013                    });
5014                    break;
5015                }
5016            }
5017        }
5018        *state.watches.borrow_mut() = watches;
5019        *state.local_extra.borrow_mut() = local_extra;
5020    }
5021
5022    /// One announced change: decide, per live query, whether it can move the result, and
5023    /// mark stale where it can. No SQL runs here; the requery happens once, after the flush.
5024    fn mark_queries_stale(&self, change: &day_model::Change) {
5025        let Some(&store) = change.components.first() else {
5026            return;
5027        };
5028        let states: Vec<Rc<QueryState>> = {
5029            let mut queries = self.inner.queries.borrow_mut();
5030            queries.retain(|w| w.strong_count() > 0);
5031            queries.iter().filter_map(|w| w.upgrade()).collect()
5032        };
5033        if states.is_empty() {
5034            return;
5035        }
5036        let key = change.components.get(1).copied();
5037        if key == Some(day_model::STRUCTURE) {
5038            return; // the shape path duplicates the row paths
5039        }
5040        // The change log speaks field names; predicates speak column names.
5041        let (column, changed_table): (Option<String>, Option<&'static str>) = {
5042            let tables = self.inner.tables.borrow();
5043            let hooks = tables.get(&store);
5044            let column = if change.components.len() >= 3 {
5045                hooks.map(|h| {
5046                    h.fields
5047                        .iter()
5048                        .position(|f| f == change.label)
5049                        .map(|i| h.columns[i].clone())
5050                        // A transient field: never a dependency column.
5051                        .unwrap_or_else(|| "\u{0}".to_string())
5052                })
5053            } else {
5054                None
5055            };
5056            (column, hooks.map(|h| h.table))
5057        };
5058        let column = column.as_deref();
5059
5060        for state in &states {
5061            if state.needs_sql.get() {
5062                continue; // already stale; nothing to learn
5063            }
5064            if let Some(raw) = &state.raw {
5065                if changed_table.is_some_and(|t| raw.tables.iter().any(|r| r == t)) {
5066                    state.needs_sql.set(true);
5067                }
5068                continue;
5069            }
5070            // Both gates apply: a self-referential relation makes the query's own store a
5071            // related store too, so the own-table check must not shadow the watch check.
5072            let own_stale = state.store_id == store
5073                && match (key, change.op) {
5074                    (None, _) => true, // a wholesale rewrite: anything may have moved
5075                    (Some(_), Op::Insert) | (Some(_), Op::Delete) => true,
5076                    (Some(_), Op::Set) => match column {
5077                        // A row-level merge without a field name: assume the worst.
5078                        None => true,
5079                        Some(c) => {
5080                            let set = state.set.borrow();
5081                            set.deps().touches_local(c) || state.local_extra.borrow().contains(&c)
5082                        }
5083                    },
5084                    (Some(_), Op::Move) => false, // user order is not a query's business
5085                };
5086            let stale = own_stale || {
5087                let watches = state.watches.borrow();
5088                watches.iter().any(|w| {
5089                    if w.join_store == Some(store) {
5090                        return true; // every link/unlink/reposition is a membership change
5091                    }
5092                    if w.store != store {
5093                        return false;
5094                    }
5095                    match (key, change.op) {
5096                        (None, _) => true,
5097                        (Some(_), Op::Insert) | (Some(_), Op::Delete) => true,
5098                        (Some(_), Op::Set) => {
5099                            w.fields.contains(&change.label)
5100                                || column.is_some_and(|c| w.columns.contains(&c))
5101                        }
5102                        (Some(_), Op::Move) => false,
5103                    }
5104                })
5105            };
5106            if stale {
5107                state.needs_sql.set(true);
5108                state.pending_commit.set(true);
5109            }
5110        }
5111    }
5112
5113    /// After a flush: every stale query re-derives (plus raw queries whose tables the flush
5114    /// touched), each answer diffed into list deltas.
5115    fn run_deferred_requeries(&self, stmts: &[(String, Vec<Value>)]) {
5116        let states: Vec<Rc<QueryState>> = self
5117            .inner
5118            .queries
5119            .borrow()
5120            .iter()
5121            .filter_map(|w| w.upgrade())
5122            .collect();
5123        for state in states {
5124            let mut due = state.needs_sql.replace(false) | state.pending_commit.replace(false);
5125            if let Some(raw) = &state.raw
5126                && !due
5127            {
5128                due = stmts
5129                    .iter()
5130                    .any(|(sql, _)| raw.tables.iter().any(|t| statement_touches(sql, t)));
5131            }
5132            if due {
5133                self.requery(&state);
5134            }
5135        }
5136    }
5137
5138    /// Mark every query stale and requery now: the external-change and rescan path.
5139    fn requery_all(&self) {
5140        let states: Vec<Rc<QueryState>> = self
5141            .inner
5142            .queries
5143            .borrow()
5144            .iter()
5145            .filter_map(|w| w.upgrade())
5146            .collect();
5147        for state in states {
5148            state.needs_sql.set(false);
5149            self.requery(&state);
5150        }
5151    }
5152
5153    /// [`ModelContainer::requery_all`] for the live queries that read `tables`: a query over
5154    /// one of them, one crossing into one through a relation (or that relation's join
5155    /// table), or a raw query naming one.
5156    fn requery_tables(&self, tables: &[&'static str]) {
5157        let stores: Vec<u64> = self
5158            .inner
5159            .tables
5160            .borrow()
5161            .iter()
5162            .filter(|(_, h)| tables.contains(&h.table))
5163            .map(|(id, _)| *id)
5164            .collect();
5165        let states: Vec<Rc<QueryState>> = self
5166            .inner
5167            .queries
5168            .borrow()
5169            .iter()
5170            .filter_map(|w| w.upgrade())
5171            .collect();
5172        for state in states {
5173            let reads = match &state.raw {
5174                Some(raw) => raw.tables.iter().any(|t| tables.contains(&t.as_str())),
5175                None => {
5176                    stores.contains(&state.store_id)
5177                        || state.watches.borrow().iter().any(|w| {
5178                            stores.contains(&w.store)
5179                                || w.join_store.is_some_and(|j| stores.contains(&j))
5180                        })
5181                }
5182            };
5183            if reads {
5184                state.needs_sql.set(false);
5185                self.requery(&state);
5186            }
5187        }
5188    }
5189
5190    /// One requery: answer through the engine, adopt, narrate.
5191    fn requery(&self, state: &Rc<QueryState>) {
5192        if state.count_only {
5193            let n = self.answer_count(state);
5194            if n != state.count.get() {
5195                state.count.set(n);
5196                state
5197                    .version
5198                    .set(state.version.get_untracked().wrapping_add(1));
5199            }
5200            return;
5201        }
5202        let ids = self.answer(state);
5203        let change = state.set.borrow_mut().adopt(ids);
5204        match change {
5205            SetChange::Same => {}
5206            SetChange::Deltas(d) => {
5207                state.pending.borrow_mut().push(&d);
5208                state
5209                    .version
5210                    .set(state.version.get_untracked().wrapping_add(1));
5211            }
5212            SetChange::Reload => {
5213                state.pending.borrow_mut().reload();
5214                state
5215                    .version
5216                    .set(state.version.get_untracked().wrapping_add(1));
5217            }
5218        }
5219    }
5220
5221    /// Answer a COUNT-shaped fetch: one `SELECT COUNT(*)`, capped by the fetch's limit
5222    /// (a limited set's length is `min(count, limit)`). The fallback form counts its
5223    /// re-checked rows the same way.
5224    fn answer_count(&self, state: &Rc<QueryState>) -> usize {
5225        let result = (|| {
5226            if self.table_detached(state.table) {
5227                return Ok(0);
5228            }
5229            if let Some(raw) = &state.raw {
5230                return self
5231                    .select_ids_checked(&raw.sql, &raw.params)
5232                    .map(|ids| ids.len());
5233            }
5234            let fetch = state.set.borrow().fetch().clone();
5235            let cap = fetch.limit.unwrap_or(usize::MAX);
5236            let snapshot = self.sql_snapshot();
5237            match compile_count(state.table, &fetch, &snapshot) {
5238                Ok(q) => {
5239                    let mut n = Ok(0_i64);
5240                    self.conn().query(&q.sql, &q.params, &mut |row| {
5241                        n = row.get(0).as_int();
5242                    })?;
5243                    Ok((n?.max(0) as usize).min(cap))
5244                }
5245                Err(CompileErr::NeedsFold) => {
5246                    let (q, cols) = compile_fallback(state.table, &fetch, &snapshot)
5247                        .map_err(|e| DbError::new(DbErrorKind::Unsupported, e.message()))?;
5248                    let mut n = 0;
5249                    self.conn().query(&q.sql, &q.params, &mut |row| {
5250                        let values = (0..row.len()).map(|i| row.get(i)).collect::<Vec<_>>();
5251                        if let Some(h) = value_to_handle(&values[0])
5252                            && fetch.pred.eval(
5253                                h,
5254                                &FallbackRow {
5255                                    cols: &cols,
5256                                    values: &values,
5257                                },
5258                            )
5259                        {
5260                            n += 1;
5261                        }
5262                    })?;
5263                    Ok(n.min(cap))
5264                }
5265                Err(e) => Err(DbError::new(DbErrorKind::Unsupported, e.message())),
5266            }
5267        })();
5268        match result {
5269            Ok(n) => {
5270                state.error.set(None);
5271                n
5272            }
5273            Err(error) => {
5274                self.inner.error.set(Some(error.to_string()));
5275                state.error.set(Some(error));
5276                state.count.get()
5277            }
5278        }
5279    }
5280
5281    /// Answer a query's fetch through the engine: the compiled form, the fallback form for
5282    /// drivers without `day_fold`, or the raw statement.
5283    fn answer(&self, state: &Rc<QueryState>) -> Vec<u64> {
5284        match self.answer_checked(state) {
5285            Ok(ids) => {
5286                state.error.set(None);
5287                ids
5288            }
5289            Err(error) => {
5290                self.inner.error.set(Some(error.to_string()));
5291                state.error.set(Some(error));
5292                // A failed read is not an empty result. Retain the last successful set.
5293                state.set.borrow().ids().to_vec()
5294            }
5295        }
5296    }
5297    /// Whether `table` belongs to an attached database that is not attached right now. The
5298    /// alias is the model's declared one, not a prefix of the name: a table of the
5299    /// container's own may be called `dot.ted`.
5300    fn table_detached(&self, table: &str) -> bool {
5301        let tables = self.inner.tables.borrow();
5302        let Some(alias) = tables
5303            .values()
5304            .find(|h| h.table == table)
5305            .and_then(|h| h.name.schema)
5306        else {
5307            return false;
5308        };
5309        alias != "main"
5310            && alias != "temp"
5311            && !self.inner.attached.borrow().iter().any(|a| a == alias)
5312    }
5313    fn answer_checked(&self, state: &Rc<QueryState>) -> Result<Vec<u64>, DbError> {
5314        // Explicit detachment is a successful transition to an empty catalog, not an I/O error.
5315        if self.table_detached(state.table) {
5316            return Ok(Vec::new());
5317        }
5318        if let Some(raw) = &state.raw {
5319            return self.select_ids_checked(&raw.sql, &raw.params);
5320        }
5321        let fetch = state.set.borrow().fetch().clone();
5322        let snapshot = self.sql_snapshot();
5323        match compile_fetch(state.table, &fetch, &snapshot) {
5324            Ok(q) => self.select_ids_checked(&q.sql, &q.params),
5325            Err(CompileErr::NeedsFold) => {
5326                let (q, cols) = compile_fallback(state.table, &fetch, &snapshot)
5327                    .map_err(|e| DbError::new(DbErrorKind::Unsupported, e.message()))?;
5328                let mut ids = Vec::new();
5329                self.conn().query(&q.sql, &q.params, &mut |row| {
5330                    let values = (0..row.len()).map(|i| row.get(i)).collect::<Vec<_>>();
5331                    if let Some(h) = value_to_handle(&values[0])
5332                        && fetch.pred.eval(
5333                            h,
5334                            &FallbackRow {
5335                                cols: &cols,
5336                                values: &values,
5337                            },
5338                        )
5339                    {
5340                        ids.push(h);
5341                    }
5342                })?;
5343                if let Some(limit) = fetch.limit {
5344                    ids.truncate(limit);
5345                }
5346                Ok(ids)
5347            }
5348            Err(e) => Err(DbError::new(DbErrorKind::Unsupported, e.message())),
5349        }
5350    }
5351
5352    /// A point-in-time view of names for the SQL compiler: tables, keys, shadows, wired
5353    /// relations, and whether the driver folds case.
5354    fn sql_snapshot(&self) -> SqlSnapshot {
5355        let tables = self.inner.tables.borrow();
5356        let relations = self.inner.relations.borrow();
5357        let joins = self.inner.joins.borrow();
5358        SqlSnapshot {
5359            tables: tables
5360                .values()
5361                .map(|h| TableInfo {
5362                    table: h.table.to_string(),
5363                    sql: h.name.sql(),
5364                    key: h.key_cols[0].clone(),
5365                    fts: h.fts.as_ref().map(|_| FtsSql {
5366                        from: h.name.shadow("_fts"),
5367                        match_col: ident(&format!("{}_fts", h.name.bare)),
5368                    }),
5369                    fts_key: h.fts_key,
5370                    spatial: h
5371                        .spatial
5372                        .as_ref()
5373                        .map(|(lat, lon, _)| (lat.clone(), lon.clone(), h.name.shadow("_geo"))),
5374                })
5375                .collect(),
5376            rels: relations
5377                .iter()
5378                .map(|r| RelInfo {
5379                    parent_table: r.parent_table,
5380                    parent_field: r.parent_field,
5381                    parent_key: r.parent_key_col,
5382                    child_table: r.child_table,
5383                    child_key: r.child_key_col,
5384                    fk_field: r.fk_field,
5385                    fk_col: r.fk_col,
5386                })
5387                .collect(),
5388            joins: joins
5389                .iter()
5390                .map(|j| JoinInfo {
5391                    join_table: j.join_table,
5392                    a_table: j.a_table,
5393                    a_field: j.a_field,
5394                    a_key: j.a_key_col,
5395                    b_table: j.b_table,
5396                    b_field: j.b_field.get(),
5397                    b_key: j.b_key_col,
5398                    parent_col: j.parent_col,
5399                    child_col: j.child_col,
5400                })
5401                .collect(),
5402            links: self.inner.links.borrow().clone(),
5403            fold: self.inner.caps.unicode_fold,
5404        }
5405    }
5406}
5407
5408/// One table as the SQL compiler sees it: `table` is the plain name it is asked about,
5409/// `sql` the quoted form it writes; the shadows come already quoted, the columns plain.
5410struct TableInfo {
5411    table: String,
5412    sql: String,
5413    key: String,
5414    fts: Option<FtsSql>,
5415    fts_key: &'static str,
5416    spatial: Option<(String, String, String)>,
5417}
5418
5419struct RelInfo {
5420    parent_table: &'static str,
5421    parent_field: &'static str,
5422    parent_key: &'static str,
5423    child_table: &'static str,
5424    child_key: &'static str,
5425    fk_field: &'static str,
5426    fk_col: &'static str,
5427}
5428
5429struct JoinInfo {
5430    join_table: &'static str,
5431    a_table: &'static str,
5432    a_field: &'static str,
5433    a_key: &'static str,
5434    b_table: &'static str,
5435    b_field: Option<&'static str>,
5436    b_key: &'static str,
5437    parent_col: &'static str,
5438    child_col: &'static str,
5439}
5440
5441struct SqlSnapshot {
5442    tables: Vec<TableInfo>,
5443    rels: Vec<RelInfo>,
5444    joins: Vec<JoinInfo>,
5445    links: Vec<LinkInfo>,
5446    fold: bool,
5447}
5448
5449impl SqlIndex for SqlSnapshot {
5450    fn relation(&self, owner: &str, field: &str) -> Option<RelSql> {
5451        for r in &self.rels {
5452            if r.parent_table == owner && r.parent_field == field {
5453                return Some(RelSql::Children {
5454                    target_key: r.child_key.to_string(),
5455                    fk_col: r.fk_col.to_string(),
5456                    owner_key: r.parent_key.to_string(),
5457                });
5458            }
5459            if r.child_table == owner && r.fk_field == field {
5460                return Some(RelSql::Referent {
5461                    target_key: r.parent_key.to_string(),
5462                    fk_col: r.fk_col.to_string(),
5463                });
5464            }
5465        }
5466        for j in &self.joins {
5467            if j.a_table == owner && j.a_field == field {
5468                return Some(RelSql::Join {
5469                    join_table: j.join_table.to_string(),
5470                    owner_col: j.parent_col.to_string(),
5471                    target_col: j.child_col.to_string(),
5472                    owner_key: j.a_key.to_string(),
5473                    target_key: j.b_key.to_string(),
5474                });
5475            }
5476            if j.b_table == owner && j.b_field == Some(field) {
5477                return Some(RelSql::Join {
5478                    join_table: j.join_table.to_string(),
5479                    owner_col: j.child_col.to_string(),
5480                    target_col: j.parent_col.to_string(),
5481                    owner_key: j.b_key.to_string(),
5482                    target_key: j.a_key.to_string(),
5483                });
5484            }
5485        }
5486        for l in &self.links {
5487            if l.owner_table == owner && l.field == field {
5488                return Some(RelSql::Linked {
5489                    local_col: l.local.to_string(),
5490                    remote_col: l.remote.to_string(),
5491                    target_key: self.key_of(l.target_table)?,
5492                });
5493            }
5494        }
5495        None
5496    }
5497
5498    fn key_of(&self, table: &str) -> Option<String> {
5499        self.tables
5500            .iter()
5501            .find(|t| t.table == table)
5502            .map(|t| t.key.clone())
5503    }
5504
5505    fn table_sql(&self, table: &str) -> String {
5506        self.tables
5507            .iter()
5508            .find(|t| t.table == table)
5509            .map(|t| t.sql.clone())
5510            .unwrap_or_else(|| ident(table))
5511    }
5512
5513    fn fts_of(&self, table: &str) -> Option<FtsSql> {
5514        self.tables
5515            .iter()
5516            .find(|t| t.table == table)
5517            .and_then(|t| t.fts.clone())
5518    }
5519
5520    fn fts_key_of(&self, table: &str) -> &str {
5521        self.tables
5522            .iter()
5523            .find(|t| t.table == table)
5524            .map(|t| t.fts_key)
5525            .unwrap_or("rowid")
5526    }
5527
5528    fn geo_of(&self, table: &str, lat: &str, lon: &str) -> Option<String> {
5529        self.tables
5530            .iter()
5531            .find(|t| t.table == table)
5532            .and_then(|t| t.spatial.as_ref())
5533            .filter(|(a, o, _)| a == lat && o == lon)
5534            .map(|(_, _, g)| g.clone())
5535    }
5536
5537    fn unicode_fold(&self) -> bool {
5538        self.fold
5539    }
5540}
5541
5542/// Whether a statement's target table is `table`, a plain-text check over the SQL the fold
5543/// itself produced (`INSERT INTO "t" … ON CONFLICT`, `UPDATE "t" SET …`, `DELETE FROM "t" …`).
5544/// The fold quotes its table names, so the name is read back through the quotes; `table` is
5545/// the plain name a raw query declared.
5546fn statement_touches(sql: &str, table: &str) -> bool {
5547    for prefix in [
5548        "INSERT INTO ",
5549        "INSERT OR REPLACE INTO ",
5550        "UPDATE ",
5551        "DELETE FROM ",
5552    ] {
5553        if let Some(rest) = sql.strip_prefix(prefix) {
5554            return quoted_name(rest).as_deref() == Some(table);
5555        }
5556    }
5557    false
5558}
5559
5560/// The identifier a quoted name at the start of `sql` spells (a doubled `"` is one quote),
5561/// `None` when `sql` does not start with one.
5562fn quoted_name(sql: &str) -> Option<String> {
5563    let mut chars = sql.strip_prefix('"')?.chars();
5564    let mut name = String::new();
5565    loop {
5566        match chars.next()? {
5567            '"' => match chars.next() {
5568                Some('"') => name.push('"'),
5569                _ => return Some(name),
5570            },
5571            ch => name.push(ch),
5572        }
5573    }
5574}
5575
5576// ---------------------------------------------------------------------------
5577// Query as a list row source (feature `pieces`)
5578// ---------------------------------------------------------------------------
5579
5580#[cfg(feature = "pieces")]
5581mod pieces_glue {
5582    use std::cell::RefCell;
5583
5584    use day_model::{Elem, Keyed, Store};
5585    use day_pieces::{ModelSlot, RowConn, RowSource};
5586
5587    use crate::{Delta, Model, Query, QueryEvents};
5588
5589    /// How far around a bound row the list source faults in one batch: the price of showing
5590    /// row N is one `SELECT` that also covers the rows about to scroll in.
5591    const FAULT_BEHIND: usize = 16;
5592    const FAULT_AHEAD: usize = 64;
5593
5594    /// `list(query, row)`: the query's ids are the display order, rows fault in as the list
5595    /// binds them (a window at a time), and set changes arrive as row deltas the native list
5596    /// can animate.
5597    impl<M: Model> RowSource for Query<M> {
5598        type Slot = ModelSlot<M>;
5599        type Ref = Elem<M>;
5600        type Conn = QueryConn<M>;
5601        fn connect(self) -> QueryConn<M> {
5602            let store = self.container.cache::<M>();
5603            QueryConn {
5604                query: self,
5605                store,
5606                keys: RefCell::new(Vec::new()),
5607            }
5608        }
5609    }
5610
5611    pub struct QueryConn<M: Model> {
5612        query: Query<M>,
5613        store: Store<Keyed<M>>,
5614        keys: RefCell<Vec<u64>>,
5615    }
5616
5617    impl<M: Model> QueryConn<M> {
5618        /// Make the row at `index` (and its neighborhood) resident before anything binds it.
5619        fn fault_window(&self, index: usize) {
5620            let len = self.keys.borrow().len();
5621            let start = index.saturating_sub(FAULT_BEHIND);
5622            let end = (index + FAULT_AHEAD).min(len);
5623            self.query.materialize(start..end);
5624        }
5625    }
5626
5627    impl<M: Model> RowConn for QueryConn<M> {
5628        type Slot = ModelSlot<M>;
5629        type Ref = Elem<M>;
5630
5631        fn refresh(&self) -> Vec<u64> {
5632            let keys: Vec<u64> = self.query.ids().iter().map(|id| id.handle()).collect();
5633            *self.keys.borrow_mut() = keys.clone();
5634            keys
5635        }
5636        fn len(&self) -> usize {
5637            self.keys.borrow().len()
5638        }
5639        fn token_at(&self, index: usize) -> u64 {
5640            self.keys.borrow().get(index).copied().unwrap_or(0)
5641        }
5642        fn tokens_now(&self) -> Vec<u64> {
5643            self.keys.borrow().clone()
5644        }
5645        fn slot_at(&self, index: usize) -> Option<ModelSlot<M>> {
5646            let key = self.keys.borrow().get(index).copied()?;
5647            if !self.store.with_untracked(|k| k.get(key).is_some()) {
5648                self.fault_window(index);
5649            }
5650            Some(ModelSlot::for_key(self.store, key))
5651        }
5652        fn rebind(&self, slot: &ModelSlot<M>, index: usize) {
5653            if let Some(key) = self.keys.borrow().get(index).copied() {
5654                if !self.store.with_untracked(|k| k.get(key).is_some()) {
5655                    self.fault_window(index);
5656                }
5657                slot.rebind_key(key);
5658            }
5659        }
5660        fn select_ref(&self, index: usize) -> Option<Elem<M>> {
5661            let key = self.keys.borrow().get(index).copied()?;
5662            if !self.store.with_untracked(|k| k.get(key).is_some()) {
5663                self.fault_window(index);
5664            }
5665            Some(self.store.elem(key))
5666        }
5667        fn values_flow_by_reload(&self) -> bool {
5668            false // row values flow through the store's own per-field notifications
5669        }
5670        fn take_row_events(&self) -> Option<Vec<day_spec::props::RowDelta>> {
5671            match self.query.take_events() {
5672                QueryEvents::Deltas(d) => Some(
5673                    d.into_iter()
5674                        .map(|d| match d {
5675                            Delta::Insert(i, _) => day_spec::props::RowDelta::Insert(i),
5676                            Delta::Remove(i, _) => day_spec::props::RowDelta::Remove(i),
5677                            Delta::Move(from, to, _) => day_spec::props::RowDelta::Move(from, to),
5678                        })
5679                        .collect(),
5680                ),
5681                // Reload means "the whole set moved"; None here means the same to the list.
5682                QueryEvents::Reload | QueryEvents::None => None,
5683            }
5684        }
5685        fn commit_move(&self, from: usize, to: usize) {
5686            // The native drag already moved the row: serve that order until the app's write
5687            // (its on_reorder, typically a rank column the query sorts by) requeries. The list
5688            // expects exactly this order back as the echo of its own move; left unrotated, the
5689            // echo never matched, and the query's own move was replayed on rows already moved.
5690            let mut keys = self.keys.borrow_mut();
5691            if from < keys.len() && to < keys.len() {
5692                let key = keys.remove(from);
5693                keys.insert(to, key);
5694            }
5695        }
5696        fn commit_delete(&self, index: usize) {
5697            // The native delete affordance: drop from the local snapshot; the app's on_delete
5698            // writes the store, and the query's own delta follows as the echo.
5699            let mut keys = self.keys.borrow_mut();
5700            if index < keys.len() {
5701                keys.remove(index);
5702            }
5703        }
5704    }
5705}
5706
5707#[cfg(feature = "pieces")]
5708pub use pieces_glue::QueryConn;
5709
5710// ---------------------------------------------------------------------------
5711// Date and time codecs (feature `datetime`; docs/persistence.md)
5712// ---------------------------------------------------------------------------
5713
5714/// Column codecs for day::datetime's [`DayDate`](day_spec::controls::datetime::DayDate) and
5715/// [`DayTime`](day_spec::controls::datetime::DayTime), for `#[derive(Model)]` fields.
5716///
5717/// Canonical, what a bare field of the type stores: `DayDate` → `INTEGER` epoch days,
5718/// `DayTime` → `INTEGER` seconds-of-day: the same numeric interchange forms the pickers
5719/// already speak across native boundaries, and both sort chronologically as plain integers.
5720///
5721/// Named, picked per field with `#[model(with = …)]`: [`Iso8601`](datetime::Iso8601) stores
5722/// `TEXT` in the form `Display` writes (readable in any sqlite tool, and lexicographic order
5723/// is chronological order, ISO-8601's design goal); [`EpochSeconds`](datetime::EpochSeconds)
5724/// and [`EpochMillis`](datetime::EpochMillis) store the `INTEGER` a server that speaks Unix
5725/// time expects, midnight-of-day for a `DayDate`.
5726///
5727/// The impls live here rather than in the piece because one of the two crates has to own
5728/// them, and this is the one that depends on the other.
5729#[cfg(feature = "datetime")]
5730pub mod datetime {
5731    use crate::{ColumnValue, DbError, DbErrorKind, SqlType, Value, ValueCodec};
5732    use day_spec::controls::datetime::{DayDate, DayTime};
5733
5734    impl ColumnValue for DayDate {
5735        const SQL_TYPE: SqlType = SqlType::Integer;
5736        fn to_sqlite_value(&self) -> Value {
5737            Value::Int(self.to_epoch_days())
5738        }
5739        fn from_sqlite_value(v: Value) -> Result<Self, DbError> {
5740            Ok(DayDate::from_epoch_days(v.as_int()?))
5741        }
5742    }
5743
5744    impl ColumnValue for DayTime {
5745        const SQL_TYPE: SqlType = SqlType::Integer;
5746        fn to_sqlite_value(&self) -> Value {
5747            Value::Int(self.seconds_of_day())
5748        }
5749        fn from_sqlite_value(v: Value) -> Result<Self, DbError> {
5750            Ok(DayTime::from_seconds_of_day(v.as_int()?))
5751        }
5752    }
5753
5754    /// `TEXT` in the `Display` form: `YYYY-MM-DD` for a date, `HH:MM[:SS]` for a time.
5755    pub struct Iso8601;
5756
5757    impl ValueCodec<DayDate> for Iso8601 {
5758        const SQL_TYPE: SqlType = SqlType::Text;
5759        fn to_sqlite_value(v: &DayDate) -> Value {
5760            Value::Text(v.to_string())
5761        }
5762        fn from_sqlite_value(v: Value) -> Result<DayDate, DbError> {
5763            let s = v.as_text()?;
5764            DayDate::parse_iso(s).ok_or_else(|| {
5765                DbError::new(DbErrorKind::Decode, format!("not an ISO-8601 date: {s:?}"))
5766            })
5767        }
5768    }
5769
5770    impl ValueCodec<DayTime> for Iso8601 {
5771        const SQL_TYPE: SqlType = SqlType::Text;
5772        fn to_sqlite_value(v: &DayTime) -> Value {
5773            Value::Text(v.to_string())
5774        }
5775        fn from_sqlite_value(v: Value) -> Result<DayTime, DbError> {
5776            let s = v.as_text()?;
5777            DayTime::parse_iso(s).ok_or_else(|| {
5778                DbError::new(DbErrorKind::Decode, format!("not an ISO-8601 time: {s:?}"))
5779            })
5780        }
5781    }
5782
5783    /// `INTEGER` seconds since 1970-01-01T00:00Z; midnight of the day, for a [`DayDate`].
5784    pub struct EpochSeconds;
5785
5786    impl ValueCodec<DayDate> for EpochSeconds {
5787        const SQL_TYPE: SqlType = SqlType::Integer;
5788        fn to_sqlite_value(v: &DayDate) -> Value {
5789            Value::Int(v.to_epoch_days() * 86_400)
5790        }
5791        fn from_sqlite_value(v: Value) -> Result<DayDate, DbError> {
5792            Ok(DayDate::from_epoch_days(v.as_int()?.div_euclid(86_400)))
5793        }
5794    }
5795
5796    /// `INTEGER` milliseconds since the same epoch, for interop that expects it.
5797    pub struct EpochMillis;
5798
5799    impl ValueCodec<DayDate> for EpochMillis {
5800        const SQL_TYPE: SqlType = SqlType::Integer;
5801        fn to_sqlite_value(v: &DayDate) -> Value {
5802            Value::Int(v.to_epoch_days() * 86_400_000)
5803        }
5804        fn from_sqlite_value(v: Value) -> Result<DayDate, DbError> {
5805            Ok(DayDate::from_epoch_days(v.as_int()?.div_euclid(86_400_000)))
5806        }
5807    }
5808}
5809
5810#[cfg(all(test, feature = "driver-rusqlite"))]
5811mod eviction_tests;