Skip to main content

syncular_client/
client.rs

1//! The Syncular v2 Rust client core (SPEC.md client-behavior contract):
2//! rusqlite local storage, §3.2/§3.3 effective-scope persistence + purge,
3//! §4 pull/cursor/bootstrap (§4.7 resume, §5.6 segment application), §6
4//! push with outbox order, §7 optimistic apply / rollback / replay-on-top,
5//! §2.3 clientCommitId idempotency, §8 realtime client rules, §10 errors.
6//!
7//! Built from `SPEC.md` and the committed `ssp2` codec alone — no
8//! reference to the v1 Rust tree or the v2 TypeScript client.
9
10use crate::{ProgressObserver, ProgressPhase, ProgressState};
11use ssp2::encode_message;
12use std::borrow::Cow;
13#[cfg(test)]
14use std::cell::Cell;
15use std::cell::RefCell;
16use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
17use std::sync::Arc;
18
19use rusqlite::types::{ToSqlOutput, Value as SqlValue, ValueRef};
20use rusqlite::{Connection, OpenFlags, OptionalExtension};
21use serde::{Deserialize, Serialize};
22use serde_json::{json, Map, Value};
23use sha2::{Digest, Sha256};
24use ssp2::decode::WIRE_VERSION;
25use ssp2::model::{
26    FencedRecord, Frame, LiveStatus, MediaType, Message, MsgKind, Op, OpResult, PushStatus,
27    SubStatus,
28};
29use ssp2::primitives::RawJson;
30use ssp2::registration::canonical_scope_json;
31use ssp2::segment::{decode_rows_segment, Column, ColumnType, ColumnValue, Row, RowsSegment};
32use ssp2::{decode_message, encode_presence_publish, parse_control, ControlMessage, PresenceKind};
33
34use crate::api::{
35    AppliedServerCommit, AppliedServerCommitSource, ClientChangeBatch, ClientDiagnosticsHost,
36    ClientDiagnosticsLease, ClientDiagnosticsReplica, ClientDiagnosticsSchema,
37    ClientDiagnosticsSnapshot, ClientDiagnosticsStorage, ClientError, ClientLimits, CommandEffects,
38    CommitDelivery, CommitOperation, CommitOperationOutcome, CommitOutcome, CommitOutcomeQuery,
39    CommitOutcomeResolution, CommitOutcomeStatus, ConflictRecord, CoverageSnapshot,
40    CurrentBaseProof, CurrentBaseProofReason, DiagnosticLastChange, DiagnosticLastRound,
41    DiagnosticQueryFailure, DiagnosticRoundCounters, DiagnosticUnits, EchoRecord, FetchedBlob,
42    LeaseState, LocalDataPurgeInput, LocalDataPurgeResult, LocalDataPurgeTarget,
43    LocalDataRebootstrapInput, LocalDataRebootstrapResult, Mutation, PresencePeer,
44    PreviousVersionStatus, QueryOwner, QueryReadFailure, QueryRow, QuerySnapshot, QueryValue,
45    RejectionDetails, RejectionRecord, ResolveCommitOutcomeInput, RetainedCommitRow,
46    RetainedUniqueConflict, RowState, SchemaFloor, SnapshotRead, SyncIntent, SyncOutcome,
47    SyncReport, SyncStatusSnapshot, TableChange, UnitCatchup, UnitRecord, UnitScopes, UnitStatus,
48    WindowBase, WindowChange, WindowCoverage, WindowState, WindowUnitRef,
49    CLIENT_DIAGNOSTICS_VERSION, MAX_DIAGNOSTIC_QUERY_FAILURES,
50};
51use crate::api::{RealtimePolicy, RealtimeState, REALTIME_UNAVAILABLE_CODE};
52#[cfg(feature = "bench-internals")]
53use crate::bench::{Phase, Recorder};
54use crate::previous_version::{
55    capture_previous_version_from_replica, first_incompatibility,
56    reconcile_previous_version_at_boot, set_local_schema_descriptor,
57    sweep_previous_version_container, PendingCommitAudit, PreviousVersionContextConfig,
58    PreviousVersionLifecycle, PreviousVersionReadSpec, PreviousVersionSnapshot,
59};
60use crate::schema::{parse_schema_json, ClientSchema, FtsIndexSchema, TableSchema};
61use crate::transport::{BlobDownload, SegmentRequest, Transport, TransportError};
62use crate::values::{
63    bytes_to_hex, column_value_to_json, decode_row_bytes, encode_sparse_row_json, full_row_values,
64    is_canonical_bytes_envelope, json_to_column_value, json_to_scope_map, normalize_values_casing,
65    render_row_id_json, scope_map_to_json, SealedRow,
66};
67
68/// §4.2 default: rows and SQLite images, both applied in committed chunks.
69/// Bit 3 (signed URLs, §5.4) is
70/// added per transport capability at request-build time.
71const DEFAULT_ACCEPT: u8 = 0b0111;
72const ACCEPT_INLINE_ROWS: u8 = 1 << 0;
73const ACCEPT_EXTERNAL_ROWS: u8 = 1 << 1;
74const ACCEPT_SQLITE: u8 = 1 << 2;
75const ACCEPT_SIGNED_URLS: u8 = 1 << 3;
76const MAX_DIAGNOSTIC_DOMAINS: usize = 256;
77
78/// §10.1 `recommendedAction` carried by a server HTTP error body.
79fn recommended_action_of(details: &Option<Value>) -> Option<String> {
80    details
81        .as_ref()?
82        .get("recommendedAction")?
83        .as_str()
84        .map(str::to_owned)
85}
86
87/// §7.4.1 persisted local schema-version marker (`_syncular_meta` key).
88pub(crate) const LOCAL_SCHEMA_VERSION_KEY: &str = "localSchemaVersion";
89const LOCAL_REVISION_KEY: &str = "localRevision";
90/// §4.5 replica cursor; absent means `-1`.
91const REPLICA_CURSOR_KEY: &str = "replicaCursor";
92/// Hex of the last pull `LIVE_END.echoDigest` (§4.2); absent means empty.
93const ECHO_DIGEST_KEY: &str = "echoDigest";
94/// Hex of the registration digest of the list the server stores (§4.1).
95const REGISTERED_DIGEST_KEY: &str = "registrationDigest";
96/// §7.4.3: table name → generation, persisted with the declaration evidence.
97const TABLE_GENERATIONS_KEY: &str = "localTableGenerations";
98/// §7.1: the identity of the outbox sequence.
99const OUTBOX_ID_KEY: &str = "outboxId";
100const CLIENT_ID_KEY: &str = "clientId";
101const LEASE_STATE_KEY: &str = "leaseState";
102const SCHEMA_FLOOR_KEY: &str = "schemaFloor";
103pub(crate) const LOG_EPOCH_KEY: &str = "logEpoch";
104/// Persisted fail-closed quarantine marker: present while a security preflight
105/// is pending and `activateSecurity` has yet to run. Storing it in the replica
106/// keeps the gate with the data it protects, so a rebuilt host handle reopening
107/// the same file returns in preflight even though its in-memory state is fresh.
108const SECURITY_PREFLIGHT_PENDING_KEY: &str = "securityPreflightPending";
109const LOCAL_REBOOTSTRAP_RECEIPT_VERSION: u8 = 2;
110const MAX_JS_SAFE_INTEGER: u64 = 9_007_199_254_740_991;
111/// §7.4.4 client-local code: a pending outbox commit cannot re-encode under
112/// the new schema after a bump. Never a wire code (§10.3).
113const OUTBOX_INCOMPATIBLE_CODE: &str = "sync.outbox_incompatible";
114/// §5.11 client-local code: an encode at the push seam resolved no usable key
115/// id or named an unknown key. Never a wire code (§10.3).
116const ENCRYPT_FAILED_CODE: &str = "client.encrypt_failed";
117const INVALID_REQUEST_CODE: &str = "sync.invalid_request:";
118/// §7.4.4: static text for a persisted commit the current codec refuses.
119const LEGACY_VALUE_REFUSAL_MESSAGE: &str =
120    "the persisted commit carries values the current codec refuses";
121/// §7.4.4: bounded reason on every intrinsic value drop.
122const INVALID_STORED_VALUES_REASON: &str = "invalid_stored_values";
123
124#[derive(Debug, Serialize, Deserialize)]
125#[serde(rename_all = "camelCase", deny_unknown_fields)]
126struct PersistedLocalDataRebootstrapReceipt {
127    version: u8,
128    retained_commits: u64,
129    reset_subscriptions: u64,
130}
131
132fn invalid_local_rebootstrap_receipt() -> String {
133    "sync.local_corrupt: persisted local rebootstrap receipt is invalid".to_owned()
134}
135
136fn encode_local_rebootstrap_receipt(
137    retained_commits: usize,
138    reset_subscriptions: usize,
139) -> Result<String, String> {
140    let retained_commits =
141        u64::try_from(retained_commits).map_err(|_| invalid_local_rebootstrap_receipt())?;
142    let reset_subscriptions =
143        u64::try_from(reset_subscriptions).map_err(|_| invalid_local_rebootstrap_receipt())?;
144    if retained_commits > MAX_JS_SAFE_INTEGER || reset_subscriptions > MAX_JS_SAFE_INTEGER {
145        return Err(invalid_local_rebootstrap_receipt());
146    }
147    serde_json::to_string(&PersistedLocalDataRebootstrapReceipt {
148        version: LOCAL_REBOOTSTRAP_RECEIPT_VERSION,
149        retained_commits,
150        reset_subscriptions,
151    })
152    .map_err(|_| invalid_local_rebootstrap_receipt())
153}
154
155fn decode_local_rebootstrap_receipt(value: &str) -> Result<(usize, usize), String> {
156    // Pre-0.15.36 markers proved application but did not retain the receipt.
157    if value == "v1" {
158        return Ok((0, 0));
159    }
160    let receipt: PersistedLocalDataRebootstrapReceipt =
161        serde_json::from_str(value).map_err(|_| invalid_local_rebootstrap_receipt())?;
162    if receipt.version != LOCAL_REBOOTSTRAP_RECEIPT_VERSION
163        || receipt.retained_commits > MAX_JS_SAFE_INTEGER
164        || receipt.reset_subscriptions > MAX_JS_SAFE_INTEGER
165    {
166        return Err(invalid_local_rebootstrap_receipt());
167    }
168    Ok((
169        usize::try_from(receipt.retained_commits)
170            .map_err(|_| invalid_local_rebootstrap_receipt())?,
171        usize::try_from(receipt.reset_subscriptions)
172            .map_err(|_| invalid_local_rebootstrap_receipt())?,
173    ))
174}
175
176#[cfg(test)]
177mod observation_tests;
178
179/// Internal state for hosts that perform network I/O outside the client owner.
180#[doc(hidden)]
181pub struct CurrentBasePreparation {
182    client_commit_id: String,
183    sequence: i64,
184    units: Vec<UnitRecord>,
185    transport_epoch: u64,
186    registration_epoch: u64,
187    security_epoch: u64,
188}
189
190#[cfg(test)]
191mod units_tests;
192
193#[derive(Debug, Clone)]
194struct OutboxOp {
195    upsert: bool,
196    table: String,
197    row_id: String,
198    base_version: Option<i64>,
199    /// Schema-agnostic local form (§0): driver JSON values, encoded with
200    /// the current codec at send time. A key's presence IS the §6.1 sparse
201    /// presence set — both the overlay and the wire payload derive from it.
202    values: Option<Map<String, Value>>,
203    /// §7.1 insert intent: `mutate` recorded this upsert while the row was
204    /// absent from the visible replica. Replay and send-time encoding present
205    /// every current-schema column; a column `values` lacks is NULL.
206    insert: bool,
207}
208
209impl OutboxOp {
210    /// The values this upsert presents under the current `table` (§7.1). An
211    /// insert-intent operation presents every column, a missing one as NULL;
212    /// `None` when a missing column is non-nullable, which leaves the commit
213    /// unencodable (§7.4.4), or when the operation carries no values.
214    fn presented_values(&self, table: &TableSchema) -> Option<Cow<'_, Map<String, Value>>> {
215        let values = self.values.as_ref()?;
216        if !self.insert || table.columns.iter().all(|c| values.contains_key(&c.name)) {
217            return Some(Cow::Borrowed(values));
218        }
219        full_row_values(table, values.clone()).ok().map(Cow::Owned)
220    }
221}
222
223impl From<CommitOperation> for OutboxOp {
224    fn from(operation: CommitOperation) -> Self {
225        Self {
226            upsert: operation.op == "upsert",
227            table: operation.table,
228            row_id: operation.row_id,
229            base_version: operation.base_version,
230            values: operation.values,
231            insert: operation.insert,
232        }
233    }
234}
235
236/// §7.5 static authoring refusal; the table and column travel in details.
237fn invalid_authoring(message: &str, details: Value) -> ClientError {
238    ClientError {
239        code: "sync.invalid_request".to_owned(),
240        message: message.to_owned(),
241        details: Some(details),
242        retryable: false,
243    }
244}
245
246/// §7.1 chained base: `later` repeats `earlier`'s positive base on the same
247/// row and presents only columns `earlier` presents.
248fn chains_on(earlier: &OutboxOp, later: &OutboxOp) -> bool {
249    // Only an upsert carries values, and it always carries its primary key.
250    earlier.row_id == later.row_id
251        && earlier.table == later.table
252        && later.base_version.is_some_and(|base| base > 0)
253        && earlier.base_version == later.base_version
254        && later.values.as_ref().is_some_and(|values| {
255            values.keys().all(|column| {
256                earlier
257                    .values
258                    .as_ref()
259                    .is_some_and(|earlier| earlier.contains_key(column))
260            })
261        })
262}
263
264impl From<&OutboxOp> for CommitOperation {
265    fn from(operation: &OutboxOp) -> Self {
266        Self {
267            table: operation.table.clone(),
268            row_id: operation.row_id.clone(),
269            op: if operation.upsert { "upsert" } else { "delete" }.to_owned(),
270            base_version: operation.base_version,
271            values: operation.values.clone(),
272            insert: operation.insert,
273        }
274    }
275}
276
277#[derive(Debug, Clone)]
278struct OutboxCommit {
279    client_commit_id: String,
280    ops: Vec<OutboxOp>,
281}
282
283/// One commit's `(table, value keys, insert intent)` per upsert operation — the classification
284/// shape the §7.4.4 send-time drop and the RFC 0005 D6 audit share, so the two
285/// can never disagree about which commits cannot re-encode.
286fn commit_audit_operations(commit: &OutboxCommit) -> Vec<(&str, Vec<&str>, bool)> {
287    commit
288        .ops
289        .iter()
290        .filter(|operation| operation.upsert)
291        .map(|operation| {
292            (
293                operation.table.as_str(),
294                operation
295                    .values
296                    .as_ref()
297                    .map(|values| values.keys().map(String::as_str).collect())
298                    .unwrap_or_default(),
299                operation.insert,
300            )
301        })
302        .collect()
303}
304
305type FailedCommit = (OutboxCommit, Map<String, Value>);
306
307#[derive(Debug, Clone)]
308struct CompiledLocalDataPurgeTarget {
309    table: String,
310    selectors: Vec<(String, Vec<String>)>,
311}
312
313struct StoredCommitOutcomeRow {
314    sequence: i64,
315    client_commit_id: String,
316    status: String,
317    recorded_at_ms: i64,
318    results_json: String,
319    operations_json: Option<String>,
320    resolution: String,
321    resolved_at_ms: Option<i64>,
322    replacement_client_commit_id: Option<String>,
323}
324
325impl StoredCommitOutcomeRow {
326    fn from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result<Self> {
327        Ok(Self {
328            sequence: row.get(0)?,
329            client_commit_id: row.get(1)?,
330            status: row.get(2)?,
331            recorded_at_ms: row.get(3)?,
332            results_json: row.get(4)?,
333            operations_json: row.get(5)?,
334            resolution: row.get(6)?,
335            resolved_at_ms: row.get(7)?,
336            replacement_client_commit_id: row.get(8)?,
337        })
338    }
339}
340
341/// Section outcome distinguishing the §5.6 unit-local fail-closed
342/// path from a round-aborting failure (§1.4 rule 5).
343/// §4.4 a unit section's `SUB_START`.
344struct SectionStart {
345    id: String,
346    status: SubStatus,
347    reason_code: String,
348    bootstrap: bool,
349    fenced: Vec<FencedRecord>,
350}
351
352enum SectionError {
353    FailClosed,
354    /// §5.5: the server refused a segment with `sync.forbidden`.
355    Refused,
356    /// §5.3 client memory: the round deferred this image's body to a later
357    /// pull of the same resume token.
358    Deferred,
359    Abort(String, String),
360    Transport(TransportError),
361}
362
363#[derive(Debug)]
364pub(crate) struct RequestMeta {
365    pushed_ids: Vec<String>,
366    /// Hex of the registration digest this pull carried; the server stores
367    /// that list once the round completes (§4.1).
368    registration_digest: String,
369    pub(crate) accept: u8,
370    /// §6.1 splitBatch: outbox commits held back from THIS request because
371    /// the running operation count reached the push cap — the next round
372    /// pushes them (`sync_needed` stays set while any remain).
373    deferred_commits: usize,
374    /// §5.6 applied-segment record per unit: segment id → rows applied at
375    /// the unit's current cursor and token.
376    pub(crate) applied: BTreeMap<String, BTreeMap<String, i64>>,
377    /// §5.3 client memory: (unit, segment id) of the first image whose body
378    /// the round did not fetch because the batch already holds two images.
379    /// That unit's section stops there without persisting `SUB_END`.
380    pub(crate) deferred: BTreeSet<(String, String)>,
381}
382
383#[derive(Default)]
384struct ChangeAccumulator {
385    /// None means table-wide; Some is the exact scoped domain.
386    tables: BTreeMap<String, Option<BTreeSet<String>>>,
387    windows: BTreeMap<String, BTreeSet<String>>,
388    status: bool,
389    conflicts: bool,
390    rejections: bool,
391    outcomes: bool,
392}
393
394impl ChangeAccumulator {
395    fn table(&mut self, table: &str) {
396        self.tables.insert(table.to_owned(), None);
397    }
398
399    fn scope(&mut self, table: &str, key: String) {
400        match self.tables.get_mut(table) {
401            Some(None) => {}
402            Some(Some(keys)) => {
403                keys.insert(key);
404            }
405            None => {
406                self.tables
407                    .insert(table.to_owned(), Some(BTreeSet::from([key])));
408            }
409        }
410    }
411
412    fn window(&mut self, base_key: &str, value: &str) {
413        self.windows
414            .entry(base_key.to_owned())
415            .or_default()
416            .insert(value.to_owned());
417    }
418
419    fn touched(&self) -> bool {
420        !self.tables.is_empty()
421            || !self.windows.is_empty()
422            || self.status
423            || self.conflicts
424            || self.rejections
425            || self.outcomes
426    }
427}
428
429/// §6.1: the server caps total operations per request (reference default
430/// 500) and rejects the whole batch with `sync.too_many_operations`; the
431/// client "splits and retries". Splitting happens at build time: commits are
432/// included IN ORDER until the operation budget is spent, the rest wait for
433/// the next round.
434const PUSH_OPS_PER_REQUEST: usize = 500;
435const MAX_LOCAL_PURGE_TARGETS: usize = 64;
436const MAX_LOCAL_PURGE_SELECTORS: usize = 8;
437const MAX_LOCAL_PURGE_VALUES: usize = 128;
438const MAX_LOCAL_PURGE_VALUE_LENGTH: usize = 256;
439pub const SECURITY_PREFLIGHT_REQUIRED_CODE: &str = "client.security_preflight_required";
440
441/// Dirty visible tables; `None` means a whole-replica recovery/reset.
442struct OverlayDirty(RefCell<Option<BTreeSet<String>>>);
443
444impl OverlayDirty {
445    fn clean() -> Self {
446        Self(RefCell::new(Some(BTreeSet::new())))
447    }
448    fn get(&self) -> bool {
449        self.0
450            .borrow()
451            .as_ref()
452            .is_none_or(|tables| !tables.is_empty())
453    }
454    fn set(&self, dirty: bool) {
455        *self.0.borrow_mut() = if dirty { None } else { Some(BTreeSet::new()) };
456    }
457    fn table(&self, table: &str) {
458        if let Some(tables) = self.0.borrow_mut().as_mut() {
459            tables.insert(table.to_owned());
460        }
461    }
462    fn snapshot(&self) -> Option<BTreeSet<String>> {
463        self.0.borrow().clone()
464    }
465    fn restore(&self, tables: Option<BTreeSet<String>>) {
466        *self.0.borrow_mut() = tables;
467    }
468}
469
470/// A copy-on-write in-memory mirror: a [`Rollback`] shares it, and the first
471/// write after a capture copies it once.
472#[derive(Debug, Clone, Default)]
473struct Shared<T>(Arc<T>);
474
475impl<T> std::ops::Deref for Shared<T> {
476    type Target = T;
477    fn deref(&self) -> &T {
478        &self.0
479    }
480}
481
482impl<T: Clone> std::ops::DerefMut for Shared<T> {
483    fn deref_mut(&mut self) -> &mut T {
484        Arc::make_mut(&mut self.0)
485    }
486}
487
488impl<T> From<T> for Shared<T> {
489    fn from(value: T) -> Self {
490        Self(Arc::new(value))
491    }
492}
493
494/// The in-memory mirrors of durable state, captured before a local
495/// transaction and restored when it rolls back. Queues only grow inside a
496/// transaction, so their lengths suffice.
497#[derive(Debug, Clone)]
498struct Rollback {
499    outbox: Shared<Vec<Arc<OutboxCommit>>>,
500    failed_commits: Shared<Vec<FailedCommit>>,
501    conflicts: Shared<Vec<ConflictRecord>>,
502    rejections: Shared<Vec<RejectionRecord>>,
503    schema_floor: Option<SchemaFloor>,
504    lease_state: Option<LeaseState>,
505    stopped: bool,
506    upgrading: bool,
507    sync_needed: bool,
508    active_round: Option<uuid::Uuid>,
509    overlay_dirty: Option<BTreeSet<String>>,
510    change_queue: usize,
511    sync_intent_queue: usize,
512    last_change: Option<DiagnosticLastChange>,
513}
514
515pub struct SyncClient {
516    #[cfg(feature = "bench-internals")]
517    benchmark_phases: Recorder,
518    conn: Connection,
519    schema: ClientSchema,
520    client_id: String,
521    limits: ClientLimits,
522    outbox: Shared<Vec<Arc<OutboxCommit>>>,
523    failed_commits: Shared<Vec<FailedCommit>>,
524    retain_failed_commits: bool,
525    conflicts: Shared<Vec<ConflictRecord>>,
526    rejections: Shared<Vec<RejectionRecord>>,
527    schema_floor: Option<SchemaFloor>,
528    /// §7.3.5: the opaque auth-lease state (from LEASE frames + lease errors).
529    lease_state: Option<LeaseState>,
530    /// §1.6: the schema-floor response stops syncing until an upgrade.
531    stopped: bool,
532    /// §7.4.5: true while a schema-bump reset + first re-bootstrap is in flight.
533    upgrading: bool,
534    /// §8.4 coalesced sync-needed signal.
535    sync_needed: bool,
536    /// §8.8: how sync rounds treat the realtime binding, and its exact
537    /// availability state with the evidence behind a lost or refused one.
538    realtime_policy: RealtimePolicy,
539    realtime_state: RealtimeState,
540    realtime_reason_code: Option<String>,
541    realtime_retry_delay_ms: Option<u64>,
542    /// §8.6 presence: scopeKey → (`actorId clientId` peer key → peer).
543    presence: HashMap<String, HashMap<String, PresencePeer>>,
544    /// Client clock (epoch ms) for the §5.4 `urlExpiresAtMs` check and the
545    /// §7.8 drive deadline; the host may pin it and move the pinned value
546    /// (conformance runs on a virtual clock).
547    now_ms: Option<std::sync::Arc<std::sync::atomic::AtomicI64>>,
548    /// §5.11 client-side encryption keys (`keyId → key bytes`). Empty ⇒ E2EE
549    /// off. The encrypt/decrypt seam (`values.rs`) is compiled only under the
550    /// `e2ee` feature; without it, a schema with encrypted columns fails loud.
551    encryption: crate::values::EncryptionConfig,
552    /// Fail-closed host bootstrap gate. While set, command hosts permit only
553    /// status/lifecycle inspection and an exact authorized local purge.
554    security_preflight: bool,
555    security_epoch: u64,
556    transport_enabled: bool,
557    transport_epoch: u64,
558    /// Raised by every registration change; a current-base proof spanning
559    /// one reports missing coverage.
560    registration_epoch: u64,
561    /// RFC 0005 D2/D8: the previous-version capture config. `None` (default)
562    /// means the feature is off; the descriptor write and the container orphan
563    /// sweep still happen.
564    previous_version: Option<PreviousVersionContextConfig>,
565    /// Per-table primary-key upsert SQL, built once per (full table name) —
566    /// the row write path runs per row during bootstrap (§5.6), so the SQL
567    /// string (and, via `prepare_cached`, its compiled statement) is reused
568    /// instead of being rebuilt and re-prepared per row. Cleared on a §7.4.3
569    /// schema reset (the column lists may have changed).
570    insert_sql: RefCell<HashMap<String, String>>,
571    /// §7.1: reconcile only tables whose base/outbox changed. Whole-replica
572    /// reset and restart explicitly invalidate every table.
573    overlay_dirty: OverlayDirty,
574    // First SQLite failure of the current operation, before savepoint cleanup.
575    storage_failure: RefCell<Option<QueryReadFailure>>,
576    /// Test-only structural performance signal: response processing must not
577    /// turn a batch of acknowledgements into one full overlay rebuild each.
578    #[cfg(test)]
579    overlay_rebuild_count: Cell<usize>,
580    #[cfg(test)]
581    acknowledged_replay_count: Cell<usize>,
582    /// Test-only structural performance signal: outcome retention is enforced
583    /// once per response rather than once per acknowledgement.
584    #[cfg(test)]
585    outcome_prune_count: Cell<usize>,
586    /// Test-only pinned duration of every §5.3 image chunk, in ms.
587    #[cfg(test)]
588    image_chunk_ms: Cell<Option<f64>>,
589    /// Exact observer-transaction output drained by command/FFI hosts.
590    progress: ProgressObserver,
591    change_queue: VecDeque<ClientChangeBatch>,
592    sync_intent_queue: VecDeque<SyncIntent>,
593    /// Explicit exponential retry policy for transient transport failures.
594    retry_delay_ms: u64,
595    /// §7.7: the background retry the current round scheduled, if any.
596    round_retry_delay_ms: Option<u64>,
597    /// §5.3: a table's next image chunk size, carried from one range image
598    /// of the table to the next.
599    image_chunk_rows: HashMap<String, u32>,
600    /// A round whose response never completed (§1.5): the client keeps its
601    /// code and reports it on every following request until a round receives a
602    /// complete, decodable response. The server already recorded the round, so
603    /// nothing else can tell it the round failed on this side.
604    round_failure: Option<String>,
605    active_round: Option<uuid::Uuid>,
606    last_round: Option<DiagnosticLastRound>,
607    last_change: Option<DiagnosticLastChange>,
608    /// §7.6: latest failed owned snapshot read per owner id, oldest first.
609    query_failures: Vec<DiagnosticQueryFailure>,
610    /// SPEC §1.4 rule 4: the open coalesced apply transaction of a response.
611    coalesced: Option<CoalescedApply>,
612    /// A coalesced apply transaction of the current response failed; every
613    /// later observation of that response fails until it ends.
614    coalesced_failure: Option<String>,
615}
616
617/// SPEC §1.4 rule 4: the frames and blocks of one response share one local
618/// write transaction of at most `COALESCED_APPLY_ROWS` rows. The in-memory
619/// state captured at its `BEGIN` is restored when its `COMMIT` fails.
620struct CoalescedApply {
621    rows: usize,
622    rollback: Rollback,
623}
624
625const COALESCED_APPLY_ROWS: usize = 1024;
626
627pub(crate) fn quote_ident(name: &str) -> String {
628    format!("\"{}\"", name.replace('"', "\"\""))
629}
630/// §7.4.1 `_syncular_meta` helpers. Free functions so RFC 0005's
631/// previous-version module reads and writes the SAME records as the client
632/// instead of growing a parallel meta layer.
633pub(crate) fn meta_get(conn: &Connection, key: &str) -> rusqlite::Result<Option<String>> {
634    conn.query_row(
635        "SELECT value FROM _syncular_meta WHERE key = ?1",
636        rusqlite::params![key],
637        |row| row.get::<_, String>(0),
638    )
639    .optional()
640}
641
642pub(crate) fn meta_set(conn: &Connection, key: &str, value: &str) -> rusqlite::Result<()> {
643    conn.execute(
644        "INSERT OR REPLACE INTO _syncular_meta (key, value) VALUES (?1, ?2)",
645        rusqlite::params![key, value],
646    )
647    .map(|_| ())
648}
649
650pub(crate) fn meta_delete(conn: &Connection, key: &str) -> rusqlite::Result<()> {
651    conn.execute(
652        "DELETE FROM _syncular_meta WHERE key = ?1",
653        rusqlite::params![key],
654    )
655    .map(|_| ())
656}
657
658/// Seek by the existing key while retaining the exact legacy text match.
659fn row_id_predicate(table: &TableSchema) -> String {
660    let key = quote_ident(&table.primary_key);
661    let text_match = format!("CAST({key} AS TEXT) = ?1");
662    match table.columns[table.pk_index].ty {
663        ColumnType::String | ColumnType::Json => {
664            format!("{key} = ?1 AND {text_match}")
665        }
666        // Native columns have no declared SQLite affinity. Match the stored
667        // integer bind type explicitly. Unary + removes expression affinity so
668        // SQLite can seek the typeless key; the text check preserves exact IDs.
669        ColumnType::Integer | ColumnType::Boolean => {
670            format!("{key} = +CAST(?1 AS INTEGER) AND {text_match}")
671        }
672        // REAL-to-text conversion can round distinct values to the same text.
673        // Narrowing those matches by numeric equality would change behavior.
674        _ => text_match,
675    }
676}
677
678/// §3.3/§4.8/§5.6 row ownership: rows whose local scope columns match
679/// `effective` and that no entry of `owners` matches. `owners` holds the
680/// pair scopes of every owner of the table (§3.3 row ownership); a scoped delete never removes a row one of them still covers.
681/// Scope values bind by the column's declared type, as `row_id_predicate`
682/// does. `Ok(None)` matches nothing (an empty map or value list); `Err(())`
683/// is a missing local scope-column mapping (the fail-closed case).
684fn local_scope_predicate(
685    table: &TableSchema,
686    effective: &[(String, Vec<String>)],
687    owners: &[Vec<(String, Vec<String>)>],
688) -> Result<Option<(String, Vec<SqlValue>)>, ()> {
689    if effective.is_empty() || effective.iter().any(|(_, values)| values.is_empty()) {
690        return Ok(None);
691    }
692    let column = |variable: &str| {
693        let name = table.scope_column(variable).ok_or(())?;
694        let column = table
695            .columns
696            .iter()
697            .find(|column| column.name == name)
698            .ok_or(())?;
699        Ok((quote_ident(name), column.ty == ColumnType::Integer))
700    };
701    let mut clauses = Vec::new();
702    let mut params = Vec::new();
703    for (variable, values) in effective {
704        let (key, integer) = column(variable)?;
705        let holes = vec!["?"; values.len()].join(", ");
706        clauses.push(if integer {
707            params.extend(values.iter().cloned().map(SqlValue::Text));
708            let seek = vec!["+CAST(? AS INTEGER)"; values.len()].join(", ");
709            format!("{key} IN ({seek}) AND CAST({key} AS TEXT) IN ({holes})")
710        } else {
711            format!("{key} IN ({holes})")
712        });
713        params.extend(values.iter().cloned().map(SqlValue::Text));
714    }
715    // Owners bind as one JSON parameter per variable set, so the statement
716    // keeps its size at any owner count; owners of one variable bind the
717    // union of their values. An owner holding other values of a variable
718    // `effective` names owns none of its rows and drops out. Values compare
719    // as text, the exact semantics of the effective clauses.
720    let mut groups: BTreeMap<Vec<&str>, Vec<Vec<&[String]>>> = BTreeMap::new();
721    for owner in owners {
722        if owner.is_empty()
723            || owner.iter().any(|(variable, values)| {
724                values.is_empty()
725                    || effective.iter().any(|(name, wanted)| {
726                        name == variable && wanted.iter().all(|value| !values.contains(value))
727                    })
728            })
729        {
730            continue;
731        }
732        let mut owner: Vec<_> = owner.iter().collect();
733        owner.sort_by(|a, b| a.0.cmp(&b.0));
734        groups
735            .entry(
736                owner
737                    .iter()
738                    .map(|(variable, _)| variable.as_str())
739                    .collect(),
740            )
741            .or_default()
742            .push(owner.iter().map(|(_, values)| values.as_slice()).collect());
743    }
744    let within = |variable: &str, list: &str| {
745        let (key, integer) = column(variable)?;
746        let key = if integer {
747            format!("CAST({key} AS TEXT)")
748        } else {
749            key
750        };
751        Ok(format!("{key} IN (SELECT value FROM json_each({list}))"))
752    };
753    for (variables, tuples) in groups {
754        if let [variable] = variables[..] {
755            let values: BTreeSet<&String> = tuples.iter().flatten().copied().flatten().collect();
756            params.push(SqlValue::Text(
757                Value::from_iter(values.into_iter().cloned()).to_string(),
758            ));
759            clauses.push(format!("NOT {}", within(variable, "?")?));
760        } else {
761            params.push(SqlValue::Text(
762                serde_json::to_string(&tuples).expect("owner tuples serialize"),
763            ));
764            let matches = variables
765                .iter()
766                .enumerate()
767                .map(|(index, variable)| within(variable, &format!("o.value,'$[{index}]'")))
768                .collect::<Result<Vec<_>, ()>>()?;
769            clauses.push(format!(
770                "NOT EXISTS (SELECT 1 FROM json_each(?) o WHERE {})",
771                matches.join(" AND ")
772            ));
773        }
774    }
775    Ok(Some((clauses.join(" AND "), params)))
776}
777
778/// [`local_scope_predicate`] over decoded row values (retained rows).
779fn owns_local_row(
780    table: &TableSchema,
781    effective: &[(String, Vec<String>)],
782    owners: &[Vec<(String, Vec<String>)>],
783    values: &Map<String, Value>,
784) -> bool {
785    let matches = |scopes: &[(String, Vec<String>)]| {
786        !scopes.is_empty()
787            && scopes.iter().all(|(variable, allowed)| {
788                table
789                    .scope_column(variable)
790                    .and_then(|column| values.get(column))
791                    .filter(|value| !value.is_null())
792                    .is_some_and(|value| {
793                        allowed.contains(
794                            &value
795                                .as_str()
796                                .map_or_else(|| value.to_string(), str::to_owned),
797                        )
798                    })
799            })
800    };
801    matches(effective) && !owners.iter().any(|owner| matches(owner))
802}
803
804fn is_local_operation_code_like(value: &str) -> bool {
805    let bytes = value.as_bytes();
806    bytes.first().is_some_and(u8::is_ascii_alphanumeric)
807        && bytes
808            .iter()
809            .all(|byte| byte.is_ascii_alphanumeric() || matches!(*byte, b'.' | b'_' | b':' | b'-'))
810}
811
812fn base_table(name: &str) -> String {
813    quote_ident(&format!("_syncular_base_{name}"))
814}
815
816/// §4.8 deferred eviction record: (unit id, table, effective scope map).
817type PendingEvict = (String, String, Vec<(String, Vec<String>)>);
818
819/// §4.8: a stable key for a window base: variable + canonical fixed scopes.
820fn window_base_key(base: &WindowBase) -> String {
821    format!(
822        "{}\0{}",
823        base.variable,
824        ssp2::registration::canonical_unit_json(&scope_pairs(&base.fixed_scopes))
825    )
826}
827
828/// The unit of one window value: the fixed scopes plus the value.
829fn window_unit_scopes(base: &WindowBase, value: &str) -> UnitScopes {
830    let mut scopes = base.fixed_scopes.clone();
831    scopes.insert(base.variable.clone(), value.to_owned());
832    scopes
833}
834
835/// A unit's scope map as the codec and the digests take it.
836fn scope_pairs(scopes: &UnitScopes) -> Vec<(String, String)> {
837    let mut pairs: Vec<(String, String)> = scopes
838        .iter()
839        .map(|(variable, value)| (variable.clone(), value.clone()))
840        .collect();
841    pairs.sort_by(|a, b| a.0.encode_utf16().cmp(b.0.encode_utf16()));
842    pairs
843}
844
845/// §4.1: the unit covers every table declaring all of its variables.
846fn covers(table: &TableSchema, scopes: &UnitScopes) -> bool {
847    scopes
848        .keys()
849        .all(|variable| table.scope_column(variable).is_some())
850}
851
852/// §4.4: the effective scopes of the (unit, table) pair under `echo`: the
853/// scope map as one-element lists plus the fenced lists. `None` when the
854/// fence revoked the pair.
855fn pair_scopes(
856    scopes: &UnitScopes,
857    table: &str,
858    echo: Option<&[EchoRecord]>,
859) -> Option<Vec<(String, Vec<String>)>> {
860    let fenced = echo.and_then(|echo| echo.iter().find(|record| record.table == table));
861    if fenced.is_some_and(|record| record.status == "revoked") {
862        return None;
863    }
864    let mut effective: BTreeMap<String, Vec<String>> = fenced
865        .map(|record| record.scopes.clone())
866        .unwrap_or_default();
867    for (variable, value) in scopes {
868        effective.insert(variable.clone(), vec![value.clone()]);
869    }
870    Some(effective.into_iter().collect())
871}
872
873/// §3.3 row ownership: the pair scopes of every owner of `table` except
874/// `except_id`. An owner is active with a persisted echo and live or
875/// post-bootstrap, or bootstrapping with a segment of the table in its
876/// current bootstrap.
877fn unit_owners(
878    units: &[UnitRecord],
879    table: &TableSchema,
880    except_id: &str,
881) -> Vec<Vec<(String, Vec<String>)>> {
882    units
883        .iter()
884        .filter(|unit| {
885            unit.id != except_id
886                && unit.status == UnitStatus::Active
887                && unit.echo.is_some()
888                && (!unit.bootstrapping() || unit.seeded.contains(&table.name))
889                && covers(table, &unit.scopes)
890        })
891        .filter_map(|unit| pair_scopes(&unit.scopes, &table.name, unit.echo.as_deref()))
892        .collect()
893}
894
895/// §5.5 / §5.6: the unit's scope map, each value a one-element list.
896fn unit_scope_lists(scopes: &UnitScopes) -> Vec<(String, Vec<String>)> {
897    scopes
898        .iter()
899        .map(|(variable, value)| (variable.clone(), vec![value.clone()]))
900        .collect()
901}
902
903/// §7.1 acknowledged intent of the rows `?1` names (`[[table, id], ...]`).
904/// It joins through the (tbl, id) index: a scan per acknowledgement would
905/// make draining a batch quadratic.
906const ACKED_ROWS_BY_ROW: &str = "SELECT a.op_json,a.intent_json FROM json_each(?1) k JOIN _syncular_acked_rows a ON a.tbl=json_extract(k.value,'$[0]') AND a.id=json_extract(k.value,'$[1]') WHERE ?2 IS NULL OR a.tbl IN(SELECT value FROM json_each(?2)) ORDER BY a.commit_seq,a.idx";
907
908const UNIT_COLUMNS: &str =
909    "id, scopes, explicit, status, reason_code, cursor, bootstrap_state, seeded, echo";
910
911/// A unit row; a field that does not parse is a conversion failure, which
912/// both read paths classify as `sync.local_corrupt`.
913fn unit_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result<UnitRecord> {
914    fn json<T: serde::de::DeserializeOwned>(index: usize, raw: &str) -> rusqlite::Result<T> {
915        serde_json::from_str(raw).map_err(|error| {
916            rusqlite::Error::FromSqlConversionFailure(
917                index,
918                rusqlite::types::Type::Text,
919                Box::new(error),
920            )
921        })
922    }
923    Ok(UnitRecord {
924        id: row.get(0)?,
925        scopes: json(1, &row.get::<_, String>(1)?)?,
926        explicit: row.get::<_, i64>(2)? == 1,
927        status: json(
928            3,
929            &serde_json::to_string(&row.get::<_, String>(3)?).expect("text"),
930        )?,
931        reason_code: row.get(4)?,
932        cursor: row.get(5)?,
933        bootstrap_state: row.get(6)?,
934        seeded: json(7, &row.get::<_, String>(7)?)?,
935        echo: row
936            .get::<_, Option<String>>(8)?
937            .map(|echo| json::<Vec<EchoRecord>>(8, &echo))
938            .transpose()?
939            .map(|echo| {
940                if echo
941                    .iter()
942                    .all(|record| record.status == "active" || record.status == "revoked")
943                {
944                    Ok(echo)
945                } else {
946                    Err(rusqlite::Error::FromSqlConversionFailure(
947                        8,
948                        rusqlite::types::Type::Text,
949                        "echo status is neither active nor revoked".into(),
950                    ))
951                }
952            })
953            .transpose()?,
954    })
955}
956
957/// Every unit in registration order (§4.1).
958fn load_units(conn: &Connection) -> rusqlite::Result<Vec<UnitRecord>> {
959    conn.prepare_cached(&format!(
960        "SELECT {UNIT_COLUMNS} FROM _syncular_units ORDER BY rowid"
961    ))?
962    .query_map([], unit_from_row)?
963    .collect()
964}
965
966fn get_unit(conn: &Connection, id: &str) -> rusqlite::Result<Option<UnitRecord>> {
967    conn.query_row(
968        &format!("SELECT {UNIT_COLUMNS} FROM _syncular_units WHERE id = ?1"),
969        [id],
970        unit_from_row,
971    )
972    .optional()
973}
974
975/// Insert or update in place: an update keeps the registration order. The
976/// §5.6 applied-segment record ends when the cursor or the token changes.
977fn save_unit(conn: &Connection, unit: &UnitRecord) -> rusqlite::Result<()> {
978    conn.prepare_cached(&format!(
979        "INSERT INTO _syncular_units({UNIT_COLUMNS}) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9)
980         ON CONFLICT(id) DO UPDATE SET explicit=excluded.explicit, status=excluded.status,
981           reason_code=excluded.reason_code, cursor=excluded.cursor,
982           bootstrap_state=excluded.bootstrap_state, seeded=excluded.seeded, echo=excluded.echo,
983           applied=CASE WHEN cursor IS excluded.cursor
984             AND bootstrap_state IS excluded.bootstrap_state THEN applied END"
985    ))?
986    .execute(rusqlite::params![
987        unit.id,
988        serde_json::to_string(&unit.scopes).expect("scope map serializes"),
989        i64::from(unit.explicit),
990        unit.status.name(),
991        unit.reason_code,
992        unit.cursor,
993        unit.bootstrap_state,
994        serde_json::to_string(&unit.seeded).expect("table list serializes"),
995        unit.echo
996            .as_ref()
997            .map(|echo| serde_json::to_string(echo).expect("echo serializes")),
998    ])
999    .map(|_| ())
1000}
1001
1002/// §5.6 applied-segment record of unit `id`: segment id → rows applied at
1003/// its current cursor and token.
1004fn applied_rows(conn: &Connection, id: &str) -> rusqlite::Result<BTreeMap<String, i64>> {
1005    conn.query_row(
1006        "SELECT applied FROM _syncular_units WHERE id = ?1",
1007        [id],
1008        |row| row.get::<_, Option<String>>(0),
1009    )
1010    .optional()?
1011    .flatten()
1012    .map_or(Ok(BTreeMap::new()), |raw| {
1013        serde_json::from_str(&raw).map_err(|error| {
1014            rusqlite::Error::FromSqlConversionFailure(
1015                0,
1016                rusqlite::types::Type::Text,
1017                Box::new(error),
1018            )
1019        })
1020    })
1021}
1022
1023/// §5.6: `rows` rows of `segment_id` applied at unit `id`'s position. Runs
1024/// in the transaction that commits them.
1025fn record_applied_rows(
1026    conn: &Connection,
1027    id: &str,
1028    segment_id: &str,
1029    rows: i64,
1030) -> rusqlite::Result<()> {
1031    conn.prepare_cached(
1032        "UPDATE _syncular_units SET applied=json_patch(coalesce(applied,'{}'),?1) WHERE id=?2",
1033    )?
1034    .execute(rusqlite::params![
1035        serde_json::json!({ segment_id: rows }).to_string(),
1036        id
1037    ])
1038    .map(|_| ())
1039}
1040
1041/// §4.1 references: every window registry entry `(base, value)` holding a unit.
1042fn window_refs(conn: &Connection, unit_id: &str) -> rusqlite::Result<Vec<(String, String)>> {
1043    conn.prepare_cached("SELECT base, unit FROM _syncular_windows WHERE sub_id = ?1")?
1044        .query_map([unit_id], |row| Ok((row.get(0)?, row.get(1)?)))?
1045        .collect()
1046}
1047
1048/// §7.4.3 reset, §2.1 log reset and local rebootstrap: keep the unit
1049/// registrations and discard the synced state of every unit outside `keep`,
1050/// so the next round sends the list and fresh-bootstraps those units. The
1051/// replica cursor survives only while a kept unit is live. The caller owns
1052/// the transaction.
1053fn reset_units(conn: &Connection, keep: &BTreeSet<String>) -> rusqlite::Result<()> {
1054    conn.execute(
1055        "UPDATE _syncular_units SET status='active', reason_code=NULL, cursor=-1,
1056           bootstrap_state=NULL, seeded='[]', echo=NULL, applied=NULL
1057         WHERE id NOT IN (SELECT value FROM json_each(?1))",
1058        [serde_json::to_string(keep).expect("unit ids serialize")],
1059    )?;
1060    conn.execute_batch(
1061        "DELETE FROM _syncular_meta WHERE key IN ('echoDigest','registrationDigest');
1062         DELETE FROM _syncular_meta WHERE key='replicaCursor'
1063           AND NOT EXISTS (SELECT 1 FROM _syncular_units WHERE cursor IS NULL);",
1064    )
1065}
1066
1067/// §4.5: the replica cursor, `-1` while no unit is live.
1068fn replica_cursor(conn: &Connection) -> rusqlite::Result<i64> {
1069    meta_get(conn, REPLICA_CURSOR_KEY)?.map_or(Ok(-1), |raw| {
1070        raw.parse().map_err(|error| {
1071            rusqlite::Error::FromSqlConversionFailure(
1072                0,
1073                rusqlite::types::Type::Text,
1074                Box::new(error),
1075            )
1076        })
1077    })
1078}
1079
1080fn visible_table(name: &str) -> String {
1081    quote_ident(name)
1082}
1083
1084const FTS_SOURCE_ID_COLUMN: &str = "_syncular_source_id";
1085
1086/// §7.4.3: is a `sqlite_master` table name a synced table (visible or base),
1087/// i.e. NOT one of the durable bookkeeping tables the reset preserves?
1088fn is_synced_table_name(name: &str) -> bool {
1089    if name.starts_with("sqlite_") {
1090        return false;
1091    }
1092    if name.starts_with("_syncular_base_") {
1093        return true; // the base half of a synced table pair
1094    }
1095    // Bookkeeping: outbox, units, meta, blob cache/uploads.
1096    !name.starts_with("_syncular_")
1097}
1098
1099/// `"sha256:" + hex` of the bytes — the content address (§5.9.1).
1100fn blob_id_for(bytes: &[u8]) -> String {
1101    let digest = Sha256::digest(bytes);
1102    format!("sha256:{}", bytes_to_hex(&digest))
1103}
1104
1105/// One [`SyncClient::write_row`] bind parameter, borrowing the row-codec
1106/// value it wraps — strings/JSON/bytes bind as borrowed TEXT/BLOB (no copy
1107/// per row on the §5.6 bootstrap path), scalars bind owned.
1108enum RowParam<'a> {
1109    Cell(&'a Option<ColumnValue>),
1110    Version(i64),
1111    Text(Option<&'a str>),
1112}
1113
1114impl rusqlite::ToSql for RowParam<'_> {
1115    fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
1116        Ok(match self {
1117            RowParam::Version(v) => ToSqlOutput::Owned(SqlValue::Integer(*v)),
1118            RowParam::Text(text) => match text {
1119                Some(text) => ToSqlOutput::Borrowed(ValueRef::Text(text.as_bytes())),
1120                None => ToSqlOutput::Owned(SqlValue::Null),
1121            },
1122            RowParam::Cell(cell) => match cell {
1123                None => ToSqlOutput::Owned(SqlValue::Null),
1124                Some(ColumnValue::String(s)) => ToSqlOutput::Borrowed(ValueRef::Text(s.as_bytes())),
1125                Some(ColumnValue::Integer(i)) => ToSqlOutput::Owned(SqlValue::Integer(*i)),
1126                Some(ColumnValue::Float(f)) => ToSqlOutput::Owned(SqlValue::Real(*f)),
1127                Some(ColumnValue::Boolean(b)) => {
1128                    ToSqlOutput::Owned(SqlValue::Integer(i64::from(*b)))
1129                }
1130                Some(ColumnValue::Json(raw)) | Some(ColumnValue::BlobRef(raw)) => {
1131                    ToSqlOutput::Borrowed(ValueRef::Text(raw.0.as_bytes()))
1132                }
1133                // §5.10: crdt bytes store as BLOB, like bytes.
1134                Some(ColumnValue::Bytes(b)) | Some(ColumnValue::Crdt(b)) => {
1135                    ToSqlOutput::Borrowed(ValueRef::Blob(b))
1136                }
1137            },
1138        })
1139    }
1140}
1141
1142// Retain a typed primary key across the current import chunk without copying row payloads.
1143fn owned_sql_value(param: impl rusqlite::ToSql) -> Result<SqlValue, String> {
1144    match param.to_sql().map_err(|error| error.to_string())? {
1145        ToSqlOutput::Owned(value) => Ok(value),
1146        ToSqlOutput::Borrowed(value) => {
1147            SqlValue::try_from(value).map_err(|error| error.to_string())
1148        }
1149        _ => Err("sync.invalid_request: unsupported SQLite key parameter".into()),
1150    }
1151}
1152
1153/// §5.3 image cell → bind parameter, strict per the declared column type
1154/// (`boolean` from INTEGER 0/1, `json` from its raw TEXT, NULL only when
1155/// nullable). Mismatches are image-producer violations. Returns the cell
1156/// borrowed when the stored representation already matches what the row
1157/// codec would write, or the normalized scalar (boolean → 0/1, float from
1158/// INTEGER → REAL) otherwise — the local write is byte-identical to the
1159/// old convert-then-insert path without allocating per cell.
1160fn image_cell_param<'a>(column: &Column, value: ValueRef<'a>) -> Result<ToSqlOutput<'a>, String> {
1161    use ssp2::segment::ColumnType;
1162    let mismatch = || {
1163        Err(format!(
1164            "image column {:?} holds a value of the wrong type",
1165            column.name
1166        ))
1167    };
1168    match value {
1169        ValueRef::Null => {
1170            if !column.nullable {
1171                return Err(format!(
1172                    "image column {:?} is NULL but not nullable",
1173                    column.name
1174                ));
1175            }
1176            Ok(ToSqlOutput::Owned(SqlValue::Null))
1177        }
1178        ValueRef::Integer(i) => match column.ty {
1179            ColumnType::Integer => Ok(ToSqlOutput::Borrowed(value)),
1180            ColumnType::Boolean => Ok(ToSqlOutput::Owned(SqlValue::Integer(i64::from(i != 0)))),
1181            ColumnType::Float => Ok(ToSqlOutput::Owned(SqlValue::Real(i as f64))),
1182            _ => mismatch(),
1183        },
1184        ValueRef::Real(_) => match column.ty {
1185            ColumnType::Float => Ok(ToSqlOutput::Borrowed(value)),
1186            _ => mismatch(),
1187        },
1188        ValueRef::Text(t) => {
1189            std::str::from_utf8(t)
1190                .map_err(|_| format!("image column {:?} is not UTF-8", column.name))?;
1191            match column.ty {
1192                ColumnType::String | ColumnType::Json | ColumnType::BlobRef => {
1193                    Ok(ToSqlOutput::Borrowed(value))
1194                }
1195                _ => mismatch(),
1196            }
1197        }
1198        ValueRef::Blob(_) => match column.ty {
1199            // §5.10: a crdt column stores its opaque bytes as BLOB, like bytes.
1200            ColumnType::Bytes | ColumnType::Crdt => Ok(ToSqlOutput::Borrowed(value)),
1201            _ => mismatch(),
1202        },
1203    }
1204}
1205
1206pub(crate) fn sql_ref_to_json(column: &Column, value: rusqlite::types::ValueRef<'_>) -> Value {
1207    use rusqlite::types::ValueRef;
1208    match value {
1209        ValueRef::Null => Value::Null,
1210        ValueRef::Integer(i) => match column.ty {
1211            ssp2::segment::ColumnType::Boolean => Value::Bool(i != 0),
1212            ssp2::segment::ColumnType::Float => {
1213                serde_json::Number::from_f64(i as f64).map_or(Value::Null, Value::Number)
1214            }
1215            _ => Value::from(i),
1216        },
1217        ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
1218        ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
1219        ValueRef::Blob(b) => {
1220            let mut map = Map::new();
1221            map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
1222            Value::Object(map)
1223        }
1224    }
1225}
1226
1227/// Bind a driver JSON value form as a rusqlite parameter for [`SyncClient::query`].
1228/// Objects are accepted in lossless `{"$bytes": hex}` and
1229/// `{"$bigint": decimal}` envelope forms.
1230fn json_param_to_sql(value: &Value) -> Result<SqlValue, String> {
1231    Ok(match value {
1232        Value::Null => SqlValue::Null,
1233        Value::Bool(b) => SqlValue::Integer(i64::from(*b)),
1234        Value::Number(n) => {
1235            if let Some(i) = n.as_i64() {
1236                SqlValue::Integer(i)
1237            } else if let Some(f) = n.as_f64() {
1238                SqlValue::Real(f)
1239            } else {
1240                return Err(format!("query param number {n} is out of range"));
1241            }
1242        }
1243        Value::String(s) => SqlValue::Text(s.clone()),
1244        Value::Object(_) => {
1245            if let Some(hex) = value.get("$bytes").and_then(Value::as_str) {
1246                SqlValue::Blob(crate::values::hex_to_bytes(hex)?)
1247            } else if let Some(decimal) = value.get("$bigint").and_then(Value::as_str) {
1248                SqlValue::Integer(decimal.parse::<i64>().map_err(|_| {
1249                    format!("query bigint param {decimal:?} is outside SQLite's i64 range")
1250                })?)
1251            } else {
1252                return Err(
1253                    "query object param must be a {$bytes: hex} or {$bigint: decimal} value"
1254                        .to_owned(),
1255                );
1256            }
1257        }
1258        Value::Array(_) => return Err("query array params are not supported".to_owned()),
1259    })
1260}
1261
1262/// Map a rusqlite value with no schema column to consult (arbitrary query
1263/// output): integers/reals/text pass through by stored affinity, blobs ride
1264/// as `{"$bytes": hex}`. Distinct from [`sql_ref_to_json`], which uses the
1265/// schema column type to recover booleans/floats/json.
1266fn sql_ref_to_json_dynamic(value: rusqlite::types::ValueRef<'_>) -> Value {
1267    use rusqlite::types::ValueRef;
1268    match value {
1269        ValueRef::Null => Value::Null,
1270        ValueRef::Integer(i) => {
1271            // JSON/Tauri IPC cannot represent every SQLite i64 exactly. Keep
1272            // ordinary UI-sized integers ergonomic and envelope only values
1273            // beyond JavaScript's safe range.
1274            const MAX_SAFE_INTEGER: i64 = 9_007_199_254_740_991;
1275            if (-MAX_SAFE_INTEGER..=MAX_SAFE_INTEGER).contains(&i) {
1276                Value::from(i)
1277            } else {
1278                let mut map = Map::new();
1279                map.insert("$bigint".to_owned(), Value::from(i.to_string()));
1280                Value::Object(map)
1281            }
1282        }
1283        ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
1284        ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
1285        ValueRef::Blob(b) => {
1286            let mut map = Map::new();
1287            map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
1288            Value::Object(map)
1289        }
1290    }
1291}
1292
1293/// §7.5: SQLite failures classify by result code into [`QueryReadFailure`].
1294fn query_connection(
1295    conn: &Connection,
1296    sql: &str,
1297    params: &[QueryValue],
1298) -> Result<Vec<QueryRow>, QueryReadFailure> {
1299    let lowered_sql = crate::query_guard::lower_public_query_sql(sql);
1300    let mut stmt = crate::query_guard::prepare_read_only(conn, &lowered_sql)?;
1301    let bound: Vec<SqlValue> = params
1302        .iter()
1303        .map(json_param_to_sql)
1304        .collect::<Result<_, _>>()?;
1305    // §2.4: an unaliased physical version column stays engine-internal, so a
1306    // `SELECT *` row feeds straight back into mutation values. An explicit
1307    // alias (`_sync_version AS v`) returns it as ordinary query data. The
1308    // §5.11 sealed columns follow the same rule.
1309    let columns: Vec<(usize, String)> = stmt
1310        .column_names()
1311        .enumerate()
1312        .filter(|(_, name)| {
1313            !matches!(
1314                *name,
1315                "_syncular_version" | "_sync_sealed_key" | "_sync_sealed"
1316            )
1317        })
1318        .map(|(i, name)| (i, name.to_owned()))
1319        .collect();
1320    let bound_refs: Vec<&dyn rusqlite::ToSql> =
1321        bound.iter().map(|v| v as &dyn rusqlite::ToSql).collect();
1322    let mut sql_rows = stmt.query(bound_refs.as_slice())?;
1323    let mut out = Vec::new();
1324    while let Some(row) = sql_rows.next()? {
1325        let mut record = Map::new();
1326        for (i, name) in &columns {
1327            let value = row.get_ref(*i)?;
1328            record.insert(name.clone(), sql_ref_to_json_dynamic(value));
1329        }
1330        out.push(record);
1331    }
1332    Ok(out)
1333}
1334
1335fn persisted_window_state(
1336    conn: &Connection,
1337    base: &WindowBase,
1338) -> Result<WindowState, QueryReadFailure> {
1339    let mut stmt = conn.prepare(
1340        "SELECT windows.unit, units.status, units.cursor, units.bootstrap_state
1341           FROM _syncular_windows AS windows
1342           LEFT JOIN _syncular_units AS units ON units.id = windows.sub_id
1343          WHERE windows.base = ?1
1344          ORDER BY windows.unit ASC",
1345    )?;
1346    let rows = stmt.query_map(rusqlite::params![window_base_key(base)], |row| {
1347        Ok((
1348            row.get::<_, String>(0)?,
1349            row.get::<_, Option<String>>(1)?,
1350            row.get::<_, Option<i64>>(2)?,
1351            row.get::<_, Option<String>>(3)?,
1352        ))
1353    })?;
1354    let mut units = Vec::new();
1355    let mut pending = Vec::new();
1356    for row in rows {
1357        let (unit, status, cursor, bootstrap_state) = row?;
1358        // §4.8: complete iff registered, active and not bootstrapping.
1359        if status.as_deref() != Some("active")
1360            || cursor.is_some_and(|cursor| cursor < 0 || bootstrap_state.is_some())
1361        {
1362            pending.push(unit.clone());
1363        }
1364        units.push(unit);
1365    }
1366    Ok(WindowState { units, pending })
1367}
1368
1369/// One read-only statement in a §7.5 snapshot read.
1370#[derive(Debug, Clone, Copy)]
1371pub struct SnapshotStatement<'a> {
1372    pub sql: &'a str,
1373    pub params: &'a [Value],
1374}
1375
1376/// A §7.5 atomic read request: read-only statements, window coverage, unit
1377/// catch-up states, and commit deliveries, all resolved from one SQLite read
1378/// transaction and one revision.
1379#[derive(Debug, Clone, Default)]
1380pub struct SnapshotReadRequest<'a> {
1381    pub statements: Vec<SnapshotStatement<'a>>,
1382    pub coverage: &'a [WindowCoverage],
1383    pub units: Vec<String>,
1384    pub commit_ids: Vec<String>,
1385}
1386
1387#[cfg(test)]
1388thread_local! {
1389    /// Test-only rendezvous that runs between the statements of one snapshot
1390    /// read. A test uses it to commit on another connection mid-read and prove
1391    /// the read transaction's snapshot and revision stay stable.
1392    static SNAPSHOT_READ_HOOK: std::cell::RefCell<Option<Box<dyn FnOnce()>>> =
1393        std::cell::RefCell::new(None);
1394}
1395
1396#[cfg(test)]
1397fn run_snapshot_read_hook() {
1398    SNAPSHOT_READ_HOOK.with(|slot| {
1399        if let Some(hook) = slot.borrow_mut().take() {
1400            hook();
1401        }
1402    });
1403}
1404
1405/// §7.5: resolve a request over one SQLite read transaction. The revision,
1406/// every statement's rows, coverage, catch-up states, and deliveries are
1407/// captured under one `SAVEPOINT`; any statement's failure rolls the whole
1408/// read back and releases the connection.
1409fn snapshot_read_connection(
1410    conn: &Connection,
1411    request: &SnapshotReadRequest<'_>,
1412) -> Result<SnapshotRead, QueryReadFailure> {
1413    conn.execute_batch("SAVEPOINT syncular_snapshot_read")?;
1414    let corrupt = |message: &str| QueryReadFailure {
1415        code: Some("sync.local_corrupt"),
1416        sqlite_code: None,
1417        sqlite_message: None,
1418        rollback_failure: None,
1419        message: message.to_owned(),
1420    };
1421    let stored_failure = |error: rusqlite::Error, message: &str| match error {
1422        rusqlite::Error::Utf8Error(..)
1423        | rusqlite::Error::InvalidColumnType(..)
1424        | rusqlite::Error::FromSqlConversionFailure(..)
1425        | rusqlite::Error::IntegralValueOutOfRange(..) => corrupt(message),
1426        _ => QueryReadFailure::from(error),
1427    };
1428    let result: Result<SnapshotRead, QueryReadFailure> = (|| {
1429        // The durable revision as a canonical decimal `u64`; a missing marker
1430        // is the legacy zero, and any other non-canonical value is corrupt.
1431        let revision: Option<String> = conn
1432            .query_row(
1433                "SELECT value FROM _syncular_meta WHERE key = ?1",
1434                rusqlite::params![LOCAL_REVISION_KEY],
1435                |row| row.get(0),
1436            )
1437            .optional()
1438            .map_err(|error| stored_failure(error, "persisted local revision is invalid"))?;
1439        let revision = match revision {
1440            None => "0".to_owned(),
1441            Some(value)
1442                if value == "0"
1443                    || (matches!(value.as_bytes().first(), Some(b'1'..=b'9'))
1444                        && value.bytes().all(|b| b.is_ascii_digit())
1445                        && value.parse::<u64>().is_ok()) =>
1446            {
1447                value
1448            }
1449            Some(_) => return Err(corrupt("persisted local revision is invalid")),
1450        };
1451        let mut queries = Vec::with_capacity(request.statements.len());
1452        for statement in &request.statements {
1453            queries.push(query_connection(conn, statement.sql, statement.params)?);
1454            #[cfg(test)]
1455            run_snapshot_read_hook();
1456        }
1457        let mut pending = Vec::new();
1458        let mut missing = Vec::new();
1459        for requested in request.coverage {
1460            let base_key = window_base_key(&requested.base);
1461            let state = persisted_window_state(conn, &requested.base)?;
1462            for unit in BTreeSet::from_iter(requested.units.iter().cloned()) {
1463                let reference = WindowUnitRef {
1464                    base_key: base_key.clone(),
1465                    unit: unit.clone(),
1466                };
1467                if !state.units.iter().any(|held| held == &unit) {
1468                    missing.push(reference);
1469                } else if state.pending.iter().any(|held| held == &unit) {
1470                    pending.push(reference);
1471                }
1472            }
1473        }
1474        let mut units = Vec::with_capacity(request.units.len());
1475        for id in &request.units {
1476            let exists: bool = conn.query_row(
1477                "SELECT EXISTS(SELECT 1 FROM _syncular_units WHERE id = ?1)",
1478                [id],
1479                |row| row.get(0),
1480            )?;
1481            if !exists {
1482                units.push(UnitCatchup::Unknown { id: id.clone() });
1483                continue;
1484            }
1485            // Every persisted field is required to be well typed; a corrupt
1486            // record fails typed instead of inventing a state.
1487            let unit = get_unit(conn, id)
1488                .map_err(|error| stored_failure(error, "persisted unit state is invalid"))?
1489                .ok_or_else(|| corrupt("persisted unit state is invalid"))?;
1490            let cursor = match unit.cursor {
1491                Some(cursor) => cursor,
1492                None => replica_cursor(conn)
1493                    .map_err(|error| stored_failure(error, "persisted unit state is invalid"))?,
1494            };
1495            if cursor.unsigned_abs() > MAX_JS_SAFE_INTEGER {
1496                return Err(corrupt("persisted unit state is invalid"));
1497            }
1498            let active = unit.status == UnitStatus::Active;
1499            let pending = unit.bootstrapping();
1500            units.push(UnitCatchup::Known {
1501                id: id.clone(),
1502                status: unit.status,
1503                live: unit.cursor.is_none(),
1504                cursor,
1505                has_resume_token: unit.bootstrap_state.is_some(),
1506                bootstrap_complete: active && !pending,
1507                known_pending_pages: active && pending,
1508                reason_code: unit.reason_code,
1509            });
1510        }
1511        let mut deliveries = Vec::with_capacity(request.commit_ids.len());
1512        for id in &request.commit_ids {
1513            let pending: bool = conn.query_row(
1514                "SELECT EXISTS(SELECT 1 FROM _syncular_outbox WHERE commit_id = ?1)",
1515                rusqlite::params![id],
1516                |row| row.get(0),
1517            )?;
1518            if pending {
1519                deliveries.push(CommitDelivery::Pending {
1520                    client_commit_id: id.clone(),
1521                });
1522                continue;
1523            }
1524            let row = conn
1525                .query_row(
1526                    "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
1527                            resolution, resolved_at_ms, replacement_client_commit_id
1528                       FROM _syncular_commit_outcomes WHERE client_commit_id = ?1",
1529                    rusqlite::params![id],
1530                    StoredCommitOutcomeRow::from_row,
1531                )
1532                .optional()
1533                .map_err(|error| stored_failure(error, "persisted commit outcome is invalid"))?;
1534            deliveries.push(match row {
1535                Some(row) => CommitDelivery::Known {
1536                    client_commit_id: id.clone(),
1537                    outcome: SyncClient::outcome_from_row(row)
1538                        .map_err(|_| corrupt("persisted commit outcome is invalid"))?,
1539                },
1540                None => CommitDelivery::Unknown {
1541                    client_commit_id: id.clone(),
1542                },
1543            });
1544        }
1545        Ok(SnapshotRead {
1546            revision,
1547            queries,
1548            coverage: CoverageSnapshot {
1549                complete: pending.is_empty() && missing.is_empty(),
1550                pending,
1551                missing,
1552            },
1553            units,
1554            deliveries,
1555        })
1556    })();
1557    match result {
1558        Ok(read) => {
1559            conn.execute_batch("RELEASE syncular_snapshot_read")?;
1560            Ok(read)
1561        }
1562        Err(mut error) => {
1563            if let Err(rollback) = conn
1564                .execute_batch("ROLLBACK TO syncular_snapshot_read; RELEASE syncular_snapshot_read")
1565            {
1566                error.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
1567            }
1568            Err(error)
1569        }
1570    }
1571}
1572
1573fn snapshot_connection(
1574    conn: &Connection,
1575    sql: &str,
1576    params: &[Value],
1577    coverage: &[WindowCoverage],
1578) -> Result<QuerySnapshot, QueryReadFailure> {
1579    let read = snapshot_read_connection(
1580        conn,
1581        &SnapshotReadRequest {
1582            statements: vec![SnapshotStatement { sql, params }],
1583            coverage,
1584            units: Vec::new(),
1585            commit_ids: Vec::new(),
1586        },
1587    )?;
1588    Ok(QuerySnapshot {
1589        revision: read.revision,
1590        rows: read
1591            .queries
1592            .into_iter()
1593            .next()
1594            .expect("single-query snapshot returns one result"),
1595        coverage: read.coverage,
1596    })
1597}
1598
1599/// A long-lived read-only SQLite sidecar for latency-critical native views.
1600/// Network rounds stay serialized on the mutable core owner, while atomic
1601/// query snapshots use this independent connection and therefore never queue
1602/// behind HTTP/WebSocket latency.
1603pub struct FileQuerySnapshotReader {
1604    path: String,
1605    conn: Option<Connection>,
1606}
1607
1608impl FileQuerySnapshotReader {
1609    #[must_use]
1610    pub fn new(path: impl Into<String>) -> Self {
1611        Self {
1612            path: path.into(),
1613            conn: None,
1614        }
1615    }
1616
1617    fn connection(&mut self) -> Result<&Connection, QueryReadFailure> {
1618        if self.conn.is_none() {
1619            let conn = Connection::open_with_flags(
1620                &self.path,
1621                OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
1622            )
1623            .map_err(QueryReadFailure::from)?;
1624            conn.busy_timeout(std::time::Duration::from_millis(250))
1625                .map_err(QueryReadFailure::from)?;
1626            self.conn = Some(conn);
1627        }
1628        self.conn
1629            .as_ref()
1630            .ok_or_else(|| QueryReadFailure::from("read sidecar connection missing".to_owned()))
1631    }
1632
1633    /// One atomic read over the shared primitive: every statement, coverage,
1634    /// catch-up, and delivery under one `SAVEPOINT` and one revision.
1635    pub fn snapshot_read(
1636        &mut self,
1637        request: &SnapshotReadRequest<'_>,
1638    ) -> Result<SnapshotRead, QueryReadFailure> {
1639        snapshot_read_connection(self.connection()?, request)
1640    }
1641
1642    /// The current durable local revision. A host can skip a read when it is
1643    /// unchanged; a corrupt or unreadable marker fails typed.
1644    pub fn local_revision(&mut self) -> Result<String, QueryReadFailure> {
1645        Ok(self
1646            .snapshot_read(&SnapshotReadRequest::default())?
1647            .revision)
1648    }
1649
1650    /// One unit's persisted catch-up state (see the §7.5 contract).
1651    pub fn unit_catchup(&mut self, id: &str) -> Result<UnitCatchup, QueryReadFailure> {
1652        let read = self.snapshot_read(&SnapshotReadRequest {
1653            units: vec![id.to_owned()],
1654            ..Default::default()
1655        })?;
1656        Ok(read
1657            .units
1658            .into_iter()
1659            .next()
1660            .expect("one requested catch-up"))
1661    }
1662
1663    /// One client commit's delivery status: pending, its retained outcome, or
1664    /// an explicit unknown (never held, or pruned by retention).
1665    pub fn commit_delivery(&mut self, id: &str) -> Result<CommitDelivery, QueryReadFailure> {
1666        let read = self.snapshot_read(&SnapshotReadRequest {
1667            commit_ids: vec![id.to_owned()],
1668            ..Default::default()
1669        })?;
1670        Ok(read
1671            .deliveries
1672            .into_iter()
1673            .next()
1674            .expect("one requested delivery"))
1675    }
1676
1677    /// Hosts report an owned failure, and the first success after it, to the
1678    /// owning core through [`SyncClient::record_query_read`] (§7.6).
1679    pub fn query_snapshot(
1680        &mut self,
1681        sql: &str,
1682        params: &[Value],
1683        coverage: &[WindowCoverage],
1684    ) -> Result<QuerySnapshot, QueryReadFailure> {
1685        snapshot_connection(self.connection()?, sql, params, coverage)
1686    }
1687}
1688
1689fn validate_authority_reads(
1690    schema: &ClientSchema,
1691    declarations: &[crate::api::AuthorityReadDeclaration],
1692) -> Result<(), String> {
1693    let mut seen = BTreeSet::new();
1694    for read in declarations {
1695        let table = schema
1696            .table(&read.table)
1697            .ok_or("client.authority_read_forbidden: authority table is not declared")?;
1698        let plain = |name: &str| {
1699            table.columns.iter().enumerate().any(|(i, c)| {
1700                c.name == name
1701                    && name != "authority_version"
1702                    && !name.starts_with("_sync")
1703                    && !table.encrypted_columns.iter().any(|e| e.index == i)
1704                    && matches!(
1705                        c.ty,
1706                        ssp2::segment::ColumnType::String
1707                            | ssp2::segment::ColumnType::Integer
1708                            | ssp2::segment::ColumnType::Float
1709                            | ssp2::segment::ColumnType::Boolean
1710                            | ssp2::segment::ColumnType::Json
1711                    )
1712            })
1713        };
1714        if !seen.insert(&read.table)
1715            || read.columns.is_empty()
1716            || read.columns.iter().collect::<BTreeSet<_>>().len() != read.columns.len()
1717            || !read.columns.contains(&table.primary_key)
1718            || !read.columns.iter().all(|c| plain(c))
1719            || read.scopes.is_empty()
1720            || !read.scopes.iter().all(|(v, values)| {
1721                table
1722                    .scope_column(v)
1723                    .is_some_and(|c| read.columns.iter().any(|column| column == c) && plain(c))
1724                    && !values.is_empty()
1725                    && values.iter().all(|v| !v.is_empty() && v != "*")
1726            })
1727        {
1728            return Err("client.authority_read_forbidden: authority reads require plain columns, a primary key and nonempty plain scope selectors".into());
1729        }
1730    }
1731    Ok(())
1732}
1733
1734// §7.4.2: every open and recreation checks before any persistent writes.
1735/// A marker read that hit a recognized SQLite storage failure keeps that
1736/// classification (BUSY/LOCKED stay retryable); only an unclassified read
1737/// failure is the invalid-marker `sync.local_corrupt`.
1738fn schema_marker_read_failure(error: rusqlite::Error) -> String {
1739    let failure = QueryReadFailure::from(error);
1740    if failure.code.is_some() {
1741        failure.to_string()
1742    } else {
1743        "sync.local_corrupt: persisted local schema marker is unreadable or invalid".to_owned()
1744    }
1745}
1746
1747/// §7.1: reject an invalid configured client limit before any storage is
1748/// opened or written. Zero would wedge the push queue and an out-of-range
1749/// value would differ per target; the TS core enforces the same bounds.
1750fn validate_client_limits(limits: &crate::api::ClientLimits) -> Result<(), String> {
1751    if limits.outcome_retention_max_entries == Some(0) {
1752        return Err("sync.invalid_request: outcomeRetentionMaxEntries must be positive".to_owned());
1753    }
1754    if [
1755        limits.max_push_commits_per_request,
1756        limits.max_push_operations_per_request,
1757        limits.max_push_request_bytes,
1758    ]
1759    .into_iter()
1760    .flatten()
1761    .any(|value| value == 0 || value > u32::MAX as usize)
1762    {
1763        return Err(
1764            "sync.invalid_request: push limit must be an integer in 1..=4294967295".to_owned(),
1765        );
1766    }
1767    Ok(())
1768}
1769
1770fn read_local_schema_version(conn: &Connection, requested: i32) -> Result<Option<i32>, String> {
1771    let corrupt =
1772        || "sync.local_corrupt: persisted local schema marker is unreadable or invalid".to_owned();
1773    let table_type: Option<String> = conn
1774        .query_row(
1775            "SELECT type FROM sqlite_master WHERE name = '_syncular_meta'",
1776            [],
1777            |row| row.get(0),
1778        )
1779        .optional()
1780        .map_err(schema_marker_read_failure)?;
1781    let Some(table_type) = table_type else {
1782        return Ok(None);
1783    };
1784    if table_type != "table" {
1785        return Err(corrupt());
1786    }
1787    // §7.4.2: a duplicated marker cannot be resolved. `query_row` would take
1788    // whichever row SQLite returns first, and an older one would drive a
1789    // destructive reset over a newer replica.
1790    let mut statement = conn
1791        .prepare("SELECT value FROM _syncular_meta WHERE key = ?1")
1792        .map_err(schema_marker_read_failure)?;
1793    let mut rows = statement
1794        .query([LOCAL_SCHEMA_VERSION_KEY])
1795        .map_err(schema_marker_read_failure)?;
1796    let value: Option<String> = rows
1797        .next()
1798        .map_err(schema_marker_read_failure)?
1799        .map(|row| row.get(0))
1800        .transpose()
1801        .map_err(schema_marker_read_failure)?;
1802    if rows.next().map_err(schema_marker_read_failure)?.is_some() {
1803        return Err(corrupt());
1804    }
1805    let Some(value) = value else {
1806        let has_descriptor = conn
1807            .query_row(
1808                "SELECT 1 FROM _syncular_meta WHERE key = ?1 LIMIT 1",
1809                [crate::previous_version::LOCAL_SCHEMA_DESCRIPTOR_KEY],
1810                |_| Ok(()),
1811            )
1812            .optional()
1813            .map_err(schema_marker_read_failure)?
1814            .is_some();
1815        if has_descriptor {
1816            return Err(corrupt());
1817        }
1818        return Ok(None);
1819    };
1820    if !matches!(value.as_bytes().first(), Some(b'1'..=b'9'))
1821        || !value.bytes().all(|b| b.is_ascii_digit())
1822    {
1823        return Err(corrupt());
1824    }
1825    let version = value.parse::<i32>().map_err(|_| corrupt())?;
1826    if version > requested {
1827        return Err(
1828            "client.schema_downgrade: persisted local schema is newer than the requested schema"
1829                .to_owned(),
1830        );
1831    }
1832    Ok(Some(version))
1833}
1834
1835/// Open a replica file. On a browser worker the file must recover a hot
1836/// journal before any statement reads it: `wasm_vfs` replaces the installed
1837/// OPFS VFS's reserved lock callback before the connection opens its file. A
1838/// new browser replica uses 64 KiB pages (§5.3), because every OPFS call costs
1839/// the same whatever its size; a database with tables keeps its page size. Its
1840/// page cache holds 64 MiB, because SQLite's 2,000 KiB default is 31 such pages
1841/// and every cache miss is an OPFS call.
1842fn open_replica(path: &str) -> Result<Connection, String> {
1843    let browser = cfg!(all(target_arch = "wasm32", target_os = "unknown"))
1844        && !path.is_empty()
1845        && path != ":memory:";
1846    #[cfg(all(target_arch = "wasm32", target_os = "unknown"))]
1847    if browser {
1848        crate::wasm_vfs::disable_foreign_lock_check()?;
1849    }
1850    let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
1851    if browser {
1852        conn.execute_batch("PRAGMA page_size = 65536; PRAGMA cache_size = -65536")
1853            .map_err(|e| format!("open db {path:?}: {e}"))?;
1854    }
1855    Ok(conn)
1856}
1857
1858impl SyncClient {
1859    pub fn new_with_identity(
1860        client_id: Option<String>,
1861        schema_json: &Value,
1862        limits: ClientLimits,
1863    ) -> Result<Self, String> {
1864        let resolved = client_id.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
1865        let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
1866        Self::with_connection(resolved, schema_json, limits, conn)
1867    }
1868
1869    pub fn new(
1870        client_id: String,
1871        schema_json: &Value,
1872        limits: ClientLimits,
1873    ) -> Result<Self, String> {
1874        let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
1875        Self::with_connection(client_id, schema_json, limits, conn)
1876    }
1877
1878    /// Build a client backed by an on-disk SQLite database at `path` — the
1879    /// seam a native host (Tauri plugin, FFI file-DB variant) uses to persist
1880    /// across process restarts. `create_tables` runs `IF NOT EXISTS`, so
1881    /// re-opening the same file reuses the persisted rows. Keeps rusqlite out
1882    /// of the command router's dependency set (the router only holds a path).
1883    pub fn open_path(
1884        client_id: String,
1885        schema_json: &Value,
1886        limits: ClientLimits,
1887        path: &str,
1888    ) -> Result<Self, String> {
1889        validate_client_limits(&limits)?;
1890        parse_schema_json(schema_json)?;
1891        let conn = open_replica(path)?;
1892        Self::with_connection(client_id, schema_json, limits, conn)
1893    }
1894
1895    /// Open a file replica and resolve its client id. Native targets require
1896    /// WAL for the independent snapshot reader; `wasm32-unknown-unknown`
1897    /// accepts the sqlite-wasm worker journal instead.
1898    pub fn open_path_with_identity(
1899        client_id: Option<String>,
1900        schema_json: &Value,
1901        limits: ClientLimits,
1902        path: &str,
1903    ) -> Result<Self, String> {
1904        Self::open_path_with_identity_journal(
1905            client_id,
1906            schema_json,
1907            limits,
1908            path,
1909            !cfg!(all(target_arch = "wasm32", target_os = "unknown")),
1910        )
1911    }
1912
1913    fn open_path_with_identity_journal(
1914        client_id: Option<String>,
1915        schema_json: &Value,
1916        limits: ClientLimits,
1917        path: &str,
1918        require_wal: bool,
1919    ) -> Result<Self, String> {
1920        validate_client_limits(&limits)?;
1921        // §7.4.2: an invalid requested version is refused before the replica is
1922        // even created, so a rejected constructor leaves no file behind.
1923        parse_schema_json(schema_json)?;
1924        let conn = open_replica(path)?;
1925        // A file-backed native client uses an independent read connection for
1926        // latency-critical snapshots. WAL is SQLite's intended reader/writer
1927        // concurrency mode: a view read never holds a rollback-journal lock
1928        // that delays the client's next commit. The busy timeout must precede
1929        // this replica's own startup transaction; WAL is applied only after
1930        // startup succeeded, because a refused open must leave the replica's
1931        // journal mode and contents untouched (§7.4.2).
1932        conn.busy_timeout(std::time::Duration::from_millis(250))
1933            .map_err(|error| format!("configure db {path:?} busy timeout: {error}"))?;
1934        // Durability is the caller's contract here, not the VFS default: a
1935        // rollback-journal commit needs FULL before it writes the database
1936        // pages, or a killed worker can leave a half-written image behind.
1937        // WAL below inherits the same setting.
1938        conn.pragma_update(None, "synchronous", "FULL")
1939            .map_err(|error| format!("configure db {path:?} synchronous: {error}"))?;
1940        let client = Self::with_connection_identity(client_id, schema_json, limits, conn)?;
1941        // A browser worker's sqlite-wasm VFS has no WAL mode, and the wasm
1942        // host has no independent snapshot reader. The WAL requirement exists
1943        // for native file replicas, where it is the reader/writer boundary.
1944        if require_wal {
1945            let journal_mode: String = client
1946                .conn
1947                .pragma_update_and_check(None, "journal_mode", "WAL", |row| row.get(0))
1948                .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
1949            if journal_mode != "wal" {
1950                return Err(
1951                    "sync.invalid_request: file-backed client requires WAL journal mode".to_owned(),
1952                );
1953            }
1954        }
1955        Ok(client)
1956    }
1957
1958    /// Delete a file replica at `path`, including its `-journal`, `-wal`, and
1959    /// `-shm` sidecars, through the VFS that owns it. Use this after
1960    /// `client.storage_corrupt`: an OPFS SAH pool stores those files under
1961    /// opaque names, so deleting the pool directory behind SQLite bypasses the
1962    /// pool's slot accounting.
1963    ///
1964    /// Close the client and every [`FileQuerySnapshotReader`] on `path` first;
1965    /// a VFS cannot delete a file another handle still owns. A file that does
1966    /// not exist counts as deleted, so a cleanly closed replica without
1967    /// sidecars resets. The next open
1968    /// creates a fresh replica (new client id) and re-bootstraps from the
1969    /// server. A memory replica has nothing to delete and fails with
1970    /// `sync.invalid_request`.
1971    pub fn reset_replica(path: &str) -> Result<(), String> {
1972        if path.is_empty() || path == ":memory:" {
1973            return Err("sync.invalid_request: a memory replica has nothing to reset".to_owned());
1974        }
1975        // SAFETY: a null VFS name asks SQLite for its process-wide default VFS,
1976        // the same one `Connection::open(path)` above selects. `xDelete` is
1977        // that VFS's own function pointer, and `c_name` outlives the call.
1978        let vfs = unsafe { rusqlite::ffi::sqlite3_vfs_find(std::ptr::null()) };
1979        if vfs.is_null() {
1980            return Err("client.storage_io: no SQLite VFS is registered".to_owned());
1981        }
1982        let delete = unsafe { (*vfs).xDelete }
1983            .ok_or_else(|| "client.storage_io: the SQLite VFS cannot delete files".to_owned())?;
1984        for suffix in ["-wal", "-shm", "-journal", ""] {
1985            let file_name = format!("{path}{suffix}");
1986            let c_name = std::ffi::CString::new(file_name.as_str()).map_err(|_| {
1987                format!("sync.invalid_request: replica path {file_name:?} contains a NUL byte")
1988            })?;
1989            // SAFETY: `vfs` and its `xDelete` pointer come from the live
1990            // SQLite library; `c_name` is a valid NUL-terminated path.
1991            let code = unsafe { delete(vfs, c_name.as_ptr(), 1) };
1992            // A cleanly closed WAL replica has no sidecars; a missing file is
1993            // already deleted.
1994            if code != rusqlite::ffi::SQLITE_OK && code != rusqlite::ffi::SQLITE_IOERR_DELETE_NOENT
1995            {
1996                return Err(format!(
1997                    "client.storage_io: deleting replica file {file_name:?} failed with SQLite code {code}"
1998                ));
1999            }
2000        }
2001        Ok(())
2002    }
2003
2004    #[must_use]
2005    pub fn client_id(&self) -> &str {
2006        &self.client_id
2007    }
2008
2009    /// Rotate the caller's client id (§1.5) and return the new one. Use it
2010    /// after a `sync.invalid_client_id` refusal (`recommendedAction:
2011    /// resetClientId`): the next round carries the new id. A live realtime
2012    /// socket is closed so the next connection registers with the new id.
2013    /// The replica and its outbox stay in place; wipe them yourself when the
2014    /// stored data belongs to the account that signed out.
2015    pub fn reset_client_id(&mut self, transport: &mut dyn Transport) -> Result<String, String> {
2016        self.disconnect_realtime(transport);
2017        let client_id = uuid::Uuid::new_v4().to_string();
2018        self.set_meta(CLIENT_ID_KEY, &client_id)?;
2019        self.client_id.clone_from(&client_id);
2020        self.round_failure = None;
2021        Ok(client_id)
2022    }
2023
2024    /// Build a client over a caller-supplied rusqlite connection — the seam a
2025    /// native host (Tauri plugin, FFI file-DB variant) uses to back the core
2026    /// with an on-disk database (`Connection::open(path)`) rather than the
2027    /// default `:memory:`. The connection MUST be fresh (no pre-existing
2028    /// syncular tables); `create_tables` runs `IF NOT EXISTS`, so re-opening
2029    /// the same file across process restarts reuses the persisted rows.
2030    pub fn with_connection(
2031        client_id: String,
2032        schema_json: &Value,
2033        limits: ClientLimits,
2034        conn: Connection,
2035    ) -> Result<Self, String> {
2036        Self::with_connection_identity(Some(client_id), schema_json, limits, conn)
2037    }
2038
2039    /// [`Self::with_connection`] for `open_path_with_identity`: no requested id
2040    /// means a durable replica's persisted `clientId` wins over a fresh one.
2041    fn with_connection_identity(
2042        client_id: Option<String>,
2043        schema_json: &Value,
2044        limits: ClientLimits,
2045        conn: Connection,
2046    ) -> Result<Self, String> {
2047        validate_client_limits(&limits)?;
2048        let schema = parse_schema_json(schema_json)?;
2049        validate_authority_reads(&schema, &limits.authority_reads)?;
2050        // RFC 0005 D8: resolve the previous-version config before the opening
2051        // reset runs — the capture happens inside it. Bad bounds fail loud.
2052        let previous_version = match limits.previous_version_context {
2053            Some(config) => {
2054                config.validate()?;
2055                Some(config)
2056            }
2057            None => None,
2058        };
2059        let mut client = SyncClient {
2060            #[cfg(feature = "bench-internals")]
2061            benchmark_phases: Recorder::default(),
2062            conn,
2063            schema,
2064            client_id: client_id
2065                .clone()
2066                .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()),
2067            limits,
2068            outbox: Shared::default(),
2069            failed_commits: Shared::default(),
2070            retain_failed_commits: false,
2071            conflicts: Shared::default(),
2072            rejections: Shared::default(),
2073            schema_floor: None,
2074            lease_state: None,
2075            stopped: false,
2076            upgrading: false,
2077            sync_needed: false,
2078            realtime_policy: RealtimePolicy::Optional,
2079            realtime_state: RealtimeState::Disconnected,
2080            realtime_reason_code: None,
2081            realtime_retry_delay_ms: None,
2082            presence: HashMap::new(),
2083            now_ms: None,
2084            encryption: crate::values::EncryptionConfig::default(),
2085            security_preflight: false,
2086            security_epoch: 0,
2087            transport_enabled: true,
2088            transport_epoch: 0,
2089            registration_epoch: 0,
2090            previous_version,
2091            insert_sql: RefCell::new(HashMap::new()),
2092            overlay_dirty: OverlayDirty::clean(),
2093            storage_failure: RefCell::new(None),
2094            #[cfg(test)]
2095            overlay_rebuild_count: Cell::new(0),
2096            #[cfg(test)]
2097            acknowledged_replay_count: Cell::new(0),
2098            #[cfg(test)]
2099            outcome_prune_count: Cell::new(0),
2100            #[cfg(test)]
2101            image_chunk_ms: Cell::new(None),
2102            progress: ProgressObserver::default(),
2103            change_queue: VecDeque::new(),
2104            sync_intent_queue: VecDeque::new(),
2105            retry_delay_ms: 250,
2106            round_retry_delay_ms: None,
2107            image_chunk_rows: HashMap::new(),
2108            round_failure: None,
2109            active_round: None,
2110            last_round: None,
2111            last_change: None,
2112            query_failures: Vec::new(),
2113            coalesced: None,
2114            coalesced_failure: None,
2115        };
2116        // The row write path leans on the prepared-statement cache (two
2117        // insert statements per synced table, plus the bookkeeping
2118        // statements); size it so a multi-table schema never thrashes.
2119        client
2120            .conn
2121            .set_prepared_statement_cache_capacity(64.max(client.schema.tables.len() * 4));
2122        // §7.4.2: one write transaction (SQLite `BEGIN IMMEDIATE`, a reserved
2123        // writer) covers the marker read, the identity check, and every
2124        // bookkeeping/schema write, so a concurrent open that upgraded the
2125        // replica cannot interleave between the guard and the writes, and a
2126        // refusal leaves the replica unchanged.
2127        client
2128            .conn
2129            .execute_batch("BEGIN IMMEDIATE")
2130            .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
2131        let startup = (|| -> Result<(), String> {
2132            // The marker guard runs inside the write transaction, so the version
2133            // it validates is the version every write below commits under.
2134            let marker = read_local_schema_version(&client.conn, client.schema.version)?;
2135            // New app indexes may reference columns that only exist after a
2136            // version-bump reset.
2137            client.create_bookkeeping_tables()?;
2138            match client.get_meta(CLIENT_ID_KEY)? {
2139                Some(existing) => match client_id {
2140                    Some(requested) if existing != requested => {
2141                        return Err(format!(
2142                            "client.identity_mismatch: this database belongs to {existing:?}; refusing to rebind it to {requested:?}"
2143                        ))
2144                    }
2145                    _ => client.client_id = existing,
2146                },
2147                None => client.set_meta(CLIENT_ID_KEY, &client.client_id)?,
2148            }
2149            client.restore_persisted_state()?;
2150            client.failed_commits = client.load_failed_commits()?.into();
2151            match marker {
2152                None => {
2153                    client.create_synced_tables()?;
2154                    client.persist_schema_version()?;
2155                    client.save_unit_scope_schema()?;
2156                }
2157                Some(version) if version == client.schema.version => {
2158                    client.create_synced_tables()?;
2159                    // RFC 0005 D1: the same-version open backfills the descriptor
2160                    // for a database first opened by an unaware binary, so the
2161                    // NEXT bump can capture without needing a schema bump first.
2162                    set_local_schema_descriptor(&client.conn, &client.schema)
2163                        .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
2164                    client.save_unit_scope_schema()?;
2165                }
2166                Some(_) => client.run_schema_reset()?,
2167            }
2168            // RFC 0005 D9: the reset sweep cannot see a container whose capture
2169            // committed in its OWN file while the replica's savepoint never
2170            // released — a crash at that seam leaves a container beside a matching
2171            // OLD marker, and the next open at that version runs no reset. Discard
2172            // it before anything can read it. No-op when no container exists.
2173            client.reconcile_previous_version_at_boot()?;
2174            client.clear_satisfied_persisted_schema_floor()?;
2175            client.prune_unknown_units(false)?;
2176            // §7.1/§7.4.4: recover intrinsically codec-invalid persisted values at
2177            // the full startup/reset reconciliation boundary, inside this open
2178            // transaction and after the schema and identity guards, so a commit
2179            // an earlier version accepted is dropped with durable rejection
2180            // evidence before any replay. A table the current schema removed
2181            // keeps the §7.4.4 send-time classification, and an unresolved
2182            // encryption key stays with the send seam because hosts configure
2183            // keys after open.
2184            client.drop_unencodable_outbox(false, None)?;
2185            if marker == Some(client.schema.version)
2186                && (!client.outbox.is_empty()
2187                    || !client.failed_commits.is_empty()
2188                    || client.has_acknowledged_rows()?)
2189            {
2190                // Reconstruct the visible optimistic overlay from the durable
2191                // base plus outbox instead of trusting a process-interrupted
2192                // mirror.
2193                client.overlay_dirty.set(true);
2194                client.rebuild_overlay()?;
2195            }
2196            // Every persisted active unit needs one catch-up pull on
2197            // open: realtime only covers changes after connection, while an
2198            // idempotent setWindow correctly creates no fresh command effect.
2199            // Pending outbox work has the same restart requirement. The core
2200            // owns this intent so native hosts never poll or require an
2201            // application-issued sync().
2202            client.enqueue_startup_sync_if_needed()?;
2203            Ok(())
2204        })();
2205        match startup {
2206            Ok(()) => match client.conn.execute_batch("COMMIT") {
2207                // §4.8: an eviction a crash interrupted finishes before any
2208                // read, with or without a transport; a replica reopened in
2209                // preflight finishes it at activation.
2210                Ok(()) => {
2211                    if !client.security_preflight {
2212                        client.drain_pending_evictions()?;
2213                    }
2214                    Ok(client)
2215                }
2216                Err(error) => {
2217                    let failure = Self::sqlite_failure(&client.storage_failure, error);
2218                    if let Err(rollback) = client.conn.execute_batch("ROLLBACK") {
2219                        // Keep a failed rollback's typed storage classification.
2220                        let _ = Self::sqlite_failure(&client.storage_failure, rollback);
2221                    }
2222                    Err(failure)
2223                }
2224            },
2225            Err(error) => {
2226                if let Err(rollback) = client.conn.execute_batch("ROLLBACK") {
2227                    // Keep a failed rollback's typed storage classification.
2228                    let _ = Self::sqlite_failure(&client.storage_failure, rollback);
2229                }
2230                Err(error)
2231            }
2232        }
2233    }
2234
2235    /// Pin the client clock (epoch ms) — the §5.4 expiry check runs
2236    /// against this instead of system time (conformance virtual clock).
2237    pub fn set_now_ms(&mut self, now_ms: i64) {
2238        self.now_ms = Some(std::sync::Arc::new(now_ms.into()));
2239    }
2240
2241    /// The pinned clock, shared: a host that stores into it moves the client
2242    /// clock while a call runs (a virtual clock that advances per request).
2243    pub fn pinned_clock(&self) -> Option<std::sync::Arc<std::sync::atomic::AtomicI64>> {
2244        self.now_ms.clone()
2245    }
2246
2247    /// §5.11: install the client-side encryption keys (`keyId → key bytes`).
2248    /// The command router parses these from the `create` command's
2249    /// `encryption` config (keys as `{$bytes: hex}`).
2250    pub fn set_encryption(&mut self, encryption: crate::values::EncryptionConfig) {
2251        self.encryption = encryption;
2252    }
2253
2254    #[must_use]
2255    pub fn security_lifecycle(&self) -> &'static str {
2256        if self.security_preflight {
2257            "preflight"
2258        } else {
2259            "active"
2260        }
2261    }
2262
2263    #[must_use]
2264    pub fn security_preflight(&self) -> bool {
2265        self.security_preflight
2266    }
2267
2268    /// Host-owned, non-persisted network gate. Local operations remain available.
2269    pub fn set_transport_enabled(&mut self, transport: &mut dyn Transport, enabled: bool) {
2270        if self.transport_enabled != enabled {
2271            self.transport_enabled = enabled;
2272            self.transport_epoch = self.transport_epoch.wrapping_add(1);
2273            self.sync_intent_queue.clear();
2274            self.sync_intent_queue
2275                .push_back(if enabled && !self.security_preflight {
2276                    SyncIntent::Interactive
2277                } else {
2278                    SyncIntent::None
2279                });
2280        }
2281        // A captured reply keeps its normal apply/revocation semantics.
2282        if !enabled && self.active_round.is_none() {
2283            self.disconnect_realtime(transport);
2284        }
2285    }
2286
2287    pub fn transport_enabled(&self) -> bool {
2288        self.transport_enabled
2289    }
2290
2291    /// Quarantine this replica: enter the fail-closed gate, release all
2292    /// core-owned key material, and record the gate in the database so
2293    /// reopening re-enters preflight until `activate_security` clears it.
2294    ///
2295    /// Use [`SyncClient::seal_security_on_teardown`] for the shutdown barrier,
2296    /// which must NOT leave that durable mark.
2297    pub fn begin_security_preflight(&mut self) -> Result<(), String> {
2298        let entering = !self.security_preflight;
2299        self.seal_security_on_teardown();
2300        // Persist the quarantine so it survives handle teardown and restart:
2301        // reopening this replica re-enters preflight until activation clears it.
2302        self.set_meta(SECURITY_PREFLIGHT_PENDING_KEY, "1")?;
2303        if entering {
2304            self.emit_status_batch()?;
2305        }
2306        Ok(())
2307    }
2308
2309    /// One change batch carrying the status snapshot (§5.11 lifecycle).
2310    fn emit_status_batch(&mut self) -> Result<(), String> {
2311        self.begin_observation("syncular_status")?;
2312        let batch = ChangeAccumulator {
2313            status: true,
2314            ..ChangeAccumulator::default()
2315        };
2316        self.finish_observation("syncular_status", batch)
2317            .inspect_err(|_| self.rollback_observation("syncular_status"))
2318    }
2319
2320    /// Release core-owned key material as a host tears the client down,
2321    /// WITHOUT recording a quarantine.
2322    ///
2323    /// Shutting an activated client down is not a quarantine event. Persisting
2324    /// the gate here marks every cleanly closed replica as pending, and the
2325    /// next plain `create` is then refused permanently: the reopen path
2326    /// restores the flag from the marker and the create guard rejects it. The
2327    /// in-memory flag still closes the gate for anything still holding this
2328    /// instance, which is all a teardown barrier needs.
2329    pub fn seal_security_on_teardown(&mut self) {
2330        self.cancel_sync_round();
2331        self.security_preflight = true;
2332        self.security_epoch = self.security_epoch.wrapping_add(1);
2333        self.encryption = crate::values::EncryptionConfig::default();
2334        self.sync_intent_queue.clear();
2335    }
2336
2337    /// Install the post-authentication keyring and release the host loop.
2338    pub fn activate_security(
2339        &mut self,
2340        encryption: crate::values::EncryptionConfig,
2341    ) -> Result<(), String> {
2342        if !self.security_preflight {
2343            return Err(
2344                "sync.invalid_request: activateSecurity requires security preflight".to_owned(),
2345            );
2346        }
2347        // §4.8: an interrupted eviction finishes before protected reads open.
2348        self.drain_pending_evictions()?;
2349        self.enqueue_startup_sync_if_needed()?;
2350        self.delete_meta(SECURITY_PREFLIGHT_PENDING_KEY)?;
2351        self.encryption = encryption;
2352        self.security_preflight = false;
2353        // §5.11: rows sealed under a key this keyring now holds open at once.
2354        self.reveal_sealed_rows()
2355            .map_err(|error| format!("{}: {}", error.code, error.message))?;
2356        self.emit_status_batch()
2357    }
2358
2359    fn clock_now_ms(&self) -> i64 {
2360        self.now_ms
2361            .as_ref()
2362            .map_or_else(crate::wall_clock_ms, |clock| {
2363                clock.load(std::sync::atomic::Ordering::Relaxed)
2364            })
2365    }
2366
2367    fn create_bookkeeping_tables(&self) -> Result<(), String> {
2368        let exists = |name: &str| {
2369            self.conn
2370                .query_row(
2371                    "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?1)",
2372                    [name],
2373                    |row| row.get::<_, bool>(0),
2374                )
2375                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2376        };
2377        let outbox_existed = exists("_syncular_outbox")?;
2378        // Durable client bookkeeping (outbox + units + meta).
2379        self.conn
2380            .execute_batch(
2381                "CREATE TABLE IF NOT EXISTS _syncular_outbox (
2382                   seq INTEGER PRIMARY KEY AUTOINCREMENT,
2383                   commit_id TEXT NOT NULL UNIQUE, ops_json TEXT NOT NULL);
2384                 CREATE TABLE IF NOT EXISTS _syncular_row_deliveries (
2385                   tbl TEXT NOT NULL, id TEXT NOT NULL, commit_seq INTEGER NOT NULL, PRIMARY KEY(tbl,id));
2386                 CREATE TABLE IF NOT EXISTS _syncular_acked_rows (
2387                   commit_id TEXT NOT NULL, idx INTEGER NOT NULL, tbl TEXT NOT NULL, id TEXT NOT NULL,
2388                   commit_seq INTEGER NOT NULL, op_json TEXT NOT NULL, intent_json TEXT,
2389                   PRIMARY KEY(commit_id, idx));
2390                 CREATE INDEX IF NOT EXISTS _syncular_acked_rows_row
2391                   ON _syncular_acked_rows(tbl, id);
2392                 CREATE TABLE IF NOT EXISTS _syncular_failed_commits (
2393                   seq INTEGER PRIMARY KEY AUTOINCREMENT, client_commit_id TEXT NOT NULL UNIQUE,
2394                   operations_json TEXT NOT NULL, initial_json TEXT NOT NULL);
2395                 CREATE TABLE IF NOT EXISTS _syncular_commit_outcomes (
2396                   seq INTEGER PRIMARY KEY AUTOINCREMENT,
2397                   client_commit_id TEXT NOT NULL UNIQUE,
2398                   status TEXT NOT NULL CHECK(status IN ('applied', 'cached', 'conflict', 'rejected')),
2399                   recorded_at_ms INTEGER NOT NULL,
2400                   results_json TEXT NOT NULL,
2401                   operations_json TEXT,
2402                   resolution TEXT NOT NULL DEFAULT 'active'
2403                     CHECK(resolution IN ('active', 'resolved_keep_server', 'superseded', 'dismissed')),
2404                   resolved_at_ms INTEGER,
2405                   replacement_client_commit_id TEXT);
2406                 CREATE INDEX IF NOT EXISTS _syncular_commit_outcomes_resolution_seq
2407                   ON _syncular_commit_outcomes(resolution, seq);
2408                 CREATE TABLE IF NOT EXISTS _syncular_units (
2409                   id TEXT PRIMARY KEY, scopes TEXT NOT NULL,
2410                   explicit INTEGER NOT NULL DEFAULT 0,
2411                   status TEXT NOT NULL DEFAULT 'active', reason_code TEXT,
2412                   cursor INTEGER DEFAULT -1, bootstrap_state TEXT,
2413                   seeded TEXT NOT NULL DEFAULT '[]', echo TEXT, applied TEXT);
2414                 CREATE TABLE IF NOT EXISTS _syncular_meta (
2415                   key TEXT PRIMARY KEY, value TEXT NOT NULL);
2416                 CREATE TABLE IF NOT EXISTS _syncular_windows (
2417                   base TEXT NOT NULL, unit TEXT NOT NULL, sub_id TEXT NOT NULL,
2418                   PRIMARY KEY (base, unit));
2419                 CREATE INDEX IF NOT EXISTS _syncular_windows_unit ON _syncular_windows(sub_id);
2420                 CREATE TABLE IF NOT EXISTS _syncular_pending_evictions (
2421                   unit_id TEXT NOT NULL, tbl TEXT NOT NULL,
2422                   effective_scopes TEXT NOT NULL, PRIMARY KEY (unit_id, tbl));
2423                 CREATE TABLE IF NOT EXISTS _syncular_blob_commit_refs(
2424                   commit_id TEXT NOT NULL, blob_id TEXT NOT NULL,
2425                   PRIMARY KEY(commit_id, blob_id));
2426                 CREATE INDEX IF NOT EXISTS _syncular_blob_commit_refs_body
2427                   ON _syncular_blob_commit_refs(blob_id);",
2428            )
2429            .map_err(|e| e.to_string())?;
2430        // §5.6 applied-segment record, added after the units table shipped.
2431        if !self
2432            .conn
2433            .prepare("SELECT 1 FROM pragma_table_info('_syncular_units') WHERE name = 'applied'")
2434            .and_then(|mut stmt| stmt.exists([]))
2435            .map_err(|e| e.to_string())?
2436        {
2437            self.conn
2438                .execute_batch("ALTER TABLE _syncular_units ADD COLUMN applied TEXT")
2439                .map_err(|e| e.to_string())?;
2440        }
2441        // Migrate an outcome journal created before failed aggregate
2442        // envelopes were retained. Historical rows intentionally stay NULL.
2443        let _ = self
2444            .conn
2445            .execute_batch("ALTER TABLE _syncular_commit_outcomes ADD COLUMN operations_json TEXT");
2446        if self.get_meta(LOCAL_REVISION_KEY)?.is_none() {
2447            self.set_meta(LOCAL_REVISION_KEY, "0")?;
2448        }
2449        // §7.1: `outboxId` names the outbox sequence. A new outbox table starts
2450        // a new sequence, so it replaces the id in the same transaction.
2451        if !outbox_existed || self.get_meta(OUTBOX_ID_KEY)?.is_none() {
2452            self.set_meta(OUTBOX_ID_KEY, &uuid::Uuid::new_v4().to_string())?;
2453        }
2454        // §9: a replica written under wire version 3 keeps its outbox, journal,
2455        // rows and pending evictions and registers its units fresh. Its
2456        // subscriptions, cursors, echoes, resume tokens and window entries go;
2457        // units left from an earlier version 4 life restart.
2458        if exists("_syncular_window_pending_evict")? {
2459            let invalid: bool = self
2460                .conn
2461                .query_row(
2462                    "SELECT EXISTS(SELECT 1 FROM _syncular_window_pending_evict
2463                       WHERE NOT json_valid(effective_scopes)
2464                          OR json_type(effective_scopes) <> 'object'
2465                          OR EXISTS(SELECT 1 FROM json_each(effective_scopes)
2466                               WHERE type <> 'array'
2467                                  OR EXISTS(SELECT 1 FROM json_each(value) AS v
2468                                       WHERE v.type <> 'text')))",
2469                    [],
2470                    |row| row.get(0),
2471                )
2472                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
2473            if invalid {
2474                return Err(
2475                    "sync.local_corrupt: a version 3 pending eviction cannot be upgraded"
2476                        .to_owned(),
2477                );
2478            }
2479            self.conn
2480                .execute_batch(
2481                    "INSERT OR REPLACE INTO _syncular_pending_evictions
2482                       SELECT sub_id, tbl, effective_scopes FROM _syncular_window_pending_evict;
2483                     DROP TABLE _syncular_window_pending_evict;
2484                     DROP TABLE IF EXISTS _syncular_subscriptions;
2485                     DELETE FROM _syncular_windows;",
2486                )
2487                .and_then(|()| reset_units(&self.conn, &BTreeSet::new()))
2488                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
2489        }
2490        // §5.9.7 blob cache. Mutable upload state stays off the body row so
2491        // completing a large upload does not rewrite its BLOB record.
2492        if self.schema_has_blobs() {
2493            if exists("_syncular_blobs")? {
2494                let mut statement = self
2495                    .conn
2496                    .prepare("PRAGMA table_info(_syncular_blobs)")
2497                    .map_err(|e| e.to_string())?;
2498                let mut columns = statement
2499                    .query_map([], |row| row.get::<_, String>(1))
2500                    .map_err(|e| e.to_string())?
2501                    .collect::<Result<Vec<_>, _>>()
2502                    .map_err(|e| e.to_string())?;
2503                columns.sort();
2504                let mut expected = vec![
2505                    "blob_id",
2506                    "byte_length",
2507                    "bytes",
2508                    "created_at_ms",
2509                    "media_type",
2510                ];
2511                expected.sort_unstable();
2512                if columns != expected {
2513                    return Err(
2514                        "sync.schema_mismatch: Local blob schema is incompatible".to_owned()
2515                    );
2516                }
2517            } else {
2518                self.conn
2519                    .execute_batch(
2520                        "CREATE TABLE _syncular_blobs(
2521                           blob_id TEXT PRIMARY KEY,
2522                           bytes BLOB NOT NULL,
2523                           byte_length INTEGER NOT NULL,
2524                           media_type TEXT,
2525                           created_at_ms INTEGER NOT NULL);",
2526                    )
2527                    .map_err(|e| e.to_string())?;
2528            }
2529            self.conn
2530                .execute_batch(
2531                    "CREATE TABLE IF NOT EXISTS _syncular_blob_uploads(
2532                       blob_id TEXT PRIMARY KEY,
2533                       media_type TEXT,
2534                       created_at_ms INTEGER NOT NULL);",
2535                )
2536                .map_err(|e| e.to_string())?;
2537        }
2538        Ok(())
2539    }
2540
2541    /// True iff any synced table declares a `blob_ref` column (§5.9).
2542    fn schema_has_blobs(&self) -> bool {
2543        self.schema
2544            .tables
2545            .iter()
2546            .any(|t| t.columns.iter().any(|c| c.ty == ColumnType::BlobRef))
2547    }
2548
2549    // -- meta (§7.4.1 marker, bookkeeping) ------------------------------------
2550
2551    fn get_meta(&self, key: &str) -> Result<Option<String>, String> {
2552        meta_get(&self.conn, key)
2553            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2554    }
2555
2556    // The caller holds the startup/reset transaction. Either both schema
2557    // records persist or the enclosing operation fails and rolls back.
2558    fn persist_schema_version(&self) -> Result<(), String> {
2559        self.conn
2560            .execute(
2561                "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
2562                rusqlite::params![LOCAL_SCHEMA_VERSION_KEY, self.schema.version.to_string()],
2563            )
2564            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
2565        set_local_schema_descriptor(&self.conn, &self.schema)
2566            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2567    }
2568
2569    pub(crate) fn set_meta(&self, key: &str, value: &str) -> Result<(), String> {
2570        meta_set(&self.conn, key, value)
2571            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2572    }
2573
2574    fn delete_meta(&self, key: &str) -> Result<(), String> {
2575        meta_delete(&self.conn, key)
2576            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2577    }
2578
2579    /// §7.1: the durable outbox in FIFO order, in schema-agnostic local form.
2580    fn load_outbox(&self) -> Result<Vec<Arc<OutboxCommit>>, String> {
2581        let mut stmt = self
2582            .conn
2583            .prepare("SELECT commit_id, ops_json FROM _syncular_outbox ORDER BY seq ASC")
2584            .map_err(|error| error.to_string())?;
2585        let rows = stmt
2586            .query_map([], |row| {
2587                Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
2588            })
2589            .map_err(|error| error.to_string())?;
2590        let mut commits = Vec::new();
2591        for row in rows {
2592            let (client_commit_id, raw) = row.map_err(|error| error.to_string())?;
2593            let entries: Vec<Value> = serde_json::from_str(&raw).map_err(|error| {
2594                format!("invalid persisted outbox {client_commit_id:?}: {error}")
2595            })?;
2596            let mut ops = Vec::with_capacity(entries.len());
2597            for entry in entries {
2598                let op = entry.get("op").and_then(Value::as_str).unwrap_or("delete");
2599                ops.push(OutboxOp {
2600                    upsert: op == "upsert",
2601                    table: entry
2602                        .get("table")
2603                        .and_then(Value::as_str)
2604                        .ok_or_else(|| "persisted outbox operation missing table".to_owned())?
2605                        .to_owned(),
2606                    row_id: entry
2607                        .get("rowId")
2608                        .and_then(Value::as_str)
2609                        .ok_or_else(|| "persisted outbox operation missing rowId".to_owned())?
2610                        .to_owned(),
2611                    base_version: entry.get("baseVersion").and_then(Value::as_i64),
2612                    // A legacy entry's `changedFields` key is ignored:
2613                    // the values map is the presence set (§6.1).
2614                    values: entry.get("values").and_then(Value::as_object).cloned(),
2615                    insert: entry.get("insert") == Some(&Value::Bool(true)),
2616                });
2617            }
2618            commits.push(Arc::new(OutboxCommit {
2619                client_commit_id,
2620                ops,
2621            }));
2622        }
2623        Ok(commits)
2624    }
2625
2626    fn restore_persisted_state(&mut self) -> Result<(), String> {
2627        self.outbox = self.load_outbox()?.into();
2628
2629        self.prune_commit_outcomes()?;
2630        let active = self.commit_outcomes(CommitOutcomeQuery {
2631            active_only: true,
2632            ..CommitOutcomeQuery::default()
2633        })?;
2634        self.conflicts = active
2635            .iter()
2636            .flat_map(|outcome| outcome.results.iter())
2637            .filter_map(|result| match result {
2638                CommitOperationOutcome::Conflict { conflict } => Some(conflict.clone()),
2639                _ => None,
2640            })
2641            .collect::<Vec<_>>()
2642            .into();
2643        self.rejections = active
2644            .iter()
2645            .flat_map(|outcome| outcome.results.iter())
2646            .filter_map(|result| match result {
2647                CommitOperationOutcome::Error { rejection } => Some(rejection.clone()),
2648                _ => None,
2649            })
2650            .collect::<Vec<_>>()
2651            .into();
2652
2653        self.lease_state = self
2654            .get_meta(LEASE_STATE_KEY)?
2655            .map(|raw| serde_json::from_str(&raw))
2656            .transpose()
2657            .map_err(|error| format!("invalid persisted lease state: {error}"))?;
2658        self.schema_floor = self
2659            .get_meta(SCHEMA_FLOOR_KEY)?
2660            .map(|raw| serde_json::from_str(&raw))
2661            .transpose()
2662            .map_err(|error| format!("invalid persisted schema floor: {error}"))?;
2663        self.stopped = self.schema_floor.is_some();
2664        // A replica left in unactivated preflight reopens gated: the quarantine
2665        // decision persists with the data across handle teardown and restart.
2666        if self.get_meta(SECURITY_PREFLIGHT_PENDING_KEY)?.as_deref() == Some("1") {
2667            self.security_preflight = true;
2668        }
2669        Ok(())
2670    }
2671
2672    fn save_unit_scope_schema(&self) -> Result<(), String> {
2673        let declarations: BTreeMap<_, BTreeMap<_, _>> = self
2674            .schema
2675            .tables
2676            .iter()
2677            .map(|table| {
2678                (
2679                    table.name.clone(),
2680                    table
2681                        .scope_variables
2682                        .iter()
2683                        .map(|scope| {
2684                            (
2685                                scope.variable.clone(),
2686                                [scope.prefix.clone(), scope.column.clone()],
2687                            )
2688                        })
2689                        .collect(),
2690                )
2691            })
2692            .collect();
2693        self.set_meta(
2694            "subscriptionScopeSchema",
2695            &serde_json::to_string(&declarations).expect("scope schema serialization"),
2696        )?;
2697        let generations: BTreeMap<_, _> = self
2698            .schema
2699            .tables
2700            .iter()
2701            .filter_map(|table| Some((&table.name, table.generation.as_ref()?)))
2702            .collect();
2703        self.set_meta(
2704            TABLE_GENERATIONS_KEY,
2705            &serde_json::to_string(&generations).expect("generations serialize"),
2706        )
2707    }
2708
2709    /// Drop units that cover no table of the running schema, with their window
2710    /// registry rows and pending evictions; never translate scopes. At a bump
2711    /// (§7.4.3) a unit also needs every variable unchanged in prefix and column
2712    /// on every table it covered before.
2713    fn prune_unknown_units(&mut self, bump: bool) -> Result<(), String> {
2714        let previous: Option<BTreeMap<String, BTreeMap<String, [String; 2]>>> = if bump {
2715            self.get_meta("subscriptionScopeSchema")?
2716                .map(|raw| serde_json::from_str(&raw))
2717                .transpose()
2718                .map_err(|_| "sync.local_corrupt: invalid persisted scope schema".to_owned())?
2719        } else {
2720            None
2721        };
2722        // A table keeps the variables' meaning when each has the same prefix
2723        // and column as before. An old binary stored no declaration evidence;
2724        // matching names cannot prove unchanged meaning, so the app registers
2725        // again.
2726        let unchanged = |name: &str, variables: &[&String]| {
2727            let (Some(old), Some(table)) = (
2728                previous.as_ref().and_then(|previous| previous.get(name)),
2729                self.schema.table(name),
2730            ) else {
2731                return false;
2732            };
2733            variables.iter().all(|variable| {
2734                table
2735                    .scope_variables
2736                    .iter()
2737                    .find(|scope| &scope.variable == *variable)
2738                    .is_some_and(|scope| {
2739                        old.get(*variable) == Some(&[scope.prefix.clone(), scope.column.clone()])
2740                    })
2741            })
2742        };
2743        let compatible = |scopes: &UnitScopes| {
2744            if !self.schema.tables.iter().any(|table| covers(table, scopes)) {
2745                return false;
2746            }
2747            if !bump {
2748                return true;
2749            }
2750            let Some(previous) = &previous else {
2751                return false;
2752            };
2753            previous.iter().all(|(name, old)| {
2754                !scopes.keys().all(|variable| old.contains_key(variable))
2755                    || unchanged(name, &scopes.keys().collect::<Vec<_>>())
2756            })
2757        };
2758        let mut stale: BTreeSet<String> = self
2759            .stored(load_units(&self.conn))?
2760            .into_iter()
2761            .filter(|unit| !compatible(&unit.scopes))
2762            .map(|unit| unit.id)
2763            .collect();
2764        // A shrunk window's deferred eviction outlives its unit row and keeps
2765        // its per-table scopes. A table the schema no longer has cannot be
2766        // evicted, and at a bump a table whose scope declarations changed must
2767        // not be evicted by the outdated rule.
2768        for (id, table, effective) in self.load_pending_evictions()? {
2769            if self.schema.table(&table).is_none()
2770                || (bump
2771                    && !unchanged(
2772                        &table,
2773                        &effective
2774                            .iter()
2775                            .map(|(variable, _)| variable)
2776                            .collect::<Vec<_>>(),
2777                    ))
2778            {
2779                stale.insert(id);
2780            }
2781        }
2782        for id in &stale {
2783            for sql in [
2784                "DELETE FROM _syncular_windows WHERE sub_id = ?1",
2785                "DELETE FROM _syncular_pending_evictions WHERE unit_id = ?1",
2786                "DELETE FROM _syncular_units WHERE id = ?1",
2787            ] {
2788                self.conn
2789                    .execute(sql, rusqlite::params![id])
2790                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
2791            }
2792        }
2793        Ok(())
2794    }
2795
2796    pub fn local_revision(&self) -> Result<u64, String> {
2797        Ok(self
2798            .get_meta(LOCAL_REVISION_KEY)?
2799            .and_then(|value| value.parse().ok())
2800            .unwrap_or(0))
2801    }
2802
2803    /// §7.5: resolve a snapshot read on the mutable owner connection through
2804    /// the same primitive the read-only sidecar uses, so the command surface
2805    /// and the file sidecar return one revision, rows, coverage, catch-up, and
2806    /// delivery shape. The structured [`QueryReadFailure`] is preserved; an
2807    /// `owner` records its failure (and its next success) in diagnostics.
2808    pub fn snapshot_read(
2809        &mut self,
2810        request: &SnapshotReadRequest<'_>,
2811        owner: Option<QueryOwner<'_>>,
2812    ) -> Result<SnapshotRead, QueryReadFailure> {
2813        let result = snapshot_read_connection(&self.conn, request);
2814        if let Some(owner) = owner {
2815            self.record_query_read(owner, result.as_ref().err());
2816        }
2817        result
2818    }
2819
2820    #[must_use]
2821    pub fn status_snapshot(&self) -> SyncStatusSnapshot {
2822        SyncStatusSnapshot {
2823            current_schema_version: self.schema.version,
2824            outbox: self.outbox.len(),
2825            upgrading: self.upgrading,
2826            lease_state: self.lease_state.clone(),
2827            schema_floor: self.schema_floor.clone(),
2828            sync_needed: self.sync_needed,
2829            previous_version_context: self.previous_version_status(),
2830            security_lifecycle: self.security_lifecycle(),
2831        }
2832    }
2833
2834    /// RFC 0005 A2: container presence for a host that must act on it (Diego's
2835    /// rollback precondition), read without creating the file.
2836    fn previous_version_status(&self) -> PreviousVersionStatus {
2837        let Some(replica_path) = self.previous_version_replica_path() else {
2838            return PreviousVersionStatus {
2839                present: false,
2840                created_at_ms: None,
2841            };
2842        };
2843        match crate::previous_version::read_previous_version_record(&replica_path) {
2844            Ok(Some(record)) => PreviousVersionStatus {
2845                present: true,
2846                created_at_ms: Some(record.created_at_ms),
2847            },
2848            _ => PreviousVersionStatus {
2849                present: std::path::Path::new(
2850                    &crate::previous_version::previous_version_container_path(&replica_path),
2851                )
2852                .exists(),
2853                created_at_ms: None,
2854            },
2855        }
2856    }
2857
2858    /// RFC 0005 D7: read captured rows from the previous local schema. `state`
2859    /// is always `previousVersion`; the container never makes
2860    /// `query_snapshot()` coverage complete. Lease stop/expiry, scope
2861    /// revocation, coverage completion and the TTL DISCARD the container and
2862    /// close the read with their own reason.
2863    pub fn previous_version_snapshot(
2864        &self,
2865        spec: &PreviousVersionReadSpec,
2866    ) -> Result<PreviousVersionSnapshot, String> {
2867        let now_ms = self.clock_now_ms();
2868        crate::previous_version::previous_version_snapshot(
2869            &self.conn,
2870            self.previous_version_replica_path().as_deref(),
2871            self.previous_version.as_ref(),
2872            PreviousVersionLifecycle {
2873                lease_inactive: self.previous_version_lease_inactive(now_ms),
2874                scope_revoked: self
2875                    .stored(load_units(&self.conn))?
2876                    .iter()
2877                    .any(|unit| unit.status == UnitStatus::Revoked),
2878                coverage_complete: self.previous_version_coverage_complete()?,
2879            },
2880            now_ms,
2881            self.schema.version,
2882            spec,
2883        )
2884    }
2885
2886    /// RFC 0005 D6: the advisory pre-reset compatibility audit.
2887    pub fn previous_version_audit(
2888        &self,
2889    ) -> Result<Option<crate::previous_version::PreviousVersionAudit>, String> {
2890        crate::previous_version::stored_previous_version_audit(&self.conn)
2891            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2892    }
2893
2894    /// RFC 0005 A2/D9: the executable downgrade step. Drop the container file
2895    /// and both metadata records, and report whether anything was present.
2896    /// Idempotent: absence is a successful no-op.
2897    pub fn previous_version_discard(
2898        &self,
2899    ) -> Result<crate::previous_version::PreviousVersionDiscardOutcome, String> {
2900        self.drop_previous_version()
2901    }
2902
2903    fn drop_previous_version(
2904        &self,
2905    ) -> Result<crate::previous_version::PreviousVersionDiscardOutcome, String> {
2906        let Some(replica_path) = self.previous_version_replica_path() else {
2907            return Ok(crate::previous_version::PreviousVersionDiscardOutcome {
2908                present: false,
2909                discarded: false,
2910            });
2911        };
2912        crate::previous_version::discard_previous_version(&self.conn, &replica_path)
2913    }
2914
2915    /// RFC 0005 D7: `leaseState.errorCode` or an expired lease.
2916    fn previous_version_lease_inactive(&self, now_ms: i64) -> bool {
2917        let Some(lease) = &self.lease_state else {
2918            return false;
2919        };
2920        if lease.error_code.is_some() {
2921            return true;
2922        }
2923        lease.expires_at_ms.is_some_and(|expires| expires <= now_ms)
2924    }
2925
2926    /// RFC 0005 D7: §7.4.5 replacement coverage — no active unit is
2927    /// bootstrapping. An empty active set is not completion (mirrors the TS
2928    /// predicate exactly).
2929    fn previous_version_coverage_complete(&self) -> Result<bool, String> {
2930        let units = self.stored(load_units(&self.conn))?;
2931        let mut active = units
2932            .iter()
2933            .filter(|unit| unit.status == UnitStatus::Active)
2934            .peekable();
2935        Ok(active.peek().is_some() && !active.any(UnitRecord::bootstrapping))
2936    }
2937
2938    /// RFC 0005 D7: lifetime triggers evaluated once per sync round, after the
2939    /// apply. A container that no longer has a reason to exist is discarded so
2940    /// the rows do not outlive the window they were kept for.
2941    fn previous_version_lifetime_check(&self) {
2942        let Some(config) = self.previous_version.filter(|config| config.enabled) else {
2943            return;
2944        };
2945        let now_ms = self.clock_now_ms();
2946        if self.previous_version_lease_inactive(now_ms) {
2947            let _ = self.drop_previous_version();
2948            return;
2949        }
2950        let Some(replica_path) = self.previous_version_replica_path() else {
2951            return;
2952        };
2953        let Ok(Some(record)) = crate::previous_version::read_previous_version_record(&replica_path)
2954        else {
2955            return;
2956        };
2957        let Ok(units) = load_units(&self.conn) else {
2958            return;
2959        };
2960        if units.iter().any(|unit| unit.status == UnitStatus::Revoked) {
2961            let _ = self.drop_previous_version();
2962            return;
2963        }
2964        if now_ms - record.created_at_ms > config.max_age_ms {
2965            let _ = self.drop_previous_version();
2966            return;
2967        }
2968        if self.previous_version_coverage_complete() == Ok(true) {
2969            let _ = self.drop_previous_version();
2970        }
2971    }
2972
2973    pub fn diagnostics_snapshot(&self) -> Result<ClientDiagnosticsSnapshot, String> {
2974        let registered = self.stored(load_units(&self.conn))?;
2975        // Both meta values in one statement: the FFI host recomputes
2976        // diagnostics after every command.
2977        let (local_revision, replica_cursor): (Option<String>, Option<String>) = self.stored(
2978            self.conn
2979                .prepare_cached(
2980                    "SELECT (SELECT value FROM _syncular_meta WHERE key = ?1),
2981                            (SELECT value FROM _syncular_meta WHERE key = ?2)",
2982                )
2983                .and_then(|mut statement| {
2984                    statement.query_row([LOCAL_REVISION_KEY, REPLICA_CURSOR_KEY], |row| {
2985                        Ok((row.get(0)?, row.get(1)?))
2986                    })
2987                }),
2988        )?;
2989        let mut units = DiagnosticUnits::default();
2990        for unit in &registered {
2991            *match unit.status {
2992                UnitStatus::Revoked => &mut units.revoked,
2993                UnitStatus::Failed => &mut units.failed,
2994                UnitStatus::Active if !unit.bootstrapping() => &mut units.complete,
2995                UnitStatus::Active
2996                    if unit.reason_code.as_deref() == Some("sync.cursor_expired") =>
2997                {
2998                    &mut units.reset
2999                }
3000                UnitStatus::Active => &mut units.bootstrapping,
3001            } += 1;
3002        }
3003        units.reason_codes = registered
3004            .iter()
3005            .filter_map(|unit| unit.reason_code.as_deref().map(Self::diagnostic_code))
3006            .collect::<BTreeSet<_>>()
3007            .into_iter()
3008            .collect();
3009        let captured_at_ms = self.clock_now_ms();
3010        let lease = if let Some(error_code) = self
3011            .lease_state
3012            .as_ref()
3013            .and_then(|state| state.error_code.clone())
3014        {
3015            ClientDiagnosticsLease {
3016                state: "stopped".to_owned(),
3017                expires_at_ms: self
3018                    .lease_state
3019                    .as_ref()
3020                    .and_then(|state| state.expires_at_ms),
3021                error_code: Some(Self::diagnostic_code(&error_code)),
3022            }
3023        } else if let Some(expires_at_ms) = self
3024            .lease_state
3025            .as_ref()
3026            .and_then(|state| state.expires_at_ms)
3027        {
3028            ClientDiagnosticsLease {
3029                state: if expires_at_ms <= captured_at_ms {
3030                    "expired".to_owned()
3031                } else {
3032                    "active".to_owned()
3033                },
3034                expires_at_ms: Some(expires_at_ms),
3035                error_code: None,
3036            }
3037        } else {
3038            ClientDiagnosticsLease {
3039                state: "none".to_owned(),
3040                expires_at_ms: None,
3041                error_code: None,
3042            }
3043        };
3044        let connectivity = match self.last_round.as_ref() {
3045            _ if !self.transport_enabled && self.active_round.is_none() => "offline",
3046            Some(round) if round.status == "succeeded" => "online",
3047            Some(round)
3048                if round.status == "failed"
3049                    && matches!(
3050                        round.cause_kind,
3051                        Some(
3052                            crate::TransportCauseKind::Network | crate::TransportCauseKind::Connect
3053                        )
3054                    ) =>
3055            {
3056                "offline"
3057            }
3058            Some(round) if round.status == "failed" && round.http_status.is_some() => "online",
3059            _ => "unknown",
3060        };
3061        Ok(ClientDiagnosticsSnapshot {
3062            version: CLIENT_DIAGNOSTICS_VERSION,
3063            captured_at_ms,
3064            host: ClientDiagnosticsHost {
3065                kind: "direct".to_owned(),
3066                role: "single".to_owned(),
3067                connectivity: connectivity.to_owned(),
3068                realtime: self.realtime_state,
3069                realtime_policy: self.realtime_policy,
3070                realtime_reason_code: self.realtime_reason_code.clone(),
3071                realtime_retry_delay_ms: self.realtime_retry_delay_ms,
3072            },
3073            security_lifecycle: self.security_lifecycle().to_owned(),
3074            schema: ClientDiagnosticsSchema {
3075                current_version: self.schema.version,
3076                upgrading: self.upgrading,
3077                required_version: self
3078                    .schema_floor
3079                    .as_ref()
3080                    .and_then(|floor| floor.required_schema_version),
3081                latest_version: self
3082                    .schema_floor
3083                    .as_ref()
3084                    .and_then(|floor| floor.latest_schema_version),
3085            },
3086            replica: ClientDiagnosticsReplica {
3087                local_revision: local_revision
3088                    .and_then(|value| value.parse::<u64>().ok())
3089                    .unwrap_or(0)
3090                    .to_string(),
3091                sync_needed: self.sync_needed,
3092                pending_outbox: self.outbox.len(),
3093            },
3094            lease,
3095            active_units: registered
3096                .iter()
3097                .filter(|unit| unit.status == UnitStatus::Active)
3098                .count(),
3099            units,
3100            replica_cursor: replica_cursor.map_or(Ok(-1), |raw| {
3101                raw.parse()
3102                    .map_err(|_| "sync.local_corrupt: persisted unit state is invalid".to_owned())
3103            })?,
3104            last_round: self.last_round.clone(),
3105            last_change: self.last_change.clone(),
3106            storage: self.diagnostics_storage(),
3107            query_failures: self.query_failures.clone(),
3108        })
3109    }
3110
3111    fn diagnostics_storage(&self) -> ClientDiagnosticsStorage {
3112        let read = || -> Result<ClientDiagnosticsStorage, rusqlite::Error> {
3113            let page_count: i64 = self
3114                .conn
3115                .query_row("PRAGMA page_count", [], |row| row.get(0))?;
3116            let page_size: i64 = self
3117                .conn
3118                .query_row("PRAGMA page_size", [], |row| row.get(0))?;
3119            // One cached aggregate, since the FFI host recomputes diagnostics
3120            // after every command; every invocation still reads current storage.
3121            let (outbox_bytes, outcome_entries, outcome_bytes): (i64, i64, i64) = self
3122                .conn
3123                .prepare_cached(
3124                    "SELECT (SELECT COALESCE(SUM(LENGTH(ops_json)), 0) FROM _syncular_outbox),
3125                            COUNT(*), COALESCE(SUM(LENGTH(results_json) + COALESCE(LENGTH(operations_json), 0)), 0)
3126                       FROM _syncular_commit_outcomes",
3127                )?
3128                .query_row([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))?;
3129            let blob_bytes = if self.schema_has_blobs() {
3130                self.conn
3131                    .prepare_cached("SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs")?
3132                    .query_row([], |row| row.get(0))?
3133            } else {
3134                0
3135            };
3136            let pressure = self
3137                .limits
3138                .blob_cache_max_bytes
3139                .is_some_and(|limit| blob_bytes > limit);
3140            Ok(ClientDiagnosticsStorage {
3141                status: if pressure { "pressure" } else { "healthy" }.to_owned(),
3142                database_bytes_approx: Some(page_count.saturating_mul(page_size).max(0)),
3143                pending_outbox_bytes_approx: Some(outbox_bytes.max(0)),
3144                retained_outcome_bytes_approx: Some(outcome_bytes.max(0)),
3145                retained_outcome_entries: Some(outcome_entries.max(0)),
3146                blob_cache_bytes_approx: Some(blob_bytes.max(0)),
3147                pressure_reason_code: pressure.then(|| "client.blob_cache_over_limit".to_owned()),
3148            })
3149        };
3150        read().unwrap_or_else(|_| ClientDiagnosticsStorage {
3151            status: "unreadable".to_owned(),
3152            database_bytes_approx: None,
3153            pending_outbox_bytes_approx: None,
3154            retained_outcome_bytes_approx: None,
3155            retained_outcome_entries: None,
3156            blob_cache_bytes_approx: None,
3157            pressure_reason_code: None,
3158        })
3159    }
3160
3161    pub fn drain_change_batches(&mut self) -> Vec<ClientChangeBatch> {
3162        self.change_queue.drain(..).collect()
3163    }
3164
3165    pub fn drain_sync_intents(&mut self) -> Vec<SyncIntent> {
3166        let intents = self.sync_intent_queue.drain(..).collect();
3167        if self.transport_enabled {
3168            intents
3169        } else {
3170            vec![SyncIntent::None]
3171        }
3172    }
3173
3174    fn schedule_background_retry(&mut self) -> u64 {
3175        let delay_ms = self.retry_delay_ms;
3176        self.sync_intent_queue
3177            .push_back(SyncIntent::Background { delay_ms });
3178        self.retry_delay_ms = (self.retry_delay_ms * 2).min(30_000);
3179        self.round_retry_delay_ms = Some(delay_ms);
3180        delay_ms
3181    }
3182
3183    fn reset_background_retry(&mut self) {
3184        self.retry_delay_ms = 250;
3185    }
3186
3187    fn retryable_transport_code(code: &str) -> bool {
3188        code == "transport.failed"
3189            || code == "transport.timeout"
3190            || code == "transport.unavailable"
3191            || code == "sync.transport_failed"
3192    }
3193
3194    /// §1.8: the transport reported a refused or unreachable endpoint. The
3195    /// native transport says `connect` (DNS failure, connection refused,
3196    /// reset, or aborted); a browser fetch transport says `network`. Repeating
3197    /// the request on a backoff repairs neither, so no retry is scheduled.
3198    fn unreachable_transport(details: &Option<serde_json::Value>) -> bool {
3199        matches!(
3200            details
3201                .as_ref()
3202                .and_then(|details| details.get("causeKind"))
3203                .and_then(serde_json::Value::as_str),
3204            Some("connect" | "network")
3205        )
3206    }
3207
3208    /// §1.5: a segment fetch that fails inside an otherwise complete round is
3209    /// invisible to the server (the round response arrived whole), so its code
3210    /// rides the following requests like a socket-round failure. A code the
3211    /// frame grammar rejects is not reportable and the previous code stays.
3212    fn segment_fetch_failed(&mut self, error: TransportError) -> SectionError {
3213        if ssp2::model::valid_round_failure_code(&error.code) {
3214            self.round_failure = Some(error.code.clone());
3215        }
3216        SectionError::Transport(error)
3217    }
3218
3219    /// A failed round worth a background retry: a transport failure, or a
3220    /// §10.2 code the catalog marks retryable (clients may hardcode the
3221    /// catalog metadata, §10.1). Matches the TypeScript core, which retries
3222    /// on the server's `retryable` flag.
3223    fn retryable_failure_code(code: &str) -> bool {
3224        Self::retryable_transport_code(code)
3225            || matches!(
3226                code,
3227                "sync.auth_required"
3228                    | "sync.auth_lease_required"
3229                    | "sync.auth_lease_revoked"
3230                    | "sync.internal_error"
3231                    | "sync.idempotency_cache_miss"
3232                    | "sync.schema_not_ready"
3233                    | "sync.segment_expired"
3234                    | "sync.rate_limited"
3235                    | "sync.websocket_connection_limit"
3236                    | "sync.idle_timeout"
3237            )
3238    }
3239
3240    fn diagnostic_code(code: &str) -> String {
3241        let valid = !code.is_empty()
3242            && code.len() <= 96
3243            && code.contains('.')
3244            && code
3245                .bytes()
3246                .next()
3247                .is_some_and(|byte| byte.is_ascii_lowercase())
3248            && code.bytes().all(|byte| {
3249                byte.is_ascii_lowercase()
3250                    || byte.is_ascii_digit()
3251                    || matches!(byte, b'.' | b'_' | b'-')
3252            });
3253        if valid {
3254            code.to_owned()
3255        } else {
3256            "client.unknown_failure".to_owned()
3257        }
3258    }
3259
3260    fn set_sync_needed(&mut self, value: bool, interactive: bool) {
3261        if self.sync_needed != value {
3262            if self.begin_observation("syncular_status").is_ok() {
3263                self.sync_needed = value;
3264                let batch = ChangeAccumulator {
3265                    status: true,
3266                    ..ChangeAccumulator::default()
3267                };
3268                if self.finish_observation("syncular_status", batch).is_err() {
3269                    self.rollback_observation("syncular_status");
3270                }
3271            } else {
3272                self.sync_needed = value;
3273            }
3274        }
3275        if value && interactive {
3276            self.sync_intent_queue.push_back(SyncIntent::Interactive);
3277        }
3278    }
3279
3280    /// A unit-store result: a row that does not parse is `sync.local_corrupt`;
3281    /// every other SQLite error keeps its storage classification.
3282    fn stored<T>(&self, result: rusqlite::Result<T>) -> Result<T, String> {
3283        result.map_err(|error| match error {
3284            rusqlite::Error::FromSqlConversionFailure(..) => {
3285                "sync.local_corrupt: persisted unit state is invalid".to_owned()
3286            }
3287            error => Self::sqlite_failure(&self.storage_failure, error),
3288        })
3289    }
3290
3291    fn sqlite_failure(
3292        storage_failure: &RefCell<Option<QueryReadFailure>>,
3293        error: rusqlite::Error,
3294    ) -> String {
3295        let message = error.to_string();
3296        let failure = QueryReadFailure::from(error);
3297        let code = failure.code;
3298        if code.is_some() && storage_failure.borrow().is_none() {
3299            *storage_failure.borrow_mut() = Some(failure);
3300        }
3301        match code {
3302            Some(code) => format!("{code}: {message}"),
3303            None => message,
3304        }
3305    }
3306
3307    fn begin_observation(&self, name: &str) -> Result<(), String> {
3308        if let Some(failure) = &self.coalesced_failure {
3309            return Err(failure.clone());
3310        }
3311        // SQLite rolls a whole transaction back on some I/O and capacity
3312        // errors; later frames must not commit on their own after that.
3313        if self.coalesced.is_some() && self.conn.is_autocommit() {
3314            return Err("the coalesced apply transaction was rolled back".to_owned());
3315        }
3316        self.conn
3317            .execute_batch(&format!("SAVEPOINT {name}"))
3318            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
3319    }
3320
3321    /// SPEC §1.4 rule 4: open a coalesced apply transaction unless a caller
3322    /// already owns one.
3323    fn begin_coalesced_apply(&mut self) -> Result<(), String> {
3324        if self.coalesced.is_some() || !self.conn.is_autocommit() {
3325            return Ok(());
3326        }
3327        self.conn
3328            .execute_batch("BEGIN")
3329            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3330        self.coalesced = Some(CoalescedApply {
3331            rows: 0,
3332            rollback: self.capture(),
3333        });
3334        Ok(())
3335    }
3336
3337    fn capture(&self) -> Rollback {
3338        Rollback {
3339            outbox: self.outbox.clone(),
3340            failed_commits: self.failed_commits.clone(),
3341            conflicts: self.conflicts.clone(),
3342            rejections: self.rejections.clone(),
3343            schema_floor: self.schema_floor.clone(),
3344            lease_state: self.lease_state.clone(),
3345            stopped: self.stopped,
3346            upgrading: self.upgrading,
3347            sync_needed: self.sync_needed,
3348            active_round: self.active_round,
3349            overlay_dirty: self.overlay_dirty.snapshot(),
3350            change_queue: self.change_queue.len(),
3351            sync_intent_queue: self.sync_intent_queue.len(),
3352            last_change: self.last_change.clone(),
3353        }
3354    }
3355
3356    fn restore(&mut self, rollback: Rollback) {
3357        self.outbox = rollback.outbox;
3358        self.failed_commits = rollback.failed_commits;
3359        self.conflicts = rollback.conflicts;
3360        self.rejections = rollback.rejections;
3361        self.schema_floor = rollback.schema_floor;
3362        self.lease_state = rollback.lease_state;
3363        self.stopped = rollback.stopped;
3364        self.upgrading = rollback.upgrading;
3365        self.sync_needed = rollback.sync_needed;
3366        self.active_round = rollback.active_round;
3367        self.overlay_dirty.restore(rollback.overlay_dirty);
3368        self.change_queue.truncate(rollback.change_queue);
3369        self.sync_intent_queue.truncate(rollback.sync_intent_queue);
3370        self.last_change = rollback.last_change;
3371    }
3372
3373    /// Account the rows of the next frame or block. A unit that would take the
3374    /// coalesced transaction past the bound commits the earlier units first.
3375    fn coalesce_unit(&mut self, rows: usize) -> Result<(), String> {
3376        let Some(open) = &self.coalesced else {
3377            return Ok(());
3378        };
3379        if open.rows > 0 && open.rows + rows > COALESCED_APPLY_ROWS {
3380            self.commit_coalesced_apply()?;
3381            self.begin_coalesced_apply()?;
3382        }
3383        if let Some(open) = &mut self.coalesced {
3384            open.rows += rows;
3385        }
3386        Ok(())
3387    }
3388
3389    /// COMMIT the coalesced transaction. A failed COMMIT rolls every coalesced
3390    /// frame back, restores the in-memory state of its `BEGIN`, drops the
3391    /// change batches and sync intents queued since, and fails the rest of the
3392    /// response.
3393    fn commit_coalesced_apply(&mut self) -> Result<(), String> {
3394        let Some(open) = self.coalesced.take() else {
3395            return Ok(());
3396        };
3397        let committed = if self.conn.is_autocommit() {
3398            Err("the coalesced apply transaction was rolled back".to_owned())
3399        } else {
3400            self.conn
3401                .execute_batch("COMMIT")
3402                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
3403        };
3404        let Err(error) = committed else {
3405            return Ok(());
3406        };
3407        if !self.conn.is_autocommit() {
3408            if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
3409                let _ = Self::sqlite_failure(&self.storage_failure, rollback);
3410            }
3411        }
3412        self.restore(open.rollback);
3413        self.coalesced_failure = Some(error.clone());
3414        Err(error)
3415    }
3416
3417    /// End the response's coalescing: COMMIT what is open and clear a failure.
3418    fn finish_coalesced_apply(&mut self) -> Result<(), String> {
3419        let committed = self.commit_coalesced_apply();
3420        match self.coalesced_failure.take() {
3421            Some(failure) => Err(failure),
3422            None => committed,
3423        }
3424    }
3425
3426    fn rollback_observation(&self, name: &str) {
3427        if let Err(error) = self
3428            .conn
3429            .execute_batch(&format!("ROLLBACK TO {name}; RELEASE {name}"))
3430        {
3431            if let Some(failure) = self.storage_failure.borrow_mut().as_mut() {
3432                if failure.rollback_failure.is_none() {
3433                    failure.rollback_failure = Some(Box::new(QueryReadFailure::from(error)));
3434                }
3435            }
3436        }
3437    }
3438
3439    fn finish_observation(
3440        &mut self,
3441        name: &str,
3442        mut batch: ChangeAccumulator,
3443    ) -> Result<(), String> {
3444        #[cfg(feature = "bench-internals")]
3445        let _phase = self.benchmark_phases.start(Phase::ObservationCommit);
3446        if self.failed_commits.iter().any(|(commit, _)| {
3447            commit
3448                .ops
3449                .iter()
3450                .any(|operation| batch.tables.contains_key(&operation.table))
3451        }) {
3452            batch.outcomes = true;
3453        }
3454        if !batch.touched() {
3455            self.conn
3456                .execute_batch(&format!("RELEASE {name}"))
3457                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3458            return Ok(());
3459        }
3460        let revision = self
3461            .local_revision()?
3462            .checked_add(1)
3463            .ok_or_else(|| "local revision exhausted u64".to_owned())?;
3464        self.conn
3465            .execute(
3466                "INSERT OR REPLACE INTO _syncular_meta(key, value) VALUES (?1, ?2)",
3467                rusqlite::params![LOCAL_REVISION_KEY, revision.to_string()],
3468            )
3469            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3470        let status = batch.status.then(|| self.status_snapshot());
3471        let event = ClientChangeBatch {
3472            revision: revision.to_string(),
3473            tables: batch
3474                .tables
3475                .into_iter()
3476                .map(|(table, scope_keys)| TableChange {
3477                    table,
3478                    scope_keys: scope_keys.map(|keys| keys.into_iter().collect()),
3479                })
3480                .collect(),
3481            windows: batch
3482                .windows
3483                .into_iter()
3484                .map(|(base_key, units)| WindowChange {
3485                    base_key,
3486                    units: units.into_iter().collect(),
3487                })
3488                .collect(),
3489            status,
3490            conflicts_changed: batch.conflicts,
3491            rejections_changed: batch.rejections,
3492            outcomes_changed: batch.outcomes,
3493            applied_commits: Vec::new(),
3494        };
3495        self.conn
3496            .execute_batch(&format!("RELEASE {name}"))
3497            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3498        let mut diagnostic_tables = event
3499            .tables
3500            .iter()
3501            .map(|entry| entry.table.clone())
3502            .collect::<BTreeSet<_>>()
3503            .into_iter()
3504            .collect::<Vec<_>>();
3505        // §7.6: the selected variables of the changed window bases.
3506        let mut diagnostic_windows = event
3507            .windows
3508            .iter()
3509            .map(|entry| {
3510                entry
3511                    .base_key
3512                    .split('\0')
3513                    .next()
3514                    .unwrap_or_default()
3515                    .to_owned()
3516            })
3517            .collect::<BTreeSet<_>>()
3518            .into_iter()
3519            .collect::<Vec<_>>();
3520        let domains_truncated = diagnostic_tables.len() > MAX_DIAGNOSTIC_DOMAINS
3521            || diagnostic_windows.len() > MAX_DIAGNOSTIC_DOMAINS;
3522        diagnostic_tables.truncate(MAX_DIAGNOSTIC_DOMAINS);
3523        diagnostic_windows.truncate(MAX_DIAGNOSTIC_DOMAINS);
3524        self.last_change = Some(DiagnosticLastChange {
3525            revision: event.revision.clone(),
3526            recorded_at_ms: self.clock_now_ms(),
3527            tables: diagnostic_tables,
3528            windows: diagnostic_windows,
3529            domains_truncated,
3530            status_changed: event.status.is_some(),
3531            conflicts_changed: event.conflicts_changed,
3532            rejections_changed: event.rejections_changed,
3533            outcomes_changed: event.outcomes_changed,
3534        });
3535        self.change_queue.push_back(event);
3536        Ok(())
3537    }
3538
3539    fn record_scope_map(
3540        &self,
3541        batch: &mut ChangeAccumulator,
3542        table_name: &str,
3543        scopes: &[(String, Vec<String>)],
3544    ) {
3545        let Some(table) = self.schema.table(table_name) else {
3546            return;
3547        };
3548        // A bulk scoped delete whose map leaves a declared variable unnamed
3549        // cannot list the deleted rows' keys for it: the change is table-wide.
3550        if table.scope_variables.iter().any(|scope| {
3551            !scopes
3552                .iter()
3553                .any(|(variable, _)| *variable == scope.variable)
3554        }) {
3555            batch.table(table_name);
3556            return;
3557        }
3558        for (variable, values) in scopes {
3559            let Some(scope) = table
3560                .scope_variables
3561                .iter()
3562                .find(|scope| &scope.variable == variable)
3563            else {
3564                continue;
3565            };
3566            for value in values {
3567                batch.scope(table_name, format!("{}:{value}", scope.prefix));
3568            }
3569        }
3570    }
3571
3572    /// Record a row's current scope keys from the base or visible table.
3573    fn record_row_scopes(
3574        &self,
3575        batch: &mut ChangeAccumulator,
3576        table_name: &str,
3577        row_id: &str,
3578        base: bool,
3579    ) -> bool {
3580        let Some(table) = self.schema.table(table_name) else {
3581            return false;
3582        };
3583        if table.scope_variables.is_empty() {
3584            return false;
3585        }
3586        let columns = table
3587            .scope_variables
3588            .iter()
3589            .map(|scope| quote_ident(&scope.column))
3590            .collect::<Vec<_>>()
3591            .join(", ");
3592        let full_table = if base {
3593            base_table(table_name)
3594        } else {
3595            visible_table(table_name)
3596        };
3597        let sql = format!(
3598            "SELECT {columns} FROM {full_table} WHERE {} LIMIT 1",
3599            row_id_predicate(table)
3600        );
3601        let Ok(mut stmt) = self.conn.prepare_cached(&sql) else {
3602            return false;
3603        };
3604        let values = stmt.query_row(rusqlite::params![row_id], |row| {
3605            let mut values = Vec::with_capacity(table.scope_variables.len());
3606            for index in 0..table.scope_variables.len() {
3607                values.push(row.get::<_, Option<String>>(index)?);
3608            }
3609            Ok(values)
3610        });
3611        let Ok(values) = values else {
3612            return false;
3613        };
3614        let mut recorded = false;
3615        for (scope, value) in table.scope_variables.iter().zip(values) {
3616            if let Some(value) = value {
3617                batch.scope(table_name, format!("{}:{value}", scope.prefix));
3618                recorded = true;
3619            }
3620        }
3621        recorded
3622    }
3623
3624    fn record_commit_changes(
3625        &self,
3626        batch: &mut ChangeAccumulator,
3627        tables: &[String],
3628        changes: &[ssp2::model::Change],
3629    ) {
3630        #[cfg(feature = "bench-internals")]
3631        let _phase = self.benchmark_phases.start(Phase::ObservationPrepare);
3632        for change in changes {
3633            let Some(table_name) = tables.get(change.table_index as usize) else {
3634                continue;
3635            };
3636            let mut precise = self.record_row_scopes(batch, table_name, &change.row_id, true);
3637            if let Some(table) = self.schema.table(table_name) {
3638                for (variable, value) in &change.scopes {
3639                    if let Some(scope) = table
3640                        .scope_variables
3641                        .iter()
3642                        .find(|scope| &scope.variable == variable)
3643                    {
3644                        batch.scope(table_name, format!("{}:{value}", scope.prefix));
3645                        precise = true;
3646                    }
3647                }
3648            }
3649            if !precise {
3650                batch.table(table_name);
3651            }
3652        }
3653    }
3654
3655    /// Whether a base or visible row matches the scope predicate; `false`
3656    /// without a local scope-column mapping. A SQLite failure is an error.
3657    fn scoped_rows_exist(
3658        &self,
3659        table_name: &str,
3660        effective: &[(String, Vec<String>)],
3661        owners: &[Vec<(String, Vec<String>)>],
3662    ) -> Result<bool, String> {
3663        let Some(table) = self.schema.table(table_name) else {
3664            return Ok(false);
3665        };
3666        let Ok(Some((predicate, params))) = local_scope_predicate(table, effective, owners) else {
3667            return Ok(false);
3668        };
3669        for source in [base_table(table_name), visible_table(table_name)] {
3670            if self
3671                .conn
3672                .query_row(
3673                    &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {predicate})"),
3674                    rusqlite::params_from_iter(&params),
3675                    |row| row.get::<_, bool>(0),
3676                )
3677                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
3678            {
3679                return Ok(true);
3680            }
3681        }
3682        Ok(false)
3683    }
3684
3685    /// §5.6 first-page clear of `unit_id`'s pair: delete its rows except
3686    /// those another owner covers (§3.3), failing closed without a local
3687    /// scope-column mapping. Owners are read only when a row matches, inside
3688    /// the caller's transaction. Returns whether rows matched.
3689    fn clear_scope(
3690        &mut self,
3691        table: &TableSchema,
3692        effective: &[(String, Vec<String>)],
3693        unit_id: &str,
3694    ) -> Result<bool, SectionError> {
3695        let storage = |message| SectionError::Abort("storage.failed".to_owned(), message);
3696        let matched = self
3697            .scoped_rows_exist(&table.name, effective, &[])
3698            .map_err(storage)?;
3699        let owners = if matched {
3700            unit_owners(
3701                &self.stored(load_units(&self.conn)).map_err(storage)?,
3702                table,
3703                unit_id,
3704            )
3705        } else {
3706            Vec::new()
3707        };
3708        let changed = matched
3709            && (owners.is_empty()
3710                || self
3711                    .scoped_rows_exist(&table.name, effective, &owners)
3712                    .map_err(storage)?);
3713        if !self
3714            .purge_scope_rows(&table.name, effective, &owners)
3715            .map_err(storage)?
3716        {
3717            return Err(SectionError::FailClosed);
3718        }
3719        Ok(changed)
3720    }
3721
3722    /// §7.4.5: true while a schema-bump reset + first re-bootstrap runs.
3723    pub fn upgrading(&self) -> bool {
3724        self.upgrading
3725    }
3726
3727    /// §7.4.2 "app ships new code": swap to a NEW generated schema while
3728    /// keeping this client's local database (identity, outbox, tables). The
3729    /// §7.4.1 marker check then fires the wipe/re-bootstrap flow when the
3730    /// version increases. Recreation runs the same marker guard as opening
3731    /// a durable replica, holds the guard and every write it authorizes under
3732    /// one write transaction, and preserves the current client (compiled
3733    /// schema, outbox, tables) on refusal.
3734    pub fn recreate_with_schema(&mut self, schema_json: &Value) -> Result<(), String> {
3735        let new_schema = parse_schema_json(schema_json)?;
3736        validate_authority_reads(&new_schema, &self.limits.authority_reads)?;
3737        self.conn
3738            .execute_batch("BEGIN IMMEDIATE")
3739            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3740        let prior_schema = std::mem::replace(&mut self.schema, new_schema);
3741        // §7.4.3/§7.4.4: a reset inside this transaction mutates in-memory
3742        // state (dropped outbox commits, rejections, upgrade and floor flags,
3743        // overlay dirtiness) and appends change/intent batches. A COMMIT that
3744        // fails after the reset succeeded rolls the database back, so every
3745        // one of those fields is captured here and restored with it.
3746        let rollback = self.capture();
3747        let guarded = (|| -> Result<(), String> {
3748            // §7.4.2: the marker is re-read under the write lock the caller
3749            // holds, so a replica another process upgraded meanwhile refuses
3750            // this recreation instead of having its marker and tables
3751            // downgraded.
3752            let marker = read_local_schema_version(&self.conn, self.schema.version)?;
3753            self.create_bookkeeping_tables()?;
3754            match marker {
3755                Some(version) if version < self.schema.version => self.run_schema_reset()?,
3756                None => {
3757                    self.create_synced_tables()?;
3758                    self.persist_schema_version()?;
3759                }
3760                _ => {}
3761            }
3762            // §7.4.2: recreation IS the boot for the in-memory core, so reload the
3763            // durable outbox the same-version path is about to replay. Only the
3764            // outbox is reloaded: the recovery below reads it, and the caller's
3765            // rollback snapshot already covers this field.
3766            self.outbox = self.load_outbox()?.into();
3767            // §7.1/§7.4.4: recover intrinsically codec-invalid persisted values at
3768            // this guarded boundary once the schema and marker agree. A bump went
3769            // through `run_schema_reset`, which performs the same recovery after
3770            // it creates the new physical tables and before its final replay.
3771            self.drop_unencodable_outbox(false, None)?;
3772            if marker == Some(self.schema.version)
3773                && (!self.outbox.is_empty()
3774                    || !self.failed_commits.is_empty()
3775                    || self.has_acknowledged_rows()?)
3776            {
3777                self.overlay_dirty.set(true);
3778                self.rebuild_overlay()?;
3779            }
3780            self.prune_unknown_units(false)?;
3781            self.save_unit_scope_schema()?;
3782            // The conformance recreate is the in-memory equivalent of reopening
3783            // a durable client. Apply the same startup catch-up contract even
3784            // when the schema itself did not change.
3785            self.enqueue_startup_sync_if_needed()?;
3786            Ok(())
3787        })();
3788        let outcome = match guarded {
3789            Ok(()) => self
3790                .conn
3791                .execute_batch("COMMIT")
3792                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error)),
3793            Err(error) => Err(error),
3794        };
3795        if let Err(error) = outcome {
3796            if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
3797                // Keep a failed rollback's typed storage classification.
3798                let _ = Self::sqlite_failure(&self.storage_failure, rollback);
3799            }
3800            self.schema = prior_schema;
3801            self.restore(rollback);
3802            // Prepared insert statements were derived from the reverted schema.
3803            self.insert_sql.borrow_mut().clear();
3804            return Err(error);
3805        }
3806        if !self.security_preflight {
3807            self.drain_pending_evictions()?;
3808        }
3809        Ok(())
3810    }
3811
3812    fn enqueue_startup_sync_if_needed(&mut self) -> Result<(), String> {
3813        let startup_work = !self.stopped
3814            && (!self.outbox.is_empty()
3815                || self.has_acknowledged_rows()?
3816                || !self.load_pending_evictions()?.is_empty()
3817                || self
3818                    .stored(load_units(&self.conn))?
3819                    .iter()
3820                    .any(|unit| unit.status == UnitStatus::Active));
3821        if startup_work {
3822            self.sync_needed = true;
3823            self.sync_intent_queue.push_back(SyncIntent::Interactive);
3824        }
3825        Ok(())
3826    }
3827
3828    /// A native client persists schema-floor stops across process restarts.
3829    /// Once the running app already satisfies that floor, the persisted stop
3830    /// is only stale evidence from an older server round (for example, the
3831    /// server was restarted after catching up to an app that was ahead).
3832    /// Clear it and let the normal startup pull re-negotiate. If the server is
3833    /// still incompatible it will return the floor again in that first round.
3834    fn clear_satisfied_persisted_schema_floor(&mut self) -> Result<(), String> {
3835        let satisfied = self
3836            .schema_floor
3837            .as_ref()
3838            .and_then(|floor| floor.required_schema_version)
3839            .is_some_and(|required| self.schema.version >= required);
3840        if !satisfied {
3841            return Ok(());
3842        }
3843        self.delete_meta(SCHEMA_FLOOR_KEY)?;
3844        self.schema_floor = None;
3845        self.stopped = false;
3846        Ok(())
3847    }
3848
3849    /// RFC 0005 D1: discard an orphan or stale container at boot. The reset
3850    /// path already swept it; this covers a same-version or fresh-install open,
3851    /// which runs no reset and therefore no sweep — plus the D7 TTL, which only
3852    /// an aware binary can apply.
3853    fn reconcile_previous_version_at_boot(&self) -> Result<(), String> {
3854        let Some(replica_path) = self.previous_version_replica_path() else {
3855            return Ok(());
3856        };
3857        let max_age_ms = self
3858            .previous_version
3859            .filter(|config| config.enabled)
3860            .map(|config| config.max_age_ms);
3861        reconcile_previous_version_at_boot(
3862            &self.conn,
3863            &replica_path,
3864            self.schema.version,
3865            max_age_ms,
3866            self.clock_now_ms(),
3867        )
3868    }
3869
3870    /// RFC 0005 D3: the sibling container path, or `None` when the replica has
3871    /// no file — an in-memory replica returns an empty path, and a `file:` URI
3872    /// has no plain sibling path — so the feature resolves as off there.
3873    fn previous_version_replica_path(&self) -> Option<String> {
3874        let path = self.conn.path()?;
3875        if path.is_empty() || path.starts_with("file:") {
3876            return None;
3877        }
3878        Some(path.to_owned())
3879    }
3880
3881    /// §7.4.3 reset: whole-database local reset EXCEPT the outbox, clientId,
3882    /// and leaseState. Drops/recreates every synced table from the new
3883    /// schema, resets unit sync state (keeping registrations), clears
3884    /// the schema-floor stop state, rewrites the marker, drops outbox commits
3885    /// that cannot re-encode (§7.4.4), and replays the survivors on top.
3886    fn run_schema_reset(&mut self) -> Result<(), String> {
3887        self.begin_observation("syncular_schema_reset")?;
3888        let rollback = self.capture();
3889        let mut batch = ChangeAccumulator::default();
3890        let result = self
3891            .run_schema_reset_observed(&mut batch, true)
3892            .and_then(|()| self.finish_observation("syncular_schema_reset", batch));
3893        if let Err(error) = result {
3894            self.rollback_observation("syncular_schema_reset");
3895            self.restore(rollback);
3896            return Err(error);
3897        }
3898        Ok(())
3899    }
3900
3901    fn run_log_epoch_reset(&mut self, log_epoch: &str) -> Result<Vec<String>, String> {
3902        if self.get_meta(LOG_EPOCH_KEY)?.is_none() {
3903            self.begin_observation("syncular_epoch_acquisition")?;
3904            let rollback = self.capture();
3905            if let Err(error) = self.conn.execute(
3906                "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
3907                rusqlite::params![LOG_EPOCH_KEY, log_epoch],
3908            ) {
3909                self.rollback_observation("syncular_epoch_acquisition");
3910                return Err(Self::sqlite_failure(&self.storage_failure, error));
3911            }
3912            self.sync_needed = true;
3913            let batch = ChangeAccumulator {
3914                status: true,
3915                ..ChangeAccumulator::default()
3916            };
3917            if let Err(error) = self.finish_observation("syncular_epoch_acquisition", batch) {
3918                self.rollback_observation("syncular_epoch_acquisition");
3919                self.restore(rollback);
3920                return Err(error);
3921            }
3922            self.sync_intent_queue.push_back(SyncIntent::Interactive);
3923            return Ok(Vec::new());
3924        }
3925        let resets = self
3926            .stored(load_units(&self.conn))?
3927            .into_iter()
3928            .map(|unit| unit.id)
3929            .collect();
3930        self.begin_observation("syncular_log_epoch_reset")?;
3931        let rollback = self.capture();
3932        let mut batch = ChangeAccumulator::default();
3933        let result = self
3934            .run_schema_reset_observed(&mut batch, false)
3935            .and_then(|()| {
3936                self.conn
3937                    .execute(
3938                        "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
3939                        rusqlite::params![LOG_EPOCH_KEY, log_epoch],
3940                    )
3941                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3942                self.sync_needed = true;
3943                batch.status = true;
3944                self.finish_observation("syncular_log_epoch_reset", batch)
3945            });
3946        if let Err(error) = result {
3947            self.rollback_observation("syncular_log_epoch_reset");
3948            self.restore(rollback);
3949            return Err(error);
3950        }
3951        self.sync_intent_queue.push_back(SyncIntent::Interactive);
3952        Ok(resets)
3953    }
3954
3955    fn run_schema_reset_observed(
3956        &mut self,
3957        batch: &mut ChangeAccumulator,
3958        capture: bool,
3959    ) -> Result<(), String> {
3960        // §7.4.2: the reset is the only path that wipes local tables for a
3961        // version bump, and it rewrites the marker last. Re-read the marker
3962        // under this reset's transaction: a replica another process upgraded
3963        // meanwhile refuses here instead of being reset to this client's
3964        // older schema.
3965        read_local_schema_version(&self.conn, self.schema.version)?;
3966        // §7.4.3: a schema bump keeps every table whose generation equals the
3967        // persisted one; the log-epoch reset keeps nothing. The previous
3968        // declaration evidence decides which units cover only kept tables.
3969        let mut kept = BTreeSet::new();
3970        let mut previous_declarations: Option<BTreeMap<String, BTreeMap<String, Value>>> = None;
3971        if capture {
3972            let persisted: BTreeMap<String, String> = self
3973                .get_meta(TABLE_GENERATIONS_KEY)?
3974                .map(|raw| serde_json::from_str(&raw))
3975                .transpose()
3976                .map_err(|_| "sync.local_corrupt: invalid persisted table generations".to_owned())?
3977                .unwrap_or_default();
3978            kept = self
3979                .schema
3980                .tables
3981                .iter()
3982                .filter(|table| {
3983                    table.generation.is_some()
3984                        && table.generation.as_ref() == persisted.get(&table.name)
3985                })
3986                .map(|table| table.name.clone())
3987                .collect();
3988            previous_declarations = self
3989                .get_meta("subscriptionScopeSchema")?
3990                .map(|raw| serde_json::from_str(&raw))
3991                .transpose()
3992                .map_err(|_| "sync.local_corrupt: invalid persisted scope schema".to_owned())?;
3993        }
3994        // Kept tables keep both halves, their FTS projections with the FTS5
3995        // shadow tables and id mapping, and their row-level bookkeeping.
3996        let mut kept_objects = BTreeSet::new();
3997        for table in self
3998            .schema
3999            .tables
4000            .iter()
4001            .filter(|table| kept.contains(&table.name))
4002        {
4003            kept_objects.insert(table.name.clone());
4004            kept_objects.insert(format!("_syncular_base_{}", table.name));
4005            for index in &table.fts_indexes {
4006                kept_objects.insert(index.name.clone());
4007                for shadow in ["data", "idx", "content", "docsize", "config"] {
4008                    kept_objects.insert(format!("{}_{shadow}", index.name));
4009                }
4010            }
4011        }
4012        let kept_json = serde_json::to_string(&kept).expect("table names serialize");
4013        for sql in [
4014            "DELETE FROM _syncular_acked_rows WHERE tbl NOT IN (SELECT value FROM json_each(?1))",
4015            "DELETE FROM _syncular_row_deliveries WHERE tbl NOT IN (SELECT value FROM json_each(?1))",
4016        ] {
4017            self.conn
4018                .execute(sql, [&kept_json])
4019                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4020        }
4021        self.cancel_sync_round();
4022        self.upgrading = true;
4023        batch.status = true;
4024        // RFC 0005 D6: classify the pending outbox against the NEW compiled
4025        // schema BEFORE the wipe, and record the advisory audit. This pass
4026        // drops nothing, so the audit sees the original pending intent; the
4027        // §7.4.4 send-time drop and the intrinsic value recovery later in this
4028        // reset are the paths that remove a commit. The classification is
4029        // shared with the send-time drop so the two cannot disagree about what
4030        // is incompatible.
4031        if self.previous_version.is_some_and(|config| config.enabled) {
4032            let previous_version = self
4033                .get_meta(LOCAL_SCHEMA_VERSION_KEY)?
4034                .and_then(|value| value.parse::<i32>().ok())
4035                .unwrap_or(self.schema.version);
4036            let pending = self
4037                .outbox
4038                .iter()
4039                .map(|commit| PendingCommitAudit {
4040                    commit_id: &commit.client_commit_id,
4041                    operations: commit_audit_operations(commit),
4042                })
4043                .collect::<Vec<_>>();
4044            crate::previous_version::write_previous_version_audit(
4045                &self.conn,
4046                &crate::previous_version::build_previous_version_audit(
4047                    &self.schema,
4048                    &pending,
4049                    previous_version,
4050                    self.schema.version,
4051                    self.clock_now_ms(),
4052                ),
4053            )?;
4054        }
4055        // RFC 0005 D5: the container lives in its own FILE, so the reset can
4056        // never reach it. Sweep first (unconditional — also on the log-epoch
4057        // reset, which captures nothing), then capture BEFORE the wipe drops
4058        // any row. A refused capture records its reason and leaves no file.
4059        if let Some(replica_path) = self.previous_version_replica_path() {
4060            sweep_previous_version_container(&self.conn, &replica_path)?;
4061            if capture {
4062                if let Some(config) = self.previous_version.filter(|config| config.enabled) {
4063                    let previous_version = self
4064                        .get_meta(LOCAL_SCHEMA_VERSION_KEY)?
4065                        .and_then(|value| value.parse::<i32>().ok())
4066                        .unwrap_or(self.schema.version);
4067                    capture_previous_version_from_replica(
4068                        &self.conn,
4069                        &replica_path,
4070                        previous_version,
4071                        self.schema.version,
4072                        &config,
4073                        self.clock_now_ms(),
4074                    )?;
4075                }
4076            }
4077        }
4078        for table in self
4079            .schema
4080            .tables
4081            .iter()
4082            .filter(|table| !kept.contains(&table.name))
4083        {
4084            batch.table(&table.name);
4085        }
4086        for (base_key, value) in self.load_window_registry()? {
4087            batch.window(&base_key, &value);
4088        }
4089        if capture {
4090            self.prune_unknown_units(true)?;
4091        }
4092        // The per-table insert SQL is derived from the OLD column lists.
4093        self.insert_sql.borrow_mut().clear();
4094        self.overlay_dirty.set(true);
4095        // Drop every synced table (base + visible) that currently exists —
4096        // discovered from sqlite_master so a bump that adds/removes tables is
4097        // handled. Bookkeeping tables (`_syncular_outbox/_units/_meta`
4098        // and the blob cache) are preserved; base tables are `_syncular_base_*`
4099        // so they are matched explicitly, not by the bookkeeping filter.
4100        // Drop virtual tables first so SQLite removes every FTS shadow table
4101        // atomically instead of the generic discovery tearing it apart.
4102        let virtual_tables: Vec<String> = {
4103            let mut stmt = self
4104                .conn
4105                .prepare(
4106                    "SELECT name FROM sqlite_master WHERE type = 'table' AND sql LIKE 'CREATE VIRTUAL TABLE%'",
4107                )
4108                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4109            let rows = stmt
4110                .query_map([], |row| row.get::<_, String>(0))
4111                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4112            rows.filter_map(Result::ok)
4113                .filter(|name| is_synced_table_name(name) && !kept_objects.contains(name))
4114                .collect()
4115        };
4116        for name in virtual_tables {
4117            self.conn
4118                .execute_batch(&format!(
4119                    "DROP TABLE IF EXISTS {}",
4120                    quote_ident(&format!("_syncular_fts_{name}"))
4121                ))
4122                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4123            self.conn
4124                .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
4125                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4126        }
4127        let existing: Vec<String> = {
4128            let mut stmt = self
4129                .conn
4130                .prepare("SELECT name FROM sqlite_master WHERE type = 'table'")
4131                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4132            let rows = stmt
4133                .query_map([], |row| row.get::<_, String>(0))
4134                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4135            rows.filter_map(Result::ok)
4136                .filter(|name| is_synced_table_name(name) && !kept_objects.contains(name))
4137                .collect()
4138        };
4139        for name in &existing {
4140            if let Some(table) = name.strip_prefix("_syncular_base_") {
4141                batch.table(table);
4142            } else if !name.starts_with("_syncular_") {
4143                batch.table(name);
4144            }
4145        }
4146        for name in existing {
4147            self.conn
4148                .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
4149                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4150        }
4151        // Recreate the synced tables from the NEW schema.
4152        self.create_synced_tables()?;
4153        // Reset the sync state of every unit that covers a table the reset
4154        // did not keep, under the previous declarations or the new schema,
4155        // keeping the registration.
4156        let keep_units = match &previous_declarations {
4157            Some(previous) if !kept.is_empty() => self
4158                .stored(load_units(&self.conn))?
4159                .into_iter()
4160                .filter(|unit| {
4161                    previous.iter().all(|(name, old)| {
4162                        kept.contains(name) || !unit.scopes.keys().all(|v| old.contains_key(v))
4163                    }) && self
4164                        .schema
4165                        .tables
4166                        .iter()
4167                        .all(|table| kept.contains(&table.name) || !covers(table, &unit.scopes))
4168                })
4169                .map(|unit| unit.id)
4170                .collect(),
4171            _ => BTreeSet::new(),
4172        };
4173        reset_units(&self.conn, &keep_units)
4174            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4175        // The stop state is over: this client now ships a servable schema.
4176        self.stopped = false;
4177        self.schema_floor = None;
4178        self.delete_meta(SCHEMA_FLOOR_KEY)?;
4179        // Rewrite the marker LAST so a crash mid-reset re-runs the reset.
4180        self.persist_schema_version()?;
4181        self.save_unit_scope_schema()?;
4182        // §7.1/§7.4.4: the previous-version capture and audit above saw the
4183        // original pending intent; the synced tables now carry the new schema, so
4184        // recover intrinsically codec-invalid persisted values here, before the
4185        // final replay. A pending upsert that names a column the new schema
4186        // removed stays pending for the send-time `drop_incompatible_outbox`
4187        // prepass. The reset wrapper snapshots and restores the outbox,
4188        // rejections, and change queues, so a failed commit rolls this recovery
4189        // back with the rest of the reset.
4190        // §7.4.4: the enclosing reset observation owns the publication, so the
4191        // recovery contributes its rejection, outcome, and table domains to it.
4192        self.drop_unencodable_outbox(false, Some(batch))?;
4193        // Re-apply the surviving outbox optimistically over the empty tables.
4194        self.rebuild_overlay()?;
4195        Ok(())
4196    }
4197
4198    /// §7.4.4 send-time prepass: a persisted upsert whose values reference a
4199    /// column the current schema lacks (or a removed table) cannot be encoded
4200    /// (`commit_audit_operations` inspects upserts, so a value-free delete stays
4201    /// encodable and is validated by the server), nor can an insert-intent
4202    /// upsert that lacks a non-nullable column (§7.1, the audit's
4203    /// `missing-required-column`). Drop the commit, undo
4204    /// its optimistic projection, and raise a client-local
4205    /// `sync.outbox_incompatible` rejection.
4206    fn drop_incompatible_outbox(&mut self) -> Result<bool, String> {
4207        let incompatible = self
4208            .outbox
4209            .iter()
4210            .filter(|commit| {
4211                first_incompatibility(&self.schema, &commit_audit_operations(commit)).is_some()
4212            })
4213            .cloned()
4214            .collect::<Vec<_>>();
4215        if incompatible.is_empty() {
4216            return Ok(false);
4217        }
4218        self.drop_outbox_commits(
4219            &incompatible,
4220            OUTBOX_INCOMPATIBLE_CODE,
4221            "the persisted commit cannot encode under the current schema",
4222        )?;
4223        Ok(true)
4224    }
4225
4226    /// §7.4.4 incompatible-commit path, also taken by a refused blob upload
4227    /// (§5.9.7) with the server's code: reject every operation of `commits`,
4228    /// remove them from the outbox, and undo their optimistic rows.
4229    fn drop_outbox_commits(
4230        &mut self,
4231        incompatible: &[Arc<OutboxCommit>],
4232        code: &str,
4233        message: &str,
4234    ) -> Result<(), String> {
4235        // §7.4.4: one observation covers the durable outcome writes, the outbox
4236        // removal, the in-memory drop, and the overlay rebuild, so a storage
4237        // fault leaves the replica and the client unchanged and publishes no
4238        // change; a successful drop publishes exactly one batch carrying the
4239        // rejections, outcomes, status, and the tables whose projection changed.
4240        let rollback = self.capture();
4241        let owns_transaction = self.conn.is_autocommit();
4242        self.begin_observation("syncular_outbox_drop")?;
4243        let mut batch = ChangeAccumulator::default();
4244        let dropped = (|| -> Result<(), String> {
4245            let mut rejections = Vec::new();
4246            for commit in incompatible {
4247                let results = commit
4248                    .ops
4249                    .iter()
4250                    .enumerate()
4251                    .map(|(op_index, operation)| {
4252                        let rejection = RejectionRecord {
4253                            client_commit_id: commit.client_commit_id.clone(),
4254                            op_index: op_index as i32,
4255                            code: code.to_owned(),
4256                            message: message.to_owned(),
4257                            retryable: false,
4258                            details: None,
4259                            operation: Some(CommitOperation::from(operation)),
4260                        };
4261                        rejections.push(rejection.clone());
4262                        CommitOperationOutcome::Error { rejection }
4263                    })
4264                    .collect::<Vec<_>>();
4265                self.persist_commit_outcome(
4266                    &commit.client_commit_id,
4267                    CommitOutcomeStatus::Rejected,
4268                    &results,
4269                    Some(&commit.ops),
4270                )?;
4271                self.delete_outbox_persisted(&commit.client_commit_id)?;
4272                // A removed table has no local projection to invalidate; every
4273                // surviving table the dropped commit touched does.
4274                for operation in &commit.ops {
4275                    if self.schema.table(&operation.table).is_some() {
4276                        batch.table(&operation.table);
4277                    }
4278                }
4279            }
4280            self.prune_commit_outcomes()?;
4281            let incompatible_ids = incompatible
4282                .iter()
4283                .map(|commit| commit.client_commit_id.as_str())
4284                .collect::<BTreeSet<_>>();
4285            self.outbox
4286                .retain(|commit| !incompatible_ids.contains(commit.client_commit_id.as_str()));
4287            self.rejections.extend(rejections);
4288            // The dropped commits' purely-optimistic rows are undone: the
4289            // visible projection is re-derived from the surviving base + outbox.
4290            self.overlay_dirty.set(true);
4291            self.rebuild_overlay()?;
4292            batch.status = true;
4293            batch.rejections = true;
4294            batch.outcomes = true;
4295            Ok(())
4296        })();
4297        let result = dropped.and_then(|()| self.finish_observation("syncular_outbox_drop", batch));
4298        if let Err(error) = result {
4299            // An outer RELEASE is the COMMIT. Roll back that transaction
4300            // directly: retrying RELEASE after ROLLBACK TO still needs the
4301            // reader's lock. A nested observation only rolls back its savepoint.
4302            if owns_transaction {
4303                if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
4304                    let mut failure = self.storage_failure.borrow_mut();
4305                    let failure =
4306                        failure.get_or_insert_with(|| QueryReadFailure::from(error.clone()));
4307                    failure.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
4308                }
4309            } else {
4310                self.rollback_observation("syncular_outbox_drop");
4311            }
4312            self.restore(rollback);
4313            return Err(error);
4314        }
4315        Ok(())
4316    }
4317
4318    /// §7.4.3: (re)create the base + visible table pair for every synced
4319    /// table in the CURRENT schema (idempotent — `IF NOT EXISTS`).
4320    fn create_synced_tables(&self) -> Result<(), String> {
4321        for table in &self.schema.tables {
4322            // The base half + the visible half form the synced-table pair. An
4323            // index name is global in SQLite, so the base half's indexes are
4324            // name-prefixed (`_syncular_base_<index>`) to stay distinct.
4325            for (full, index_prefix) in [
4326                (base_table(&table.name), "_syncular_base_"),
4327                (visible_table(&table.name), ""),
4328            ] {
4329                let mut cols: Vec<String> =
4330                    table.columns.iter().map(|c| quote_ident(&c.name)).collect();
4331                cols.push("\"_syncular_version\" INTEGER NOT NULL".to_owned());
4332                cols.extend(
4333                    table
4334                        .sealed_columns()
4335                        .iter()
4336                        .map(|name| format!("{} TEXT", quote_ident(name))),
4337                );
4338                let sql = format!(
4339                    "CREATE TABLE IF NOT EXISTS {full} ({} , PRIMARY KEY ({}))",
4340                    cols.join(", "),
4341                    quote_ident(&table.primary_key)
4342                );
4343                self.conn
4344                    .execute(&sql, [])
4345                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4346                // §5.11: a replica created before sealed apply gains the
4347                // reserved columns in place, on both halves in the same order,
4348                // so data, indexes and the FTS projection stay.
4349                for name in table.sealed_columns() {
4350                    let present: bool = self
4351                        .conn
4352                        .query_row(
4353                            "SELECT EXISTS(SELECT 1 FROM pragma_table_info(?1) WHERE name = ?2)",
4354                            rusqlite::params![format!("{index_prefix}{}", table.name), name],
4355                            |row| row.get(0),
4356                        )
4357                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4358                    if !present {
4359                        self.conn
4360                            .execute(
4361                                &format!(
4362                                    "ALTER TABLE {full} ADD COLUMN {} TEXT",
4363                                    quote_ident(name)
4364                                ),
4365                                [],
4366                            )
4367                            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4368                    }
4369                }
4370                // The reveal at every round start reads the sealed rows of the
4371                // base half through this partial index.
4372                if table.has_encrypted_columns() && !index_prefix.is_empty() {
4373                    self.conn
4374                        .execute(
4375                            &format!(
4376                                "CREATE INDEX IF NOT EXISTS {} ON {full} (\"_sync_sealed_key\") WHERE \"_sync_sealed_key\" IS NOT NULL",
4377                                quote_ident(&format!("_syncular_sealed_{}", table.name))
4378                            ),
4379                            [],
4380                        )
4381                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4382                }
4383                // Local secondary indexes (CREATE INDEX subset). Created on
4384                // both halves so mirror reads hit an index on either. Runs on
4385                // both the initial create and the §7.4.3 reset recreate path.
4386                for index in &table.indexes {
4387                    let unique = if index.unique { "UNIQUE " } else { "" };
4388                    let index_name = quote_ident(&format!("{index_prefix}{}", index.name));
4389                    let cols_sql = index
4390                        .columns
4391                        .iter()
4392                        .map(|c| quote_ident(c))
4393                        .collect::<Vec<_>>()
4394                        .join(", ");
4395                    let index_sql = format!(
4396                        "CREATE {unique}INDEX IF NOT EXISTS {index_name} ON {full} ({cols_sql})"
4397                    );
4398                    self.conn
4399                        .execute(&index_sql, [])
4400                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4401                }
4402            }
4403        }
4404        // FTS exists only on the visible half. It is a local search
4405        // projection, never a synced/base table or a wire-schema member.
4406        for table in &self.schema.tables {
4407            for index in &table.fts_indexes {
4408                self.create_fts_projection(table, index)?;
4409            }
4410        }
4411        Ok(())
4412    }
4413
4414    fn fts_projection_exists(&self, index: &FtsIndexSchema) -> Result<bool, String> {
4415        let count: i64 = self
4416            .conn
4417            .query_row(
4418                "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?1",
4419                rusqlite::params![index.name],
4420                |row| row.get(0),
4421            )
4422            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4423        Ok(count > 0)
4424    }
4425
4426    fn drop_fts_triggers(&self, index: &FtsIndexSchema) -> Result<(), String> {
4427        for suffix in ["ai", "ad", "au"] {
4428            self.conn
4429                .execute(
4430                    &format!(
4431                        "DROP TRIGGER IF EXISTS {}",
4432                        quote_ident(&format!("{}_{suffix}", index.name))
4433                    ),
4434                    [],
4435                )
4436                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4437        }
4438        Ok(())
4439    }
4440
4441    fn create_fts_triggers(
4442        &self,
4443        table: &TableSchema,
4444        index: &FtsIndexSchema,
4445    ) -> Result<(), String> {
4446        let fts = quote_ident(&index.name);
4447        let source = visible_table(&table.name);
4448        let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
4449        let pk = quote_ident(&table.primary_key);
4450        let projection_columns = std::iter::once(source_id.clone())
4451            .chain(index.columns.iter().map(|column| quote_ident(column)))
4452            .collect::<Vec<_>>()
4453            .join(", ");
4454        let new_values = std::iter::once(format!("CAST(new.{pk} AS TEXT)"))
4455            .chain(
4456                index
4457                    .columns
4458                    .iter()
4459                    .map(|column| format!("new.{}", quote_ident(column))),
4460            )
4461            .collect::<Vec<_>>()
4462            .join(", ");
4463        let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
4464        let delete_new = format!("DELETE FROM {fts} WHERE rowid = (SELECT id FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)); DELETE FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)");
4465        let delete_old = format!("DELETE FROM {fts} WHERE rowid = (SELECT id FROM {mapping} WHERE source_id = CAST(old.{pk} AS TEXT)); DELETE FROM {mapping} WHERE source_id = CAST(old.{pk} AS TEXT)");
4466        let insert_new = format!("INSERT INTO {mapping}(source_id) VALUES (CAST(new.{pk} AS TEXT)); INSERT INTO {fts} (rowid, {projection_columns}) VALUES ((SELECT id FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)), {new_values})");
4467        let bi = quote_ident(&format!("{}_bi", index.name));
4468        let ai = quote_ident(&format!("{}_ai", index.name));
4469        let ad = quote_ident(&format!("{}_ad", index.name));
4470        let au = quote_ident(&format!("{}_au", index.name));
4471        let replacement_exists = format!("EXISTS (SELECT 1 FROM {source} WHERE {pk} = new.{pk})");
4472        let sql = format!(
4473            "DROP TRIGGER IF EXISTS {bi};
4474             DROP TRIGGER IF EXISTS {ai};
4475             DROP TRIGGER IF EXISTS {ad};
4476             DROP TRIGGER IF EXISTS {au};
4477             CREATE TRIGGER {bi} BEFORE INSERT ON {source} WHEN {replacement_exists} BEGIN {delete_new}; END;
4478             CREATE TRIGGER {ai} AFTER INSERT ON {source} BEGIN {insert_new}; END;
4479             CREATE TRIGGER {ad} AFTER DELETE ON {source} BEGIN {delete_old}; END;
4480             CREATE TRIGGER {au} AFTER UPDATE ON {source} BEGIN {delete_old}; {delete_new}; {insert_new}; END;",
4481        );
4482        self.conn
4483            .execute_batch(&sql)
4484            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
4485    }
4486
4487    fn rebuild_fts_projection(
4488        &self,
4489        table: &TableSchema,
4490        index: &FtsIndexSchema,
4491    ) -> Result<(), String> {
4492        let fts = quote_ident(&index.name);
4493        let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
4494        let pk = quote_ident(&table.primary_key);
4495        let indexed_columns = index
4496            .columns
4497            .iter()
4498            .map(|column| quote_ident(column))
4499            .collect::<Vec<_>>();
4500        let projection_columns = std::iter::once(source_id.clone())
4501            .chain(indexed_columns.iter().cloned())
4502            .collect::<Vec<_>>()
4503            .join(", ");
4504        let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
4505        let sql = format!(
4506            "DELETE FROM {fts}; DELETE FROM {mapping}; INSERT INTO {fts} ({projection_columns}) SELECT CAST({pk} AS TEXT), {columns} FROM {source}; INSERT INTO {mapping}(id, source_id) SELECT rowid, {source_id} FROM {fts};",
4507            columns = indexed_columns.join(", "),
4508            source = visible_table(&table.name),
4509        );
4510        self.conn
4511            .execute_batch(&sql)
4512            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
4513    }
4514
4515    fn create_fts_projection(
4516        &self,
4517        table: &TableSchema,
4518        index: &FtsIndexSchema,
4519    ) -> Result<(), String> {
4520        self.begin_observation("syncular_fts_schema")?;
4521        let result = (|| {
4522            let existed = self.fts_projection_exists(index)?;
4523            let tokenizer = index.tokenize.replace('\'', "''");
4524            let columns = index
4525                .columns
4526                .iter()
4527                .map(|column| quote_ident(column))
4528                .collect::<Vec<_>>()
4529                .join(", ");
4530            let sql = format!(
4531            "CREATE VIRTUAL TABLE IF NOT EXISTS {fts} USING fts5({source_id} UNINDEXED, {columns}, tokenize='{tokenizer}')",
4532            fts = quote_ident(&index.name),
4533            source_id = quote_ident(FTS_SOURCE_ID_COLUMN),
4534        );
4535            self.conn.execute(&sql, []).map_err(|error| {
4536                format!(
4537                    "cannot create local FTS5 projection {:?}: {error}",
4538                    index.name
4539                )
4540            })?;
4541            let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
4542            let mapped: bool = self
4543                .conn
4544                .query_row(
4545                    "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?1)",
4546                    [format!("_syncular_fts_{}", index.name)],
4547                    |row| row.get(0),
4548                )
4549                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4550            self.conn.execute_batch(&format!("CREATE TABLE IF NOT EXISTS {mapping}(id INTEGER PRIMARY KEY, source_id TEXT NOT NULL UNIQUE)")).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4551            if !mapped && existed {
4552                self.conn
4553                    .execute_batch(&format!(
4554                        "INSERT INTO {mapping}(id, source_id) SELECT rowid, {} FROM {}",
4555                        quote_ident(FTS_SOURCE_ID_COLUMN),
4556                        quote_ident(&index.name)
4557                    ))
4558                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4559            }
4560            self.create_fts_triggers(table, index)?;
4561            // Every visible row has exactly one mapping row, so a count
4562            // mismatch marks a projection an earlier release damaged; the
4563            // open rebuilds it.
4564            let stale: bool = !existed
4565                || self
4566                    .conn
4567                    .query_row(
4568                        &format!(
4569                            "SELECT (SELECT count(*) FROM {mapping}) != (SELECT count(*) FROM {})",
4570                            visible_table(&table.name)
4571                        ),
4572                        [],
4573                        |row| row.get(0),
4574                    )
4575                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4576            if stale {
4577                self.rebuild_fts_projection(table, index)?;
4578            }
4579            Ok(())
4580        })();
4581        if let Err(error) = result {
4582            self.rollback_observation("syncular_fts_schema");
4583            return Err(error);
4584        }
4585        if let Err(error) = self.conn.execute_batch("RELEASE syncular_fts_schema") {
4586            self.rollback_observation("syncular_fts_schema");
4587            return Err(error.to_string());
4588        }
4589        Ok(())
4590    }
4591
4592    // -- persistence write-through --------------------------------------------
4593
4594    fn persist_outbox_insert(&self, commit: &OutboxCommit) -> Result<(), String> {
4595        let ops: Vec<Value> = commit
4596            .ops
4597            .iter()
4598            .map(|op| {
4599                let mut entry = serde_json::json!({
4600                    "op": if op.upsert { "upsert" } else { "delete" },
4601                    "table": op.table,
4602                    "rowId": op.row_id,
4603                    "baseVersion": op.base_version,
4604                    "values": op.values.clone().map(Value::Object),
4605                });
4606                if op.insert {
4607                    entry["insert"] = Value::Bool(true);
4608                }
4609                entry
4610            })
4611            .collect();
4612        self.conn
4613            .execute(
4614                "INSERT OR REPLACE INTO _syncular_outbox (commit_id, ops_json) VALUES (?1, ?2)",
4615                rusqlite::params![commit.client_commit_id, Value::Array(ops).to_string()],
4616            )
4617            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4618        for operation in &commit.ops {
4619            let Some(values) = &operation.values else {
4620                continue;
4621            };
4622            let Some(table) = self.schema.table(&operation.table) else {
4623                continue;
4624            };
4625            for column in table
4626                .columns
4627                .iter()
4628                .filter(|column| column.ty == ColumnType::BlobRef)
4629            {
4630                let Some(Value::String(raw)) = values.get(&column.name) else {
4631                    continue;
4632                };
4633                let Ok(blob_ref) = serde_json::from_str::<Value>(raw) else {
4634                    continue;
4635                };
4636                let Some(blob_id) = blob_ref.get("blobId").and_then(Value::as_str) else {
4637                    continue;
4638                };
4639                self.conn
4640                    .execute(
4641                        "INSERT OR IGNORE INTO _syncular_blob_commit_refs(commit_id, blob_id)
4642                         SELECT ?1, ?2 WHERE EXISTS (SELECT 1 FROM _syncular_blobs WHERE blob_id = ?2)",
4643                        rusqlite::params![commit.client_commit_id, blob_id],
4644                    )
4645                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4646            }
4647        }
4648        Ok(())
4649    }
4650
4651    fn delete_outbox_persisted(&self, client_commit_id: &str) -> Result<(), String> {
4652        self.conn
4653            .execute(
4654                "DELETE FROM _syncular_blob_commit_refs WHERE commit_id = ?1",
4655                rusqlite::params![client_commit_id],
4656            )
4657            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4658        self.conn
4659            .execute(
4660                "DELETE FROM _syncular_outbox WHERE commit_id = ?1",
4661                rusqlite::params![client_commit_id],
4662            )
4663            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4664        self.conn.execute("DELETE FROM _syncular_row_deliveries WHERE NOT EXISTS(SELECT 1 FROM _syncular_outbox,json_each(ops_json) WHERE json_extract(json_each.value,'$.table')=tbl AND json_extract(json_each.value,'$.rowId')=id)", [])
4665            .map(|_| ())
4666            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
4667    }
4668
4669    fn outcome_status_name(status: CommitOutcomeStatus) -> &'static str {
4670        match status {
4671            CommitOutcomeStatus::Applied => "applied",
4672            CommitOutcomeStatus::Cached => "cached",
4673            CommitOutcomeStatus::Conflict => "conflict",
4674            CommitOutcomeStatus::Rejected => "rejected",
4675        }
4676    }
4677
4678    fn outcome_resolution_name(resolution: CommitOutcomeResolution) -> &'static str {
4679        match resolution {
4680            CommitOutcomeResolution::Active => "active",
4681            CommitOutcomeResolution::ResolvedKeepServer => "resolved_keep_server",
4682            CommitOutcomeResolution::Superseded => "superseded",
4683            CommitOutcomeResolution::Dismissed => "dismissed",
4684        }
4685    }
4686
4687    fn parse_outcome_status(value: &str) -> Result<CommitOutcomeStatus, String> {
4688        match value {
4689            "applied" => Ok(CommitOutcomeStatus::Applied),
4690            "cached" => Ok(CommitOutcomeStatus::Cached),
4691            "conflict" => Ok(CommitOutcomeStatus::Conflict),
4692            "rejected" => Ok(CommitOutcomeStatus::Rejected),
4693            _ => Err(format!("invalid persisted commit outcome status {value:?}")),
4694        }
4695    }
4696
4697    fn parse_outcome_resolution(value: &str) -> Result<CommitOutcomeResolution, String> {
4698        match value {
4699            "active" => Ok(CommitOutcomeResolution::Active),
4700            "resolved_keep_server" => Ok(CommitOutcomeResolution::ResolvedKeepServer),
4701            "superseded" => Ok(CommitOutcomeResolution::Superseded),
4702            "dismissed" => Ok(CommitOutcomeResolution::Dismissed),
4703            _ => Err(format!(
4704                "invalid persisted commit outcome resolution {value:?}"
4705            )),
4706        }
4707    }
4708
4709    fn persist_commit_outcome(
4710        &self,
4711        client_commit_id: &str,
4712        status: CommitOutcomeStatus,
4713        results: &[CommitOperationOutcome],
4714        operations: Option<&[OutboxOp]>,
4715    ) -> Result<(), String> {
4716        let results_json = serde_json::to_string(results).map_err(|error| error.to_string())?;
4717        let operations_json = operations
4718            .map(|items| {
4719                serde_json::to_string(&items.iter().map(CommitOperation::from).collect::<Vec<_>>())
4720            })
4721            .transpose()
4722            .map_err(|error| error.to_string())?;
4723        self.conn
4724            .execute(
4725                "INSERT INTO _syncular_commit_outcomes (
4726                   client_commit_id, status, recorded_at_ms, results_json,
4727                   operations_json, resolution
4728                 ) VALUES (?1, ?2, ?3, ?4, ?5, 'active')",
4729                rusqlite::params![
4730                    client_commit_id,
4731                    Self::outcome_status_name(status),
4732                    self.clock_now_ms(),
4733                    results_json,
4734                    operations_json
4735                ],
4736            )
4737            .map(|_| ())
4738            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
4739    }
4740
4741    /// §7.2: the journal holds a stored cell only in its canonical shape. A cell
4742    /// the byte-envelope rule refuses cannot be journaled verbatim.
4743    fn stored_cell_is_representable(cell: &Value) -> bool {
4744        cell.is_null()
4745            || cell.is_string()
4746            || cell.is_boolean()
4747            || cell.is_number()
4748            || is_canonical_bytes_envelope(cell)
4749    }
4750
4751    fn stored_row_is_representable(value: &Value) -> bool {
4752        value
4753            .as_object()
4754            .is_some_and(|row| row.values().all(Self::stored_cell_is_representable))
4755    }
4756
4757    fn stored_operation_is_representable(value: &Value) -> bool {
4758        value.get("table").is_some_and(Value::is_string)
4759            && value.get("rowId").is_some_and(Value::is_string)
4760            && value.get("baseVersion").is_none_or(|v| {
4761                v.as_i64()
4762                    .is_some_and(|n| n.unsigned_abs() <= MAX_JS_SAFE_INTEGER)
4763            })
4764            && match value.get("op").and_then(Value::as_str) {
4765                Some("upsert") => {
4766                    value
4767                        .get("values")
4768                        .is_some_and(Self::stored_row_is_representable)
4769                        && value
4770                            .get("insert")
4771                            .is_none_or(|insert| *insert == Value::Bool(true))
4772                }
4773                Some("delete") => value.get("values").is_none() && value.get("insert").is_none(),
4774                _ => false,
4775            }
4776    }
4777
4778    /// §7.2/§7.4.4: a dropped commit keeps its operation envelope in the journal
4779    /// only when the journal reader accepts the serialized operation, which is the
4780    /// rule `outcome_from_row` applies to this same row later.
4781    fn outbox_op_is_journalable(operation: &OutboxOp) -> bool {
4782        serde_json::to_value(CommitOperation::from(operation))
4783            .is_ok_and(|value| Self::stored_operation_is_representable(&value))
4784    }
4785
4786    fn outcome_from_row(row: StoredCommitOutcomeRow) -> Result<CommitOutcome, String> {
4787        let StoredCommitOutcomeRow {
4788            sequence,
4789            client_commit_id,
4790            status,
4791            recorded_at_ms,
4792            results_json,
4793            operations_json,
4794            resolution,
4795            resolved_at_ms,
4796            replacement_client_commit_id,
4797        } = row;
4798        if sequence.unsigned_abs() > MAX_JS_SAFE_INTEGER
4799            || recorded_at_ms.unsigned_abs() > MAX_JS_SAFE_INTEGER
4800            || resolved_at_ms.is_some_and(|value| value.unsigned_abs() > MAX_JS_SAFE_INTEGER)
4801        {
4802            return Err("sync.local_corrupt: persisted commit outcome is invalid".to_owned());
4803        }
4804        let invalid = || "sync.local_corrupt: persisted commit outcome is invalid".to_owned();
4805        let stored: Vec<Value> = serde_json::from_str(&results_json).map_err(|_| invalid())?;
4806        for result in &stored {
4807            let nested = match result.get("status").and_then(Value::as_str) {
4808                Some("conflict") => result.get("conflict"),
4809                Some("error") => result.get("rejection"),
4810                _ => None,
4811            };
4812            if nested
4813                .and_then(|record| record.get("operation"))
4814                .is_some_and(|op| !Self::stored_operation_is_representable(op))
4815                || (result.get("status").and_then(Value::as_str) == Some("conflict")
4816                    && nested
4817                        .and_then(|record| record.get("serverRow"))
4818                        .is_some_and(|row| !Self::stored_row_is_representable(row)))
4819            {
4820                return Err(invalid());
4821            }
4822        }
4823        let results: Vec<CommitOperationOutcome> =
4824            serde_json::from_value(Value::Array(stored)).map_err(|_| invalid())?;
4825        let has_conflict = results
4826            .iter()
4827            .any(|result| matches!(result, CommitOperationOutcome::Conflict { .. }));
4828        let has_error = results
4829            .iter()
4830            .any(|result| matches!(result, CommitOperationOutcome::Error { .. }));
4831        let status = Self::parse_outcome_status(&status).map_err(|_| invalid())?;
4832        let redacted = results.is_empty() && operations_json.is_none();
4833        if match status {
4834            CommitOutcomeStatus::Applied | CommitOutcomeStatus::Cached => has_conflict || has_error,
4835            CommitOutcomeStatus::Conflict => !(has_conflict || redacted),
4836            CommitOutcomeStatus::Rejected => has_conflict || !(has_error || redacted),
4837        } {
4838            return Err(invalid());
4839        }
4840        let operations: Option<Vec<CommitOperation>> = operations_json
4841            .map(|raw| {
4842                let stored: Vec<Value> = serde_json::from_str(&raw).map_err(|_| invalid())?;
4843                if !stored.iter().all(Self::stored_operation_is_representable) {
4844                    return Err(invalid());
4845                }
4846                serde_json::from_value(Value::Array(stored)).map_err(|_| invalid())
4847            })
4848            .transpose()?;
4849        for result in &results {
4850            match result {
4851                CommitOperationOutcome::Conflict { conflict }
4852                    if conflict.server_version.unsigned_abs() > MAX_JS_SAFE_INTEGER =>
4853                {
4854                    return Err("sync.local_corrupt: persisted commit outcome is invalid".to_owned())
4855                }
4856                CommitOperationOutcome::Error { rejection } => {
4857                    if let Some(details) = &rejection.details {
4858                        let encoded = serde_json::to_string(details).map_err(|_| {
4859                            "sync.local_corrupt: persisted commit outcome is invalid".to_owned()
4860                        })?;
4861                        RejectionDetails::parse(&encoded).map_err(|_| {
4862                            "sync.local_corrupt: persisted commit outcome is invalid".to_owned()
4863                        })?;
4864                    }
4865                }
4866                _ => {}
4867            }
4868        }
4869        Ok(CommitOutcome {
4870            sequence,
4871            client_commit_id,
4872            status,
4873            recorded_at_ms,
4874            results,
4875            operations,
4876            retained_rows: None,
4877            resolution: Self::parse_outcome_resolution(&resolution)?,
4878            resolved_at_ms,
4879            replacement_client_commit_id,
4880        })
4881    }
4882
4883    /// Resolve declared unique keys through SQLite, including affinity and NULL semantics.
4884    fn unique_conflicts(
4885        &self,
4886        table: &TableSchema,
4887        values: &Map<String, Value>,
4888        base: bool,
4889    ) -> Result<Vec<RetainedUniqueConflict>, String> {
4890        let source = if base {
4891            base_table(&table.name)
4892        } else {
4893            visible_table(&table.name)
4894        };
4895        let mut conflicts = Vec::new();
4896        for index in table.indexes.iter().filter(|index| index.unique) {
4897            let mut params = vec![json_param_to_sql(
4898                values.get(&table.primary_key).ok_or("sync.local_corrupt")?,
4899            )?];
4900            for name in &index.columns {
4901                let column = table
4902                    .columns
4903                    .iter()
4904                    .find(|column| column.name == *name)
4905                    .ok_or("sync.local_corrupt")?;
4906                let cell = json_to_column_value(column, values.get(name))?;
4907                params.push(owned_sql_value(RowParam::Cell(&cell))?);
4908            }
4909            let sql = format!(
4910                "SELECT * FROM {source} WHERE {} != ? AND {}",
4911                quote_ident(&table.primary_key),
4912                index
4913                    .columns
4914                    .iter()
4915                    .map(|column| format!("{} = ?", quote_ident(column)))
4916                    .collect::<Vec<_>>()
4917                    .join(" AND ")
4918            );
4919            let mut statement = self
4920                .conn
4921                .prepare_cached(&sql)
4922                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4923            let records = statement
4924                .query_map(rusqlite::params_from_iter(params), |record| {
4925                    let values = table
4926                        .columns
4927                        .iter()
4928                        .enumerate()
4929                        .map(|(index, column)| {
4930                            let value = sql_ref_to_json(column, record.get_ref(index)?);
4931                            json_to_column_value(column, Some(&value)).map_err(|message| {
4932                                rusqlite::Error::ToSqlConversionFailure(message.into())
4933                            })
4934                        })
4935                        .collect::<Result<Row, _>>()?;
4936                    Ok((values, record.get::<_, i64>(table.columns.len())?))
4937                })
4938                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4939            for record in records {
4940                let (row, version) =
4941                    record.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4942                let row_id = render_row_id_json(Some(&column_value_to_json(&row[table.pk_index])))?;
4943                conflicts.push(RetainedUniqueConflict {
4944                    index: index.name.clone(),
4945                    columns: index.columns.clone(),
4946                    row_id,
4947                    server_row: table
4948                        .columns
4949                        .iter()
4950                        .zip(&row)
4951                        .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
4952                        .collect(),
4953                    server_version: version,
4954                });
4955            }
4956        }
4957        Ok(conflicts)
4958    }
4959
4960    fn attach_retained_rows(&self, mut outcome: CommitOutcome) -> Result<CommitOutcome, String> {
4961        if let Some((commit, initial)) = self
4962            .failed_commits
4963            .iter()
4964            .find(|(commit, _)| commit.client_commit_id == outcome.client_commit_id)
4965        {
4966            let mut rows = Vec::new();
4967            for operation in &commit.ops {
4968                // A table the current schema removed has no rows to retain.
4969                let Some(table) = self.schema.table(&operation.table) else {
4970                    continue;
4971                };
4972                let key =
4973                    serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
4974                        .map_err(|_| "sync.local_corrupt".to_owned())?;
4975                let base = self.stored_row(table, &operation.row_id, true)?;
4976                rows.push(RetainedCommitRow {
4977                    table: operation.table.clone(),
4978                    row_id: operation.row_id.clone(),
4979                    unique_conflicts: if operation.upsert {
4980                        self.unique_conflicts(
4981                            table,
4982                            initial
4983                                .get(&key)
4984                                .and_then(Value::as_object)
4985                                .or(operation.values.as_ref())
4986                                .ok_or("sync.local_corrupt")?,
4987                            true,
4988                        )?
4989                    } else {
4990                        Vec::new()
4991                    },
4992                    local_row: if operation.upsert {
4993                        initial
4994                            .get(&key)
4995                            .and_then(Value::as_object)
4996                            .cloned()
4997                            .or_else(|| operation.values.clone())
4998                    } else {
4999                        None
5000                    },
5001                    server_row: base.as_ref().map(|(values, _)| {
5002                        table
5003                            .columns
5004                            .iter()
5005                            .zip(values)
5006                            .map(|(column, value)| {
5007                                (column.name.clone(), column_value_to_json(value))
5008                            })
5009                            .collect()
5010                    }),
5011                    server_version: base.map(|(_, version)| version),
5012                });
5013            }
5014            outcome.retained_rows = Some(rows);
5015        }
5016        Ok(outcome)
5017    }
5018
5019    pub fn commit_outcome(&self, client_commit_id: &str) -> Result<Option<CommitOutcome>, String> {
5020        let row = self
5021            .conn
5022            .query_row(
5023                "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
5024                        resolution, resolved_at_ms, replacement_client_commit_id
5025                   FROM _syncular_commit_outcomes WHERE client_commit_id = ?1",
5026                rusqlite::params![client_commit_id],
5027                |row| {
5028                    Ok(StoredCommitOutcomeRow {
5029                        sequence: row.get(0)?,
5030                        client_commit_id: row.get(1)?,
5031                        status: row.get(2)?,
5032                        recorded_at_ms: row.get(3)?,
5033                        results_json: row.get(4)?,
5034                        operations_json: row.get(5)?,
5035                        resolution: row.get(6)?,
5036                        resolved_at_ms: row.get(7)?,
5037                        replacement_client_commit_id: row.get(8)?,
5038                    })
5039                },
5040            )
5041            .optional()
5042            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5043        row.map(|row| self.attach_retained_rows(Self::outcome_from_row(row)?))
5044            .transpose()
5045    }
5046
5047    /// Internal host step. The host must run its idle drive before finishing.
5048    #[doc(hidden)]
5049    pub fn prepare_current_base(
5050        &self,
5051        client_commit_id: &str,
5052        sequence: i64,
5053    ) -> Result<CurrentBasePreparation, CurrentBaseProof> {
5054        let outcome = self
5055            .commit_outcome(client_commit_id)
5056            .map_err(|_| CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed))?;
5057        let valid = outcome.as_ref().is_some_and(|outcome| {
5058            outcome.sequence == sequence
5059                && outcome.resolution == CommitOutcomeResolution::Active
5060                && matches!(
5061                    outcome.status,
5062                    CommitOutcomeStatus::Conflict | CommitOutcomeStatus::Rejected
5063                )
5064                && outcome
5065                    .operations
5066                    .as_ref()
5067                    .zip(outcome.retained_rows.as_ref())
5068                    .is_some_and(|(operations, rows)| {
5069                        !rows.is_empty() && operations.len() == rows.len()
5070                    })
5071        });
5072        if !valid {
5073            return Err(CurrentBaseProof::failure(
5074                CurrentBaseProofReason::OutcomeUnavailable,
5075            ));
5076        }
5077        if !self.transport_enabled {
5078            return Err(CurrentBaseProof::failure(CurrentBaseProofReason::Offline));
5079        }
5080        Ok(CurrentBasePreparation {
5081            client_commit_id: client_commit_id.to_owned(),
5082            sequence,
5083            units: load_units(&self.conn)
5084                .map_err(|_| CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed))?,
5085            transport_epoch: self.transport_epoch,
5086            registration_epoch: self.registration_epoch,
5087            security_epoch: self.security_epoch,
5088        })
5089    }
5090
5091    /// Internal host step after the completed post-rejection idle drive.
5092    #[doc(hidden)]
5093    pub fn finish_current_base(
5094        &mut self,
5095        prepared: CurrentBasePreparation,
5096        drive: SyncOutcome,
5097    ) -> CurrentBaseProof {
5098        if !self.transport_enabled || self.transport_epoch != prepared.transport_epoch {
5099            return CurrentBaseProof::failure(CurrentBaseProofReason::Offline);
5100        }
5101        if self.registration_epoch != prepared.registration_epoch {
5102            return CurrentBaseProof::failure(CurrentBaseProofReason::MissingCoverage);
5103        }
5104        if self.security_preflight || self.security_epoch != prepared.security_epoch {
5105            return CurrentBaseProof::failure(CurrentBaseProofReason::SyncFailed);
5106        }
5107        let report = match drive {
5108            SyncOutcome::Ok(report) => report,
5109            SyncOutcome::BudgetExhausted(_) => {
5110                return CurrentBaseProof::failure(CurrentBaseProofReason::BudgetExhausted)
5111            }
5112            _ => return CurrentBaseProof::failure(CurrentBaseProofReason::SyncFailed),
5113        };
5114        if report.schema_floor.is_some() {
5115            return CurrentBaseProof::failure(CurrentBaseProofReason::SchemaFloor);
5116        }
5117        if self
5118            .conn
5119            .execute_batch("SAVEPOINT syncular_base_proof")
5120            .is_err()
5121        {
5122            return CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed);
5123        }
5124        let proof = (|| {
5125            let snapshot = snapshot_read_connection(
5126                &self.conn,
5127                &SnapshotReadRequest {
5128                    units: prepared.units.iter().map(|unit| unit.id.clone()).collect(),
5129                    commit_ids: vec![prepared.client_commit_id.clone()],
5130                    ..Default::default()
5131                },
5132            )
5133            .map_err(|_| CurrentBaseProofReason::SnapshotFailed)?;
5134            let current = self
5135                .commit_outcome(&prepared.client_commit_id)
5136                .map_err(|_| CurrentBaseProofReason::SnapshotFailed)?;
5137            let unavailable = if report.revoked.is_empty() {
5138                CurrentBaseProofReason::OutcomeUnavailable
5139            } else {
5140                CurrentBaseProofReason::Revoked
5141            };
5142            let Some(outcome) = current else {
5143                return Err(unavailable);
5144            };
5145            if !matches!(
5146                snapshot.deliveries.first(),
5147                Some(CommitDelivery::Known { .. })
5148            ) || outcome.sequence != prepared.sequence
5149                || outcome.resolution != CommitOutcomeResolution::Active
5150                || !matches!(
5151                    outcome.status,
5152                    CommitOutcomeStatus::Conflict | CommitOutcomeStatus::Rejected
5153                )
5154            {
5155                return Err(unavailable);
5156            }
5157            let (Some(operations), Some(rows)) = (outcome.operations, outcome.retained_rows) else {
5158                return Err(unavailable);
5159            };
5160            if rows.is_empty() || operations.len() != rows.len() {
5161                return Err(unavailable);
5162            }
5163            let after =
5164                load_units(&self.conn).map_err(|_| CurrentBaseProofReason::SnapshotFailed)?;
5165            let complete: HashSet<&str> = snapshot
5166                .units
5167                .iter()
5168                .filter_map(|unit| match unit {
5169                    UnitCatchup::Known {
5170                        id,
5171                        bootstrap_complete: true,
5172                        ..
5173                    } => Some(id.as_str()),
5174                    _ => None,
5175                })
5176                .collect();
5177            // A unit id names one scope map (§4.1), so the same id before and
5178            // after the drive is the same registration.
5179            let candidates: Vec<&UnitRecord> = after
5180                .iter()
5181                .filter(|unit| {
5182                    complete.contains(unit.id.as_str())
5183                        && prepared.units.iter().any(|before| before.id == unit.id)
5184                })
5185                .collect();
5186            let covered = |table_name: &str, row: &Map<String, Value>| {
5187                let Some(table) = self.schema.table(table_name) else {
5188                    return false;
5189                };
5190                candidates.iter().any(|unit| {
5191                    covers(table, &unit.scopes)
5192                        && pair_scopes(&unit.scopes, table_name, unit.echo.as_deref()).is_some_and(
5193                            |scopes| {
5194                                scopes.iter().all(|(variable, allowed)| {
5195                                    table
5196                                        .scope_column(variable)
5197                                        .and_then(|column| match row.get(column) {
5198                                            Some(Value::String(value)) => Some(value.clone()),
5199                                            Some(Value::Bool(value)) => Some(value.to_string()),
5200                                            Some(Value::Number(value)) => {
5201                                                value.as_f64().map(|number| {
5202                                                    ryu_js::Buffer::new()
5203                                                        .format_finite(number)
5204                                                        .to_owned()
5205                                                })
5206                                            }
5207                                            _ => None,
5208                                        })
5209                                        .is_some_and(|value| {
5210                                            !value.is_empty() && allowed.contains(&value)
5211                                        })
5212                                })
5213                            },
5214                        )
5215                })
5216            };
5217            for row in &rows {
5218                let Some(base) = row.server_row.as_ref().or(row.local_row.as_ref()) else {
5219                    return Err(if report.revoked.is_empty() {
5220                        CurrentBaseProofReason::MissingCoverage
5221                    } else {
5222                        CurrentBaseProofReason::Revoked
5223                    });
5224                };
5225                if !covered(&row.table, base)
5226                    || row
5227                        .unique_conflicts
5228                        .iter()
5229                        .any(|conflict| !covered(&row.table, &conflict.server_row))
5230                {
5231                    return Err(if report.revoked.is_empty() {
5232                        CurrentBaseProofReason::MissingCoverage
5233                    } else {
5234                        CurrentBaseProofReason::Revoked
5235                    });
5236                }
5237            }
5238            Ok(rows)
5239        })();
5240        if self
5241            .conn
5242            .execute_batch("RELEASE syncular_base_proof")
5243            .is_err()
5244        {
5245            return CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed);
5246        }
5247        match proof {
5248            Ok(rows) => CurrentBaseProof {
5249                proven: true,
5250                rows: Some(rows),
5251                reason: None,
5252            },
5253            Err(reason) => CurrentBaseProof::failure(reason),
5254        }
5255    }
5256
5257    pub fn prove_current_base(
5258        &mut self,
5259        transport: &mut dyn Transport,
5260        client_commit_id: &str,
5261        sequence: i64,
5262        max_rounds: Option<u32>,
5263    ) -> CurrentBaseProof {
5264        let prepared = match self.prepare_current_base(client_commit_id, sequence) {
5265            Ok(prepared) => prepared,
5266            Err(proof) => return proof,
5267        };
5268        let drive = self.sync_until_idle(transport, max_rounds);
5269        self.finish_current_base(prepared, drive)
5270    }
5271
5272    pub fn commit_outcomes(&self, query: CommitOutcomeQuery) -> Result<Vec<CommitOutcome>, String> {
5273        if query.limit == Some(0) {
5274            return Err("sync.invalid_request: commit outcome limit must be positive".to_owned());
5275        }
5276        let mut sql = String::from(
5277            "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
5278                    resolution, resolved_at_ms, replacement_client_commit_id
5279               FROM _syncular_commit_outcomes",
5280        );
5281        if query.active_only {
5282            sql.push_str(" WHERE resolution = 'active' AND status IN ('conflict', 'rejected')");
5283        }
5284        sql.push_str(" ORDER BY seq DESC");
5285        if let Some(limit) = query.limit {
5286            sql.push_str(&format!(" LIMIT {limit}"));
5287        }
5288        let mut stmt = self
5289            .conn
5290            .prepare(&sql)
5291            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5292        let rows = stmt
5293            .query_map([], |row| {
5294                Ok(StoredCommitOutcomeRow {
5295                    sequence: row.get(0)?,
5296                    client_commit_id: row.get(1)?,
5297                    status: row.get(2)?,
5298                    recorded_at_ms: row.get(3)?,
5299                    results_json: row.get(4)?,
5300                    operations_json: row.get(5)?,
5301                    resolution: row.get(6)?,
5302                    resolved_at_ms: row.get(7)?,
5303                    replacement_client_commit_id: row.get(8)?,
5304                })
5305            })
5306            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5307        let mut outcomes = Vec::new();
5308        for row in rows {
5309            outcomes.push(self.attach_retained_rows(Self::outcome_from_row(
5310                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
5311            )?)?);
5312        }
5313        Ok(outcomes)
5314    }
5315
5316    fn prune_commit_outcomes(&self) -> Result<(), String> {
5317        #[cfg(test)]
5318        self.outcome_prune_count
5319            .set(self.outcome_prune_count.get() + 1);
5320        let max_entries = self.limits.outcome_retention_max_entries.unwrap_or(1_000);
5321        let count = self
5322            .conn
5323            .query_row(
5324                "SELECT COUNT(*) FROM _syncular_commit_outcomes",
5325                [],
5326                |row| row.get::<_, i64>(0),
5327            )
5328            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
5329            as usize;
5330        let excess = count.saturating_sub(max_entries);
5331        if excess == 0 {
5332            return Ok(());
5333        }
5334        let mut stmt = self
5335            .conn
5336            .prepare(
5337                "SELECT seq FROM _syncular_commit_outcomes
5338                  WHERE status IN ('applied', 'cached') OR resolution != 'active'
5339                  ORDER BY seq ASC LIMIT ?1",
5340            )
5341            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5342        let rows = stmt
5343            .query_map(rusqlite::params![excess as i64], |row| row.get::<_, i64>(0))
5344            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5345        let sequences = rows
5346            .collect::<Result<Vec<_>, _>>()
5347            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5348        drop(stmt);
5349        for sequence in sequences {
5350            self.conn
5351                .execute(
5352                    "DELETE FROM _syncular_commit_outcomes WHERE seq = ?1",
5353                    rusqlite::params![sequence],
5354                )
5355                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5356        }
5357        Ok(())
5358    }
5359
5360    // -- units (§4.1) ------------------------------------------------------------
5361
5362    /// Every registered unit in registration order (§4.1).
5363    pub fn units(&self) -> Result<Vec<UnitRecord>, String> {
5364        self.stored(load_units(&self.conn))
5365    }
5366
5367    pub fn unit(&self, id: &str) -> Result<Option<UnitRecord>, String> {
5368        self.stored(get_unit(&self.conn, id))
5369    }
5370
5371    /// Register one unit (§4.1) and return its id.
5372    pub fn subscribe(&mut self, scopes: UnitScopes) -> Result<String, String> {
5373        Ok(self.subscribe_many(vec![scopes])?.remove(0))
5374    }
5375
5376    /// Register several units in one local transaction and return their ids:
5377    /// one commit for all of them (on OPFS a commit costs milliseconds, so a
5378    /// cold start with hundreds of units pays it once). An error rolls back
5379    /// every registration of the call. Registering an active unit again
5380    /// changes nothing; a revoked unit re-arms (§3.3). A call that registers or
5381    /// re-arms a unit raises sync-needed with an interactive intent.
5382    pub fn subscribe_many(&mut self, units: Vec<UnitScopes>) -> Result<Vec<String>, String> {
5383        self.begin_observation("syncular_subscribe")?;
5384        let mut changed = false;
5385        let registered = (|| {
5386            let mut ids = Vec::with_capacity(units.len());
5387            for scopes in units {
5388                let (id, held) = self.hold_unit(scopes, true)?;
5389                changed |= held;
5390                ids.push(id);
5391            }
5392            self.conn
5393                .execute_batch("RELEASE syncular_subscribe")
5394                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5395            Ok(ids)
5396        })();
5397        let ids = match registered {
5398            Ok(ids) => ids,
5399            Err(error) => {
5400                self.rollback_observation("syncular_subscribe");
5401                return Err(error);
5402            }
5403        };
5404        if changed {
5405            self.registration_epoch = self.registration_epoch.wrapping_add(1);
5406            self.set_sync_needed(true, true);
5407        }
5408        Ok(ids)
5409    }
5410
5411    /// Release the explicit reference to a unit. The unit leaves the
5412    /// registration list when no window base holds it (§4.1); its rows stay.
5413    pub fn unsubscribe(&mut self, id: &str) -> Result<(), String> {
5414        let Some(unit) = self.stored(get_unit(&self.conn, id))? else {
5415            return Ok(());
5416        };
5417        if !unit.explicit {
5418            return Ok(());
5419        }
5420        if !self.stored(window_refs(&self.conn, id))?.is_empty() {
5421            return self.stored(save_unit(
5422                &self.conn,
5423                &UnitRecord {
5424                    explicit: false,
5425                    ..unit
5426                },
5427            ));
5428        }
5429        self.conn
5430            .execute("DELETE FROM _syncular_units WHERE id = ?1", [id])
5431            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5432        self.registration_epoch = self.registration_epoch.wrapping_add(1);
5433        if unit.status == UnitStatus::Active {
5434            self.set_sync_needed(true, true);
5435        }
5436        Ok(())
5437    }
5438
5439    /// Take one reference on a unit (§4.1): validate the scope map, register a
5440    /// new unit catching up with cursor `-1`, re-arm a revoked one (§3.3).
5441    /// Returns the id and whether the registration list changed.
5442    fn hold_unit(&mut self, scopes: UnitScopes, explicit: bool) -> Result<(String, bool), String> {
5443        if scopes.is_empty()
5444            || scopes.values().any(|value| value == "*")
5445            || !self
5446                .schema
5447                .tables
5448                .iter()
5449                .any(|table| covers(table, &scopes))
5450        {
5451            return Err("sync.invalid_subscription: a unit names one non-wildcard value per variable and covers a table".to_owned());
5452        }
5453        let id = ssp2::registration::unit_id(&scope_pairs(&scopes));
5454        let existing = self.stored(get_unit(&self.conn, &id))?;
5455        if let Some(existing) = existing
5456            .as_ref()
5457            .filter(|unit| unit.status != UnitStatus::Revoked)
5458        {
5459            if explicit && !existing.explicit {
5460                self.stored(save_unit(
5461                    &self.conn,
5462                    &UnitRecord {
5463                        explicit,
5464                        ..existing.clone()
5465                    },
5466                ))?;
5467            }
5468            return Ok((id, false));
5469        }
5470        self.delete_pending_evict(&id)?;
5471        self.stored(save_unit(
5472            &self.conn,
5473            &UnitRecord {
5474                id: id.clone(),
5475                scopes,
5476                explicit: explicit || existing.is_some_and(|unit| unit.explicit),
5477                status: UnitStatus::Active,
5478                reason_code: None,
5479                cursor: Some(-1),
5480                bootstrap_state: None,
5481                seeded: Vec::new(),
5482                echo: None,
5483            },
5484        ))?;
5485        Ok((id, true))
5486    }
5487
5488    // -- windows (§4.8) -----------------------------------------------------------
5489
5490    /// §4.8: set the windowed-in values of a base, one unit per value. Added
5491    /// values take a reference on their unit (a new unit bootstraps on the
5492    /// next sync); removed values release theirs, and a unit no reference
5493    /// holds leaves the registration list fused with its first eviction chunk
5494    /// (E1–E4). Idempotent; re-entry cancels a deferred eviction.
5495    pub fn set_window(
5496        &mut self,
5497        base: &WindowBase,
5498        units: &[String],
5499    ) -> Result<CommandEffects, String> {
5500        let base_key = window_base_key(base);
5501        let live = self.load_window_units(&base_key)?;
5502        let mut changed = false;
5503        for value in units {
5504            if live.iter().any(|(held, _)| held == value) {
5505                continue;
5506            }
5507            let rollback = self.capture();
5508            self.begin_observation("syncular_window")?;
5509            let widened = (|| {
5510                let (id, registered) = self.hold_unit(window_unit_scopes(base, value), false)?;
5511                self.sync_needed |= registered;
5512                self.conn
5513                    .execute(
5514                        "INSERT OR REPLACE INTO _syncular_windows(base, unit, sub_id) VALUES (?1, ?2, ?3)",
5515                        rusqlite::params![base_key, value, id],
5516                    )
5517                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5518                let mut batch = ChangeAccumulator::default();
5519                batch.window(&base_key, value);
5520                batch.status = true;
5521                self.finish_observation("syncular_window", batch)
5522            })();
5523            if let Err(error) = widened {
5524                self.rollback_observation("syncular_window");
5525                self.restore(rollback);
5526                return Err(error);
5527            }
5528            changed = true;
5529            self.registration_epoch = self.registration_epoch.wrapping_add(1);
5530        }
5531        for (value, id) in live {
5532            if units.contains(&value) {
5533                continue;
5534            }
5535            self.rebuild_overlay_if_dirty()?;
5536            let rollback = self.capture();
5537            self.begin_observation("syncular_window")?;
5538            let remaining = self.release_window_unit(&base_key, &value, &id);
5539            let remaining = match remaining {
5540                Ok(remaining) => remaining,
5541                Err(error) => {
5542                    self.rollback_observation("syncular_window");
5543                    self.restore(rollback);
5544                    return Err(error);
5545                }
5546            };
5547            changed = true;
5548            self.registration_epoch = self.registration_epoch.wrapping_add(1);
5549            if remaining {
5550                self.drain_pending_evictions()?;
5551            }
5552        }
5553        Ok(if changed && self.transport_enabled {
5554            CommandEffects::interactive()
5555        } else {
5556            CommandEffects::none()
5557        })
5558    }
5559
5560    /// §4.8 completeness oracle: the windowed-in values of a base and those
5561    /// whose unit is not complete. A value is complete once its unit is active
5562    /// and no longer bootstrapping; registration alone is not completeness.
5563    pub fn window_state(&self, base: &WindowBase) -> Result<WindowState, String> {
5564        persisted_window_state(&self.conn, base).map_err(|failure| failure.message)
5565    }
5566
5567    /// Registry rows for a base, ordered by value: (value, unit id).
5568    fn load_window_units(&self, base_key: &str) -> Result<Vec<(String, String)>, String> {
5569        let mut statement = self
5570            .conn
5571            .prepare_cached(
5572                "SELECT unit, sub_id FROM _syncular_windows WHERE base = ?1 ORDER BY unit ASC",
5573            )
5574            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5575        let rows = statement
5576            .query_map([base_key], |row| Ok((row.get(0)?, row.get(1)?)))
5577            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5578        rows.collect::<Result<_, _>>()
5579            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5580    }
5581
5582    /// Every window registry row: (base key, value).
5583    fn load_window_registry(&self) -> Result<Vec<(String, String)>, String> {
5584        let mut statement = self
5585            .conn
5586            .prepare_cached("SELECT base, unit FROM _syncular_windows ORDER BY base, unit")
5587            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5588        let rows = statement
5589            .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))
5590            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5591        rows.collect::<Result<_, _>>()
5592            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5593    }
5594
5595    /// §4.8: release a window value's reference inside the caller's
5596    /// observation. When no reference holds the unit, unregister it fused with
5597    /// its first eviction chunk (E3): delete the rows of every covered table
5598    /// matching the persisted echo, except rows pinned by the outbox (E1) or
5599    /// owned by another unit (§3.3), and park the per-table scopes of what
5600    /// remains in the pending-eviction registry. Fails closed without a local
5601    /// scope-column mapping. Returns whether unpinned rows remain.
5602    fn release_window_unit(
5603        &mut self,
5604        base_key: &str,
5605        value: &str,
5606        id: &str,
5607    ) -> Result<bool, String> {
5608        self.conn
5609            .execute(
5610                "DELETE FROM _syncular_windows WHERE base = ?1 AND unit = ?2",
5611                rusqlite::params![base_key, value],
5612            )
5613            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5614        let mut batch = ChangeAccumulator::default();
5615        batch.window(base_key, value);
5616        batch.status = true;
5617        let unit = self.stored(get_unit(&self.conn, id))?;
5618        let mut remaining = false;
5619        if let Some(unit) = unit.filter(|unit| !unit.explicit) {
5620            if self.stored(window_refs(&self.conn, id))?.is_empty() {
5621                let units = self.stored(load_units(&self.conn))?;
5622                self.conn
5623                    .execute("DELETE FROM _syncular_units WHERE id = ?1", [id])
5624                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5625                self.sync_needed = true;
5626                for table in self.schema.tables.clone() {
5627                    let Some(effective) = unit
5628                        .echo
5629                        .as_deref()
5630                        .filter(|_| covers(&table, &unit.scopes))
5631                        .and_then(|echo| pair_scopes(&unit.scopes, &table.name, Some(echo)))
5632                    else {
5633                        continue;
5634                    };
5635                    let pinned = self.pinned_row_ids(&table.name);
5636                    let owners = unit_owners(&units, &table, id);
5637                    let (more, deferred) =
5638                        self.evict_scope_rows(&table.name, &effective, &owners, &pinned)?;
5639                    remaining |= more;
5640                    if deferred {
5641                        self.save_pending_evict(id, &table.name, &effective)?;
5642                    }
5643                    self.record_scope_map(&mut batch, &table.name, &effective);
5644                }
5645            }
5646        }
5647        self.finish_observation("syncular_window", batch)?;
5648        Ok(remaining)
5649    }
5650
5651    /// §4.8 E1: delete base rows owned by the effective scopes
5652    /// ([`local_scope_predicate`]) EXCEPT pinned primary keys. Returns (more
5653    /// unpinned work, any scoped rows remain). A missing scope-column mapping
5654    /// fails without deleting rows.
5655    fn evict_scope_rows(
5656        &self,
5657        table_name: &str,
5658        effective: &[(String, Vec<String>)],
5659        owners: &[Vec<(String, Vec<String>)>],
5660        pinned: &std::collections::HashSet<String>,
5661    ) -> Result<(bool, bool), String> {
5662        let table = self.schema.table(table_name).ok_or("sync.unknown_table")?;
5663        let Some((scope, mut params)) = local_scope_predicate(table, effective, owners)
5664            .map_err(|()| "sync.scope_revoked".to_owned())?
5665        else {
5666            return Ok((false, false));
5667        };
5668        let scope_params = params.len();
5669        let pk = quote_ident(&table.primary_key);
5670        // Pinned ids are rowId text; the text form matches every key type.
5671        let predicate = if pinned.is_empty() {
5672            scope.clone()
5673        } else {
5674            params.extend(pinned.iter().cloned().map(SqlValue::Text));
5675            let holes = vec!["?"; pinned.len()].join(", ");
5676            format!("{scope} AND CAST({pk} AS TEXT) NOT IN ({holes})")
5677        };
5678        let base = base_table(table_name);
5679        let visible = visible_table(table_name);
5680        let mut ids: Vec<SqlValue> = Vec::new();
5681        for source in [&base, &visible] {
5682            let mut stmt = self
5683                .conn
5684                .prepare_cached(&format!(
5685                    "SELECT {pk} FROM {source} WHERE {predicate} LIMIT 1024"
5686                ))
5687                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5688            ids = stmt
5689                .query_map(rusqlite::params_from_iter(&params), |row| row.get(0))
5690                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
5691                .collect::<Result<_, _>>()
5692                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5693            if !ids.is_empty() {
5694                break;
5695            }
5696        }
5697        if !ids.is_empty() {
5698            for id in &ids {
5699                self.conn
5700                    .execute(
5701                        "DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2",
5702                        rusqlite::params![table_name, id],
5703                    )
5704                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5705            }
5706            let holes = vec!["?"; ids.len()].join(", ");
5707            for source in [&base, &visible] {
5708                self.conn
5709                    .execute(
5710                        &format!("DELETE FROM {source} WHERE {pk} IN ({holes})"),
5711                        rusqlite::params_from_iter(&ids),
5712                    )
5713                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5714            }
5715        }
5716        let mut remaining = false;
5717        let mut deferred = false;
5718        for source in [&base, &visible] {
5719            remaining |= self
5720                .conn
5721                .query_row(
5722                    &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {predicate})"),
5723                    rusqlite::params_from_iter(&params),
5724                    |row| row.get::<_, bool>(0),
5725                )
5726                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5727            deferred |= self
5728                .conn
5729                .query_row(
5730                    &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {scope})"),
5731                    rusqlite::params_from_iter(&params[..scope_params]),
5732                    |row| row.get::<_, bool>(0),
5733                )
5734                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5735        }
5736        Ok((remaining, deferred))
5737    }
5738
5739    /// §4.8 E1: continue deferred evictions after the outbox drains or between
5740    /// chunks, retaining outbox pins and their durable marker. A unit that
5741    /// re-entered the window has no pending entry left.
5742    fn drain_pending_evictions(&mut self) -> Result<(), String> {
5743        let pending = self.load_pending_evictions()?;
5744        if pending.is_empty() {
5745            return Ok(());
5746        }
5747        self.rebuild_overlay_if_dirty()?;
5748        for (unit_id, table_name, effective) in pending {
5749            let Some(table) = self.schema.table(&table_name).cloned() else {
5750                self.delete_pending_evict_table(&unit_id, &table_name)?;
5751                continue;
5752            };
5753            loop {
5754                std::thread::yield_now();
5755                self.begin_observation("syncular_evict")?;
5756                let chunk = (|| {
5757                    let pinned = self.pinned_row_ids(&table_name);
5758                    let owners =
5759                        unit_owners(&self.stored(load_units(&self.conn))?, &table, &unit_id);
5760                    let (remaining, deferred) =
5761                        self.evict_scope_rows(&table_name, &effective, &owners, &pinned)?;
5762                    if !deferred {
5763                        self.delete_pending_evict_table(&unit_id, &table_name)?;
5764                    }
5765                    let mut batch = ChangeAccumulator::default();
5766                    self.record_scope_map(&mut batch, &table_name, &effective);
5767                    self.finish_observation("syncular_evict", batch)?;
5768                    Ok(remaining)
5769                })();
5770                match chunk {
5771                    Ok(true) => {}
5772                    Ok(false) => break,
5773                    Err(error) => {
5774                        self.rollback_observation("syncular_evict");
5775                        return Err(error);
5776                    }
5777                }
5778            }
5779        }
5780        Ok(())
5781    }
5782
5783    /// §4.8 E1: primary keys of `table` referenced by a pending outbox
5784    /// commit — rows that MUST NOT be evicted until the commit drains.
5785    fn pinned_row_ids(&self, table: &str) -> std::collections::HashSet<String> {
5786        let mut pinned = std::collections::HashSet::new();
5787        for commit in self
5788            .outbox
5789            .iter()
5790            .map(|commit| &**commit)
5791            .chain(self.failed_commits.iter().map(|(commit, _)| commit))
5792        {
5793            for op in &commit.ops {
5794                if op.table == table {
5795                    pinned.insert(op.row_id.clone());
5796                }
5797            }
5798        }
5799        pinned
5800    }
5801
5802    fn save_pending_evict(
5803        &self,
5804        unit_id: &str,
5805        table: &str,
5806        effective: &[(String, Vec<String>)],
5807    ) -> Result<(), String> {
5808        self.conn.execute(
5809            "INSERT OR REPLACE INTO _syncular_pending_evictions(unit_id, tbl, effective_scopes) VALUES (?1, ?2, ?3)",
5810            rusqlite::params![unit_id, table, scope_map_to_json(effective).to_string()],
5811        ).map(|_| ()).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5812    }
5813
5814    /// Re-entry cancels every pending eviction of the unit.
5815    fn delete_pending_evict(&self, unit_id: &str) -> Result<(), String> {
5816        self.conn
5817            .execute(
5818                "DELETE FROM _syncular_pending_evictions WHERE unit_id = ?1",
5819                [unit_id],
5820            )
5821            .map(|_| ())
5822            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5823    }
5824
5825    fn delete_pending_evict_table(&self, unit_id: &str, table: &str) -> Result<(), String> {
5826        self.conn
5827            .execute(
5828                "DELETE FROM _syncular_pending_evictions WHERE unit_id = ?1 AND tbl = ?2",
5829                [unit_id, table],
5830            )
5831            .map(|_| ())
5832            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5833    }
5834
5835    fn load_pending_evictions(&self) -> Result<Vec<PendingEvict>, String> {
5836        let mut stmt = self
5837            .conn
5838            .prepare("SELECT unit_id, tbl, effective_scopes FROM _syncular_pending_evictions")
5839            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5840        let rows = stmt
5841            .query_map([], |row| {
5842                Ok((
5843                    row.get::<_, String>(0)?,
5844                    row.get::<_, String>(1)?,
5845                    row.get::<_, String>(2)?,
5846                ))
5847            })
5848            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5849        rows.map(|row| {
5850            let (unit_id, table, json) =
5851                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5852            let value: Value = serde_json::from_str(&json)
5853                .map_err(|_| "sync.local_corrupt: invalid pending eviction scopes".to_owned())?;
5854            let effective = json_to_scope_map(&value)
5855                .map_err(|_| "sync.local_corrupt: invalid pending eviction scopes".to_owned())?;
5856            Ok((unit_id, table, effective))
5857        })
5858        .collect()
5859    }
5860
5861    /// Select failed-intent retention; reload durable retained aggregates.
5862    pub fn set_retain_failed_commits(&mut self, enabled: bool) -> Result<(), String> {
5863        self.retain_failed_commits = enabled;
5864        for (commit, _) in self.failed_commits.iter() {
5865            for operation in &commit.ops {
5866                self.overlay_dirty.table(&operation.table);
5867            }
5868        }
5869        self.failed_commits = self.load_failed_commits()?.into();
5870        for (commit, _) in self.failed_commits.iter() {
5871            for operation in &commit.ops {
5872                self.overlay_dirty.table(&operation.table);
5873            }
5874        }
5875        Ok(())
5876    }
5877
5878    fn load_failed_commits(&self) -> Result<Vec<FailedCommit>, String> {
5879        let mut statement = self.conn.prepare("SELECT client_commit_id, operations_json, initial_json FROM _syncular_failed_commits ORDER BY seq")
5880            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5881        let rows = statement
5882            .query_map([], |row| {
5883                Ok((
5884                    row.get::<_, String>(0)?,
5885                    row.get::<_, String>(1)?,
5886                    row.get::<_, String>(2)?,
5887                ))
5888            })
5889            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5890        rows.map(|row| {
5891            let (id, operations, initial) =
5892                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5893            let operations: Vec<CommitOperation> =
5894                serde_json::from_str(&operations).map_err(|_| "sync.local_corrupt".to_owned())?;
5895            let initial =
5896                serde_json::from_str(&initial).map_err(|_| "sync.local_corrupt".to_owned())?;
5897            Ok((
5898                OutboxCommit {
5899                    client_commit_id: id,
5900                    ops: operations
5901                        .into_iter()
5902                        .map(|op| OutboxOp {
5903                            upsert: op.op == "upsert",
5904                            table: op.table,
5905                            row_id: op.row_id,
5906                            base_version: op.base_version,
5907                            values: op.values,
5908                            insert: op.insert,
5909                        })
5910                        .collect(),
5911                },
5912                initial,
5913            ))
5914        })
5915        .collect()
5916    }
5917
5918    fn retain_failed_commit(&mut self, id: &str, operations: &[OutboxOp]) -> Result<(), String> {
5919        let mut initial = Map::new();
5920        for operation in operations {
5921            // A table the current schema removed has no rows to retain.
5922            let Some(table) = self.schema.table(&operation.table) else {
5923                continue;
5924            };
5925            let initial_row = match self.stored_row(table, &operation.row_id, true)? {
5926                Some(row) => Some(row),
5927                None => self.visible_row(table, &operation.row_id)?,
5928            };
5929            if let Some((values, _)) = initial_row {
5930                let mut intended: Map<String, Value> = table
5931                    .columns
5932                    .iter()
5933                    .zip(values.iter())
5934                    .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
5935                    .collect();
5936                if let Some(patch) = &operation.values {
5937                    intended.extend(patch.clone());
5938                }
5939                initial.insert(
5940                    serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
5941                        .map_err(|_| "sync.local_corrupt".to_owned())?,
5942                    Value::Object(intended),
5943                );
5944            }
5945        }
5946        self.conn.execute("INSERT INTO _syncular_failed_commits(client_commit_id, operations_json, initial_json) VALUES (?1, ?2, ?3)", rusqlite::params![id,
5947            serde_json::to_string(&operations.iter().map(CommitOperation::from).collect::<Vec<_>>()).map_err(|_| "sync.local_corrupt".to_owned())?,
5948            serde_json::to_string(&initial).map_err(|_| "sync.local_corrupt".to_owned())?])
5949            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5950        self.failed_commits.push((
5951            OutboxCommit {
5952                client_commit_id: id.to_owned(),
5953                ops: operations.to_vec(),
5954            },
5955            initial,
5956        ));
5957        Ok(())
5958    }
5959
5960    fn drop_failed_commits(
5961        &mut self,
5962        matches: impl Fn(&str, &Map<String, Value>) -> bool,
5963    ) -> Result<Vec<OutboxCommit>, String> {
5964        let mut doomed = self.drop_acknowledged_rows(&matches)?;
5965        for (commit, initial) in self.failed_commits.iter() {
5966            for operation in &commit.ops {
5967                let key =
5968                    serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
5969                        .map_err(|_| "sync.local_corrupt".to_owned())?;
5970                if initial
5971                    .get(&key)
5972                    .and_then(Value::as_object)
5973                    .is_some_and(|values| matches(&operation.table, values))
5974                {
5975                    doomed.push(commit.clone());
5976                    break;
5977                }
5978            }
5979        }
5980        for commit in &doomed {
5981            self.conn
5982                .execute(
5983                    "DELETE FROM _syncular_failed_commits WHERE client_commit_id = ?1",
5984                    rusqlite::params![commit.client_commit_id],
5985                )
5986                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5987        }
5988        for commit in &doomed {
5989            self.conn.execute("UPDATE _syncular_commit_outcomes SET operations_json = NULL, results_json = '[]' WHERE client_commit_id = ?1", rusqlite::params![commit.client_commit_id]).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5990        }
5991        self.conflicts.retain(|record| {
5992            !doomed
5993                .iter()
5994                .any(|commit| commit.client_commit_id == record.client_commit_id)
5995        });
5996        self.rejections.retain(|record| {
5997            !doomed
5998                .iter()
5999                .any(|commit| commit.client_commit_id == record.client_commit_id)
6000        });
6001        self.failed_commits.retain(|(commit, _)| {
6002            !doomed
6003                .iter()
6004                .any(|dropped| dropped.client_commit_id == commit.client_commit_id)
6005        });
6006        for commit in &doomed {
6007            for operation in &commit.ops {
6008                self.overlay_dirty.table(&operation.table);
6009            }
6010        }
6011        Ok(doomed)
6012    }
6013
6014    /// Record one commit of local mutations. The returned commit ID is the
6015    /// §2.3 idempotency key. Failures are structured so callers read
6016    /// `code`/`message`/`details` instead of parsing a string prefix.
6017    pub fn mutate(&mut self, mutations: Vec<Mutation>) -> Result<String, ClientError> {
6018        self.mutate_with_commit_id(mutations, None)
6019    }
6020
6021    /// [`Self::mutate`] under a caller-supplied `clientCommitId` (§7.1), so an
6022    /// app that loses the reply knows its commit. The ID is non-empty (§6.1);
6023    /// one this replica still holds fails with `client.commit_id_reused`.
6024    pub fn mutate_with_commit_id(
6025        &mut self,
6026        mutations: Vec<Mutation>,
6027        client_commit_id: Option<String>,
6028    ) -> Result<String, ClientError> {
6029        let prior = self.storage_failure.borrow_mut().take();
6030        if let Some(id) = &client_commit_id {
6031            let held = if id.is_empty() {
6032                Err("sync.invalid_request: clientCommitId is empty".to_owned())
6033            } else {
6034                self.conn
6035                    .query_row(
6036                        "SELECT EXISTS(SELECT 1 FROM _syncular_outbox WHERE commit_id=?1 UNION ALL SELECT 1 FROM _syncular_commit_outcomes WHERE client_commit_id=?1)",
6037                        [id],
6038                        |row| row.get(0),
6039                    )
6040                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
6041            };
6042            match held {
6043                Ok(false) => {}
6044                Ok(true) => {
6045                    return self.finish_authoring(
6046                        prior,
6047                        Err(ClientError {
6048                            code: "client.commit_id_reused".to_owned(),
6049                            message: "clientCommitId is held".to_owned(),
6050                            details: Some(json!({ "clientCommitId": id })),
6051                            retryable: false,
6052                        }),
6053                    )
6054                }
6055                Err(message) => return self.finish_authoring(prior, Err(message.into())),
6056            }
6057        }
6058        let outcome = (|| -> Result<String, ClientError> {
6059            if mutations.is_empty() {
6060                return Err(
6061                    "sync.invalid_request: a commit must contain at least one operation (§6.1)"
6062                        .to_owned()
6063                        .into(),
6064                );
6065            }
6066            let mut ops = Vec::with_capacity(mutations.len());
6067            for mutation in mutations {
6068                match mutation {
6069                    Mutation::Patch {
6070                        table,
6071                        values,
6072                        base_version,
6073                    } => {
6074                        let schema_table = self
6075                            .schema
6076                            .table(&table)
6077                            .ok_or_else(|| "sync.unknown_table".to_owned())?;
6078                        let values = normalize_values_casing(schema_table, values)?;
6079                        let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
6080                        ops.push(self.prepare_patch(&table, &row_id, values, base_version)?);
6081                    }
6082                    Mutation::Upsert {
6083                        table,
6084                        values,
6085                        base_version,
6086                    } => {
6087                        let schema_table = self.schema.table(&table).ok_or_else(|| {
6088                            format!("sync.unknown_table: unknown table {table:?}")
6089                        })?;
6090                        // §5: value keys are accepted in snake_case AND the
6091                        // generated row types' camelCase; normalize to SQL truth
6092                        // before the pk lookup / codec see them.
6093                        let values = normalize_values_casing(schema_table, values)?;
6094                        let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
6095                        // §6.7 `mutate` marks every column present: fill missing
6096                        // nullable columns with NULL before the payload is built.
6097                        let values = full_row_values(schema_table, values)?;
6098                        self.refuse_sealed_row(schema_table, &row_id)?;
6099                        // §5.11: validate the payload encodes with the current
6100                        // codec. A key-selection or unknown-key failure is a
6101                        // durable rejection at the push seam (§10.3), never an
6102                        // author-time error. Full-row `mutate` presents every
6103                        // column, so no stored fallback is needed.
6104                        self.validate_author_encode(schema_table, &row_id, &values)?;
6105                        // §7.1 insert intent: the row is absent from the
6106                        // visible replica at call time, before this batch
6107                        // applies anything.
6108                        let insert = !self.visible_row_exists(schema_table, &row_id)?;
6109                        ops.push(OutboxOp {
6110                            upsert: true,
6111                            table,
6112                            row_id,
6113                            base_version,
6114                            values: Some(values),
6115                            insert,
6116                        });
6117                    }
6118                    Mutation::Delete {
6119                        table,
6120                        row_id,
6121                        base_version,
6122                    } => {
6123                        let schema_table = self.schema.table(&table).ok_or_else(|| {
6124                            format!("sync.unknown_table: unknown table {table:?}")
6125                        })?;
6126                        self.refuse_sealed_row(schema_table, &row_id)?;
6127                        // §2.4: the server rejects a rowId holding U+0000.
6128                        if row_id.contains('\0') {
6129                            return Err("sync.invalid_request: rowId holds U+0000 (§2.4)"
6130                                .to_owned()
6131                                .into());
6132                        }
6133                        ops.push(OutboxOp {
6134                            upsert: false,
6135                            table,
6136                            row_id,
6137                            base_version,
6138                            values: None,
6139                            insert: false,
6140                        });
6141                    }
6142                }
6143            }
6144            Ok(self.record_outbox_commit(ops, client_commit_id)?)
6145        })();
6146        self.finish_authoring(prior, outcome)
6147    }
6148
6149    /// Scope the retained `storage_failure` diagnostic to this call so an
6150    /// earlier operation's failure cannot classify a new one, and surface the
6151    /// call's own classified failure with its details. A fresh classified
6152    /// failure replaces the retained diagnostic; otherwise the earlier one is
6153    /// kept for the next sync.
6154    fn finish_authoring(
6155        &self,
6156        prior: Option<QueryReadFailure>,
6157        outcome: Result<String, ClientError>,
6158    ) -> Result<String, ClientError> {
6159        let fresh = self.storage_failure.borrow_mut().take();
6160        *self.storage_failure.borrow_mut() = fresh.clone().or(prior);
6161        outcome.map_err(|error| fresh.map(ClientError::from).unwrap_or(error))
6162    }
6163
6164    fn record_outbox_commit(
6165        &mut self,
6166        ops: Vec<OutboxOp>,
6167        client_commit_id: Option<String>,
6168    ) -> Result<String, String> {
6169        let commit = OutboxCommit {
6170            client_commit_id: client_commit_id.unwrap_or_else(|| uuid::Uuid::new_v4().to_string()),
6171            ops,
6172        };
6173        self.begin_observation("syncular_mutation")?;
6174        let mut batch = ChangeAccumulator::default();
6175        for op in &commit.ops {
6176            let mut precise = self.record_row_scopes(&mut batch, &op.table, &op.row_id, false);
6177            if let Some(values) = &op.values {
6178                if let Some(table) = self.schema.table(&op.table) {
6179                    for scope in &table.scope_variables {
6180                        if let Some(Value::String(value)) = values.get(&scope.column) {
6181                            batch.scope(&op.table, format!("{}:{value}", scope.prefix));
6182                            precise = true;
6183                        }
6184                    }
6185                }
6186            }
6187            if !precise {
6188                batch.table(&op.table);
6189            }
6190        }
6191        if let Err(error) = self.persist_outbox_insert(&commit) {
6192            self.rollback_observation("syncular_mutation");
6193            return Err(error);
6194        }
6195        // A clean overlay contains the FIFO fold of every previous commit, so
6196        // the new commit applies over it. §7.1: an operation that does not
6197        // apply (a secondary unique collision) fails the whole commit; the
6198        // savepoint rollback removes the outbox entry and every visible write.
6199        // The append undoes itself: a rollback capture would copy the outbox.
6200        let was_dirty = self.overlay_dirty.snapshot();
6201        let id = commit.client_commit_id.clone();
6202        let appended = self.rebuild_overlay_if_dirty().and_then(|()| {
6203            commit
6204                .ops
6205                .iter()
6206                .try_for_each(|op| self.apply_outbox_op(op))?;
6207            self.outbox.push(Arc::new(commit));
6208            batch.status = true;
6209            self.finish_observation("syncular_mutation", batch)
6210                .inspect_err(|_| drop(self.outbox.pop()))
6211        });
6212        if let Err(error) = appended {
6213            self.rollback_observation("syncular_mutation");
6214            self.overlay_dirty.restore(was_dirty);
6215            return Err(error);
6216        }
6217        Ok(id)
6218    }
6219
6220    /// Record one §6.1 sparse upsert: the primary key plus the supplied
6221    /// non-scope columns are present; every other column is absent and stays
6222    /// untouched on the server and in the local overlay. An absent local base
6223    /// refuses the whole batch with sync.row_missing before enqueueing (§7.1).
6224    /// Failures are structured so callers read `code`/`message`/`details`.
6225    pub fn patch(
6226        &mut self,
6227        table: &str,
6228        row_id: &str,
6229        partial: Map<String, Value>,
6230        base_version: Option<i64>,
6231    ) -> Result<String, ClientError> {
6232        let prior = self.storage_failure.borrow_mut().take();
6233        let outcome = self.perform_patch(table, row_id, partial, base_version);
6234        self.finish_authoring(prior, outcome)
6235    }
6236
6237    /// The unscoped patch body. Public authoring entry points scope
6238    /// `storage_failure` around the whole operation, including any pre-read.
6239    fn perform_patch(
6240        &mut self,
6241        table: &str,
6242        row_id: &str,
6243        partial: Map<String, Value>,
6244        base_version: Option<i64>,
6245    ) -> Result<String, ClientError> {
6246        let operation = self.prepare_patch(table, row_id, partial, base_version)?;
6247        Ok(self.record_outbox_commit(vec![operation], None)?)
6248    }
6249
6250    fn prepare_patch(
6251        &self,
6252        table: &str,
6253        row_id: &str,
6254        partial: Map<String, Value>,
6255        base_version: Option<i64>,
6256    ) -> Result<OutboxOp, ClientError> {
6257        let schema_table = self
6258            .schema
6259            .table(table)
6260            .ok_or_else(|| format!("sync.unknown_table: unknown table {table:?}"))?;
6261        let partial = normalize_values_casing(schema_table, partial)?;
6262        if let Some(pk) = partial.get(&schema_table.primary_key) {
6263            if render_row_id_json(Some(pk)).ok().as_deref() != Some(row_id) {
6264                return Err(invalid_authoring(
6265                    "patch cannot change the primary key",
6266                    json!({ "table": table }),
6267                ));
6268            }
6269        }
6270        if !self.visible_row_exists(schema_table, row_id)? {
6271            return Err("sync.row_missing: a sparse patch requires a local row"
6272                .to_owned()
6273                .into());
6274        }
6275        self.refuse_sealed_row(schema_table, row_id)?;
6276        // §3.4 rule 5 / §6.2: scope columns are immutable on update. The
6277        // server accepts a present scope column whose value equals the
6278        // stored row and applies it as a no-op, so a decoded envelope
6279        // round-trips. The local row is the only value the client can prove
6280        // equality against; when the row is absent locally the client cannot
6281        // prove equality against the server's stored row and fails closed.
6282        // A primary key that is also a scope column needs no stored-row
6283        // read: the primary key in a sparse payload is by construction the
6284        // row id being patched (§6.1), so its value is proven equal already.
6285        for scope in &schema_table.scope_variables {
6286            if !partial.contains_key(&scope.column) || scope.column == schema_table.primary_key {
6287                continue;
6288            }
6289            let Some(column) = schema_table
6290                .columns
6291                .iter()
6292                .find(|column| column.name == scope.column)
6293            else {
6294                continue;
6295            };
6296            let stored = self
6297                .conn
6298                .query_row(
6299                    &format!(
6300                        "SELECT {} FROM {} WHERE {}",
6301                        quote_ident(&scope.column),
6302                        visible_table(&schema_table.name),
6303                        row_id_predicate(schema_table)
6304                    ),
6305                    rusqlite::params![row_id],
6306                    |row| Ok(sql_ref_to_json(column, row.get_ref(0)?)),
6307                )
6308                .optional()
6309                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6310            // Decode the supplied value through the same column-type path the
6311            // rest of the patch encoding uses, so a value that only equals
6312            // the stored one in another number representation still compares
6313            // equal.
6314            let incoming = json_to_column_value(column, partial.get(&scope.column))
6315                .ok()
6316                .map(|value| column_value_to_json(&value));
6317            // A stored blob never equals a patch value, matching the server's
6318            // scope-column comparison (§3.4): bytes are unstorable as scope
6319            // values.
6320            let equal = stored.as_ref().is_some_and(|stored| {
6321                stored.get("$bytes").is_none() && incoming.as_ref() == Some(stored)
6322            });
6323            if !equal {
6324                return Err(invalid_authoring(
6325                    "patch cannot write a scope column (§3.4)",
6326                    json!({ "table": table, "column": scope.column }),
6327                ));
6328            }
6329        }
6330        let mut values = partial;
6331        // Presence-set semantics: a proven-equal scope column is dropped, so
6332        // the patch leaves the stored value untouched (§6.2).
6333        for scope in &schema_table.scope_variables {
6334            values.remove(&scope.column);
6335        }
6336        let pk_value = match schema_table.columns[schema_table.pk_index].ty {
6337            // §2.4: the rowId of an integer key is its canonical decimal form.
6338            ColumnType::Integer => match row_id.parse::<i64>() {
6339                Ok(v)
6340                    if v.to_string() == row_id
6341                        && (-ssp2::primitives::I64_SAFE_MAX..=ssp2::primitives::I64_SAFE_MAX)
6342                            .contains(&v) =>
6343                {
6344                    Value::from(v)
6345                }
6346                _ => {
6347                    return Err(format!(
6348                    "sync.invalid_request: table {table:?}: invalid integer primary key {row_id:?}"
6349                )
6350                    .into())
6351                }
6352            },
6353            ColumnType::Boolean => match row_id {
6354                "true" => Value::Bool(true),
6355                "false" => Value::Bool(false),
6356                _ => {
6357                    return Err(format!(
6358                    "sync.invalid_request: table {table:?}: invalid boolean primary key {row_id:?}"
6359                )
6360                    .into())
6361                }
6362            },
6363            _ => Value::from(row_id.to_owned()),
6364        };
6365        values.insert(schema_table.primary_key.clone(), pk_value);
6366        // §6.1: a patch cannot null a non-nullable column.
6367        if let Some(column) = schema_table
6368            .columns
6369            .iter()
6370            .find(|column| !column.nullable && values.get(&column.name) == Some(&Value::Null))
6371        {
6372            return Err(invalid_authoring(
6373                "a non-nullable column is null (§6.1)",
6374                json!({ "table": table, "column": column.name }),
6375            ));
6376        }
6377        // §5.11: structural validation only; an absent key-id selector is
6378        // resolved at the push seam from the stored local row, and a
6379        // key-selection or unknown-key failure becomes a durable
6380        // `client.encrypt_failed` rejection there (§10.3), never an
6381        // author-time error.
6382        self.validate_author_encode(schema_table, row_id, &values)?;
6383        Ok(OutboxOp {
6384            upsert: true,
6385            table: table.to_owned(),
6386            row_id: row_id.to_owned(),
6387            base_version,
6388            values: Some(values),
6389            insert: false,
6390        })
6391    }
6392
6393    /// §5.11 key-id fallback: the stored local row's selector value as a
6394    /// positional row (only the selector slot filled). Returns None when no
6395    /// selector is configured, the row is locally absent, or the stored
6396    /// value is not a usable key id. A failed storage read is an error.
6397    fn stored_key_fallback(
6398        &self,
6399        table: &TableSchema,
6400        row_id: &str,
6401    ) -> Result<Option<Row>, String> {
6402        let Some(selector) = self.encryption.key_id_columns.get(&table.name) else {
6403            return Ok(None);
6404        };
6405        let Some(index) = table
6406            .columns
6407            .iter()
6408            .position(|column| &column.name == selector)
6409        else {
6410            return Ok(None);
6411        };
6412        let sql = format!(
6413            "SELECT {} FROM {} WHERE {}",
6414            quote_ident(selector),
6415            visible_table(&table.name),
6416            row_id_predicate(table)
6417        );
6418        let key = self
6419            .conn
6420            .query_row(&sql, rusqlite::params![row_id], |row| {
6421                row.get::<_, Option<String>>(0)
6422            })
6423            .optional()
6424            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
6425            .flatten()
6426            .filter(|key| !key.is_empty());
6427        let Some(key) = key else {
6428            return Ok(None);
6429        };
6430        let mut fallback: Row = vec![None; table.columns.len()];
6431        fallback[index] = Some(ColumnValue::String(key));
6432        Ok(Some(fallback))
6433    }
6434
6435    /// §5.11/§10.3 author-time encode check: a structural failure (a
6436    /// non-nullable NULL, a value the codec rejects) is an author-time
6437    /// error; a key-selection or unknown-key failure is deferred to the push
6438    /// seam, where it becomes a durable `client.encrypt_failed` rejection.
6439    fn validate_author_encode(
6440        &self,
6441        table: &TableSchema,
6442        row_id: &str,
6443        values: &Map<String, Value>,
6444    ) -> Result<(), String> {
6445        match encode_sparse_row_json(table, row_id, values, &self.encryption, None) {
6446            Err(error) if error.starts_with(ENCRYPT_FAILED_CODE) => Ok(()),
6447            result => result.map(|_| ()),
6448        }
6449    }
6450
6451    /// §5.11/§10.3: drop every pending commit whose push encode fails with
6452    /// `client.encrypt_failed` (an unresolvable key id or an unknown key) and
6453    /// raise one durable rejection per commit. `sync()` never aborts for an
6454    /// encode failure. Mirrors §7.4.4's `drop_incompatible_outbox`, including
6455    /// undoing the dropped commit's purely-optimistic rows.
6456    /// §7.1/§7.4.4: drop pending commits whose values the current codec refuses
6457    /// (an envelope or a type an earlier version accepted). `with_encryption`
6458    /// additionally drops commits whose §5.11 encryption cannot resolve a key;
6459    /// the startup boundary leaves those to the send seam, where hosts configure
6460    /// keys after open.
6461    /// Recovery runs as one observation. An enclosing observation (the guarded
6462    /// schema reset) passes its `batch`, so the rejection, outcome, and table
6463    /// domains join that change event instead of publishing a second one.
6464    fn drop_unencodable_outbox(
6465        &mut self,
6466        with_encryption: bool,
6467        batch: Option<&mut ChangeAccumulator>,
6468    ) -> Result<bool, String> {
6469        let mut failures = Vec::new();
6470        for commit in self.outbox.iter() {
6471            for op in &commit.ops {
6472                let Some(table) = self.schema.table(&op.table).filter(|_| op.upsert) else {
6473                    continue;
6474                };
6475                // An insert lacking a non-nullable column is left to
6476                // `drop_incompatible_outbox` (§7.4.4).
6477                let Some(values) = op.presented_values(table) else {
6478                    continue;
6479                };
6480                let fallback = self.stored_key_fallback(table, &op.row_id)?;
6481                let failure = match encode_sparse_row_json(
6482                    table,
6483                    &op.row_id,
6484                    &values,
6485                    &self.encryption,
6486                    fallback.as_ref(),
6487                ) {
6488                    Err(error) if error.starts_with(ENCRYPT_FAILED_CODE) => {
6489                        with_encryption.then_some((error, ENCRYPT_FAILED_CODE))
6490                    }
6491                    // §7.1/§7.4.4: a value the current codec refuses was
6492                    // persisted by a version that accepted it; it leaves
6493                    // through the incompatible-outbox path, so the FIFO
6494                    // queue never wedges.
6495                    Err(error) if error.starts_with(INVALID_REQUEST_CODE) => Some((
6496                        LEGACY_VALUE_REFUSAL_MESSAGE.to_owned(),
6497                        OUTBOX_INCOMPATIBLE_CODE,
6498                    )),
6499                    _ => None,
6500                };
6501                if let Some((error, code)) = failure {
6502                    failures.push((commit.clone(), error, code));
6503                    break;
6504                }
6505            }
6506        }
6507        if failures.is_empty() {
6508            return Ok(false);
6509        }
6510        // §7.4.4: one observation covers the durable outcome writes, the outbox
6511        // removal, the in-memory drop, and the overlay rebuild, so a storage fault
6512        // leaves the replica and the client unchanged, a successful recovery
6513        // publishes one batch with the rejection, outcome, and table domains, and
6514        // an ordinary open with nothing to recover publishes nothing.
6515        let rollback = self.capture();
6516        let owns_observation = batch.is_none();
6517        let owns_transaction = self.conn.is_autocommit();
6518        let mut domains = ChangeAccumulator::default();
6519        if owns_observation {
6520            self.begin_observation("syncular_value_recovery")?;
6521        }
6522        let dropped = (|| -> Result<(), String> {
6523            for (commit, message, code) in failures {
6524                if self.retain_failed_commits && code == ENCRYPT_FAILED_CODE {
6525                    self.retain_failed_commit(&commit.client_commit_id, &commit.ops)?;
6526                }
6527                let representable = commit.ops.iter().all(Self::outbox_op_is_journalable);
6528                let rejection = RejectionRecord {
6529                    client_commit_id: commit.client_commit_id.clone(),
6530                    op_index: 0,
6531                    code: code.to_owned(),
6532                    message,
6533                    retryable: false,
6534                    details: (code == OUTBOX_INCOMPATIBLE_CODE).then(|| RejectionDetails {
6535                        field_paths: None,
6536                        reason: Some(INVALID_STORED_VALUES_REASON.to_owned()),
6537                        required_action: None,
6538                        references: None,
6539                    }),
6540                    operation: representable
6541                        .then(|| commit.ops.first())
6542                        .flatten()
6543                        .map(CommitOperation::from),
6544                };
6545                self.persist_commit_outcome(
6546                    &commit.client_commit_id,
6547                    CommitOutcomeStatus::Rejected,
6548                    &[CommitOperationOutcome::Error {
6549                        rejection: rejection.clone(),
6550                    }],
6551                    representable.then_some(commit.ops.as_slice()),
6552                )?;
6553                self.delete_outbox_persisted(&commit.client_commit_id)?;
6554                self.outbox
6555                    .retain(|candidate| candidate.client_commit_id != commit.client_commit_id);
6556                self.rejections.push(rejection);
6557                for operation in &commit.ops {
6558                    domains.table(operation.table.as_str());
6559                }
6560            }
6561            domains.status = true;
6562            domains.rejections = true;
6563            domains.outcomes = true;
6564            self.prune_commit_outcomes()?;
6565            // The visible projection is re-derived from the surviving base + outbox.
6566            self.overlay_dirty.set(true);
6567            self.rebuild_overlay()?;
6568            Ok(())
6569        })();
6570        let result = match batch {
6571            Some(batch) => {
6572                if dropped.is_ok() {
6573                    for table in domains.tables.keys() {
6574                        batch.table(table.as_str());
6575                    }
6576                    batch.status |= domains.status;
6577                    batch.rejections |= domains.rejections;
6578                    batch.outcomes |= domains.outcomes;
6579                }
6580                dropped
6581            }
6582            None => {
6583                dropped.and_then(|()| self.finish_observation("syncular_value_recovery", domains))
6584            }
6585        };
6586        if let Err(error) = result {
6587            if owns_observation {
6588                if owns_transaction {
6589                    if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
6590                        let mut failure = self.storage_failure.borrow_mut();
6591                        let failure =
6592                            failure.get_or_insert_with(|| QueryReadFailure::from(error.clone()));
6593                        failure.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
6594                    }
6595                } else {
6596                    self.rollback_observation("syncular_value_recovery");
6597                }
6598            }
6599            self.restore(rollback);
6600            return Err(error);
6601        }
6602        Ok(true)
6603    }
6604
6605    pub fn pending_commit_ids(&self) -> Vec<String> {
6606        self.outbox
6607            .iter()
6608            .map(|c| c.client_commit_id.clone())
6609            .collect()
6610    }
6611
6612    /// §6.1: the sparse push payload bytes of every pending upsert operation,
6613    /// FIFO — exactly what the next push round would put on the wire. Used by
6614    /// conformance B.21(h) to pin cross-core byte identity.
6615    pub fn pending_payloads(&self) -> Result<Vec<Vec<u8>>, String> {
6616        let mut payloads = Vec::new();
6617        for commit in self.outbox.iter() {
6618            for op in &commit.ops {
6619                let Some(table) = self.schema.table(&op.table).filter(|_| op.upsert) else {
6620                    continue;
6621                };
6622                let Some(values) = op.presented_values(table) else {
6623                    continue;
6624                };
6625                let fallback = self.stored_key_fallback(table, &op.row_id)?;
6626                if let Ok(payload) = encode_sparse_row_json(
6627                    table,
6628                    &op.row_id,
6629                    &values,
6630                    &self.encryption,
6631                    fallback.as_ref(),
6632                ) {
6633                    payloads.push(payload);
6634                }
6635            }
6636        }
6637        Ok(payloads)
6638    }
6639
6640    pub fn conflicts(&self) -> &[ConflictRecord] {
6641        &self.conflicts
6642    }
6643
6644    pub fn rejections(&self) -> &[RejectionRecord] {
6645        &self.rejections
6646    }
6647
6648    pub fn resolve_commit_outcome(
6649        &mut self,
6650        input: ResolveCommitOutcomeInput,
6651    ) -> Result<CommitOutcome, String> {
6652        let current = self
6653            .commit_outcome(&input.client_commit_id)?
6654            .ok_or_else(|| {
6655                format!(
6656                    "sync.outcome_not_found: no durable outcome exists for {:?}",
6657                    input.client_commit_id
6658                )
6659            })?;
6660        if current.resolution != CommitOutcomeResolution::Active {
6661            return Ok(current);
6662        }
6663        if input.resolution == CommitOutcomeResolution::Active {
6664            return Err("sync.invalid_request: resolution must leave active state".to_owned());
6665        }
6666        match input.resolution {
6667            CommitOutcomeResolution::Superseded => {
6668                let replacement = input
6669                    .replacement_client_commit_id
6670                    .as_deref()
6671                    .filter(|value| !value.is_empty() && *value != input.client_commit_id)
6672                    .ok_or_else(|| {
6673                        "sync.invalid_request: superseded outcomes require a distinct replacementClientCommitId"
6674                            .to_owned()
6675                    })?;
6676                let _ = replacement;
6677            }
6678            _ if input.replacement_client_commit_id.is_some() => {
6679                return Err(
6680                    "sync.invalid_request: replacementClientCommitId is valid only for superseded outcomes"
6681                        .to_owned(),
6682                );
6683            }
6684            _ => {}
6685        }
6686        let allowed = match current.status {
6687            CommitOutcomeStatus::Conflict => matches!(
6688                input.resolution,
6689                CommitOutcomeResolution::ResolvedKeepServer | CommitOutcomeResolution::Superseded
6690            ),
6691            CommitOutcomeStatus::Rejected => {
6692                matches!(
6693                    input.resolution,
6694                    CommitOutcomeResolution::Superseded
6695                        | CommitOutcomeResolution::ResolvedKeepServer
6696                )
6697            }
6698            CommitOutcomeStatus::Applied | CommitOutcomeStatus::Cached => {
6699                input.resolution == CommitOutcomeResolution::Dismissed
6700            }
6701        };
6702        if !allowed {
6703            return Err(format!(
6704                "sync.invalid_request: resolution {:?} is invalid for {:?} outcome",
6705                input.resolution, current.status
6706            ));
6707        }
6708
6709        let rollback = self.capture();
6710        self.begin_observation("syncular_outcome_resolution")?;
6711        let result = (|| {
6712            self.conn
6713                .execute(
6714                    "UPDATE _syncular_commit_outcomes
6715                        SET resolution = ?1, resolved_at_ms = ?2,
6716                            replacement_client_commit_id = ?3
6717                      WHERE client_commit_id = ?4 AND resolution = 'active'",
6718                    rusqlite::params![
6719                        Self::outcome_resolution_name(input.resolution),
6720                        self.clock_now_ms(),
6721                        input.replacement_client_commit_id,
6722                        input.client_commit_id
6723                    ],
6724                )
6725                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6726            let resolved = self
6727                .commit_outcome(&current.client_commit_id)?
6728                .ok_or_else(|| "sync.outcome_not_found: outcome disappeared".to_owned())?;
6729            self.conn
6730                .execute(
6731                    "DELETE FROM _syncular_failed_commits WHERE client_commit_id = ?1",
6732                    rusqlite::params![input.client_commit_id],
6733                )
6734                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6735            for (commit, _) in self.failed_commits.iter() {
6736                if commit.client_commit_id == input.client_commit_id {
6737                    for operation in &commit.ops {
6738                        self.overlay_dirty.table(&operation.table);
6739                    }
6740                }
6741            }
6742            self.failed_commits
6743                .retain(|(commit, _)| commit.client_commit_id != input.client_commit_id);
6744            self.prune_commit_outcomes()?;
6745            Ok(resolved)
6746        })();
6747        let resolved = match result {
6748            Ok(outcome) => outcome,
6749            Err(error) => {
6750                self.rollback_observation("syncular_outcome_resolution");
6751                self.restore(rollback);
6752                return Err(error);
6753            }
6754        };
6755        self.conflicts
6756            .retain(|record| record.client_commit_id != current.client_commit_id);
6757        self.rejections
6758            .retain(|record| record.client_commit_id != current.client_commit_id);
6759        let mut batch = ChangeAccumulator {
6760            conflicts: current.status == CommitOutcomeStatus::Conflict,
6761            rejections: current.status == CommitOutcomeStatus::Rejected,
6762            outcomes: true,
6763            ..ChangeAccumulator::default()
6764        };
6765        if let Some(operations) = &current.operations {
6766            for operation in operations {
6767                batch.table(&operation.table);
6768            }
6769        }
6770        if let Err(error) = self
6771            .rebuild_overlay_if_dirty()
6772            .and_then(|()| self.finish_observation("syncular_outcome_resolution", batch))
6773        {
6774            self.rollback_observation("syncular_outcome_resolution");
6775            self.restore(rollback);
6776            return Err(error);
6777        }
6778        Ok(resolved)
6779    }
6780
6781    pub fn schema_floor(&self) -> Option<&SchemaFloor> {
6782        self.schema_floor.as_ref()
6783    }
6784
6785    /// §7.3.5: the client's opaque auth-lease state, if any.
6786    pub fn lease_state(&self) -> Option<&LeaseState> {
6787        self.lease_state.as_ref()
6788    }
6789
6790    /// §7.3.5: record a request-level lease error (stop-and-surface). Only
6791    /// the two lease codes set it; other errors leave leaseState untouched.
6792    fn record_lease_error(&mut self, code: &str) -> Result<(), String> {
6793        if code != "sync.auth_lease_required" && code != "sync.auth_lease_revoked" {
6794            return Ok(());
6795        }
6796        let mut next = self.lease_state.clone().unwrap_or_default();
6797        next.error_code = Some(code.to_owned());
6798        self.set_lease_state(Some(next))
6799    }
6800
6801    fn set_lease_state(&mut self, next: Option<LeaseState>) -> Result<(), String> {
6802        if self.lease_state == next {
6803            return Ok(());
6804        }
6805        self.begin_observation("syncular_lease")?;
6806        let rollback = self.capture();
6807        self.lease_state = next;
6808        let persisted = match &self.lease_state {
6809            Some(lease) => serde_json::to_string(lease)
6810                .map_err(|error| error.to_string())
6811                .and_then(|json| self.set_meta(LEASE_STATE_KEY, &json)),
6812            None => self.delete_meta(LEASE_STATE_KEY),
6813        };
6814        let batch = ChangeAccumulator {
6815            status: true,
6816            ..ChangeAccumulator::default()
6817        };
6818        if let Err(error) =
6819            persisted.and_then(|()| self.finish_observation("syncular_lease", batch))
6820        {
6821            self.rollback_observation("syncular_lease");
6822            self.restore(rollback);
6823            return Err(error);
6824        }
6825        Ok(())
6826    }
6827
6828    fn set_schema_floor(&mut self, next: Option<SchemaFloor>) -> Result<(), String> {
6829        if self.schema_floor == next {
6830            return Ok(());
6831        }
6832        self.begin_observation("syncular_schema_floor")?;
6833        let rollback = self.capture();
6834        self.stopped = next.is_some();
6835        self.schema_floor = next;
6836        let persisted = match &self.schema_floor {
6837            Some(floor) => serde_json::to_string(floor)
6838                .map_err(|error| error.to_string())
6839                .and_then(|json| self.set_meta(SCHEMA_FLOOR_KEY, &json)),
6840            None => self.delete_meta(SCHEMA_FLOOR_KEY),
6841        };
6842        let batch = ChangeAccumulator {
6843            status: true,
6844            ..ChangeAccumulator::default()
6845        };
6846        if let Err(error) =
6847            persisted.and_then(|()| self.finish_observation("syncular_schema_floor", batch))
6848        {
6849            self.rollback_observation("syncular_schema_floor");
6850            self.restore(rollback);
6851            return Err(error);
6852        }
6853        Ok(())
6854    }
6855
6856    fn set_upgrading(&mut self, value: bool) {
6857        if self.upgrading == value {
6858            return;
6859        }
6860        if self.begin_observation("syncular_upgrading").is_err() {
6861            return;
6862        }
6863        self.upgrading = value;
6864        let batch = ChangeAccumulator {
6865            status: true,
6866            ..ChangeAccumulator::default()
6867        };
6868        if self
6869            .finish_observation("syncular_upgrading", batch)
6870            .is_err()
6871        {
6872            self.rollback_observation("syncular_upgrading");
6873        }
6874    }
6875
6876    pub fn sync_needed(&self) -> bool {
6877        self.sync_needed
6878    }
6879
6880    pub fn read_rows(&self, table: &str) -> Result<Vec<RowState>, String> {
6881        let schema_table = self
6882            .schema
6883            .table(table)
6884            .ok_or_else(|| format!("unknown table {table:?}"))?;
6885        let sql = format!(
6886            "SELECT * FROM {} ORDER BY {} ASC",
6887            visible_table(table),
6888            quote_ident(&schema_table.primary_key)
6889        );
6890        let mut stmt = self
6891            .conn
6892            .prepare(&sql)
6893            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6894        let mut rows = stmt
6895            .query([])
6896            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6897        let mut out = Vec::new();
6898        while let Some(row) = rows
6899            .next()
6900            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
6901        {
6902            let mut values = Map::new();
6903            for (i, column) in schema_table.columns.iter().enumerate() {
6904                let value = row
6905                    .get_ref(i)
6906                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6907                values.insert(column.name.clone(), sql_ref_to_json(column, value));
6908            }
6909            let version: i64 = row
6910                .get(schema_table.columns.len())
6911                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6912            let row_id = match values.get(&schema_table.primary_key) {
6913                Some(Value::String(s)) => s.clone(),
6914                Some(Value::Number(n)) => n.to_string(),
6915                Some(Value::Bool(b)) => b.to_string(),
6916                other => format!("{}", other.cloned().unwrap_or(Value::Null)),
6917            };
6918            let sealed_key = if schema_table.has_encrypted_columns() {
6919                row.get(schema_table.columns.len() + 1)
6920                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
6921            } else {
6922                None
6923            };
6924            out.push(RowState {
6925                row_id,
6926                version,
6927                values,
6928                sealed_key,
6929            });
6930        }
6931        Ok(out)
6932    }
6933
6934    // -- §5.10.5 native CRDT (the `crdt-yjs` feature) --------------------------
6935    //
6936    // The Rust face of the §5.10.4 client model: a local crdt edit loads the
6937    // current stored (server-merged ⊕ pending-overlay) column bytes, applies
6938    // the op with `yrs`, re-encodes the whole doc state, and records a
6939    // baseVersion-less sparse upsert carrying only the primary key and the
6940    // crdt column (§5.10.3, §6.6). No local merge — merging is server-side;
6941    // the overlay re-materializes the edit immediately (optimistic apply,
6942    // §7.1) and the server-merged bytes arrive on the next pull, idempotently.
6943    // Byte-compatible with `@syncular/crdt-yjs`.
6944
6945    /// The current stored value of a `crdt` column for one row — the visible
6946    /// (optimistic) bytes, or `None` when the row is absent or the column is
6947    /// NULL (the empty document, §5.10.1). Errors if the column is not a
6948    /// `crdt` column (guards the app against a typo'd column name).
6949    #[cfg(feature = "crdt-yjs")]
6950    fn crdt_column_bytes(
6951        &self,
6952        table: &str,
6953        row_id: &str,
6954        column: &str,
6955    ) -> Result<Option<Vec<u8>>, String> {
6956        let schema_table = self
6957            .schema
6958            .table(table)
6959            .ok_or_else(|| format!("unknown table {table:?}"))?;
6960        let col = schema_table
6961            .columns
6962            .iter()
6963            .find(|c| c.name == column)
6964            .ok_or_else(|| format!("table {table:?} has no column {column:?}"))?;
6965        if col.ty != ColumnType::Crdt {
6966            return Err(format!("column {column:?} is not a crdt column (§5.10.1)"));
6967        }
6968        let sql = format!(
6969            "SELECT {} FROM {} WHERE {}",
6970            quote_ident(column),
6971            visible_table(table),
6972            row_id_predicate(schema_table)
6973        );
6974        let bytes: Option<Vec<u8>> = self
6975            .conn
6976            .prepare_cached(&sql)
6977            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
6978            .query_row(rusqlite::params![row_id], |row| {
6979                row.get::<_, Option<Vec<u8>>>(0)
6980            })
6981            .map_err(|e| match e {
6982                rusqlite::Error::QueryReturnedNoRows => "no such row".to_owned(),
6983                other => Self::sqlite_failure(&self.storage_failure, other),
6984            })?;
6985        Ok(bytes)
6986    }
6987
6988    /// §5.10.4 materialize: the collaborative text of a `crdt` column, decoded
6989    /// from the stored bytes with `yrs` — `YjsColumn.text(name).toString()`.
6990    /// An absent row / NULL column is the empty document (empty string).
6991    #[cfg(feature = "crdt-yjs")]
6992    pub fn crdt_text(
6993        &self,
6994        table: &str,
6995        row_id: &str,
6996        column: &str,
6997        name: &str,
6998    ) -> Result<String, String> {
6999        let bytes = self
7000            .crdt_column_bytes(table, row_id, column)?
7001            .unwrap_or_default();
7002        crate::crdt::text(&bytes, name)
7003    }
7004
7005    /// §5.10.4 push-an-update: apply a text insert to a `crdt` column and
7006    /// record the resulting crdt-only sparse operation. Returns the enqueued
7007    /// `clientCommitId`.
7008    #[cfg(feature = "crdt-yjs")]
7009    pub fn crdt_insert_text(
7010        &mut self,
7011        table: &str,
7012        row_id: &str,
7013        column: &str,
7014        name: &str,
7015        index: u32,
7016        value: &str,
7017    ) -> Result<String, ClientError> {
7018        let prior = self.storage_failure.borrow_mut().take();
7019        let outcome = (|| -> Result<String, ClientError> {
7020            let current = self
7021                .crdt_column_bytes(table, row_id, column)?
7022                .unwrap_or_default();
7023            let update = crate::crdt::insert_text(&current, name, index, value)?;
7024            self.perform_crdt_push_update(table, row_id, column, &update)
7025        })();
7026        self.finish_authoring(prior, outcome)
7027    }
7028
7029    /// §5.10.4 push-an-update: apply a text delete to a `crdt` column and
7030    /// record the resulting crdt-only sparse operation. Returns the enqueued
7031    /// `clientCommitId`.
7032    #[cfg(feature = "crdt-yjs")]
7033    pub fn crdt_delete_text(
7034        &mut self,
7035        table: &str,
7036        row_id: &str,
7037        column: &str,
7038        name: &str,
7039        index: u32,
7040        len: u32,
7041    ) -> Result<String, ClientError> {
7042        let prior = self.storage_failure.borrow_mut().take();
7043        let outcome = (|| -> Result<String, ClientError> {
7044            let current = self
7045                .crdt_column_bytes(table, row_id, column)?
7046                .unwrap_or_default();
7047            let update = crate::crdt::delete_text(&current, name, index, len)?;
7048            self.perform_crdt_push_update(table, row_id, column, &update)
7049        })();
7050        self.finish_authoring(prior, outcome)
7051    }
7052
7053    /// §5.10.4 generic escape hatch: apply an arbitrary Yjs update onto a
7054    /// `crdt` column's current state and record the resulting crdt-only sparse
7055    /// operation. The app authored the update with its own `yrs` model.
7056    /// Returns the enqueued `clientCommitId`.
7057    #[cfg(feature = "crdt-yjs")]
7058    pub fn crdt_apply_update(
7059        &mut self,
7060        table: &str,
7061        row_id: &str,
7062        column: &str,
7063        update: &[u8],
7064    ) -> Result<String, ClientError> {
7065        let prior = self.storage_failure.borrow_mut().take();
7066        let outcome = (|| -> Result<String, ClientError> {
7067            let current = self
7068                .crdt_column_bytes(table, row_id, column)?
7069                .unwrap_or_default();
7070            let next = crate::crdt::apply_update(&current, update)?;
7071            self.perform_crdt_push_update(table, row_id, column, &next)
7072        })();
7073        self.finish_authoring(prior, outcome)
7074    }
7075
7076    /// Shared tail of the crdt edit methods: record a §6.1 sparse crdt-only
7077    /// upsert — the primary key plus the crdt column, no `baseVersion` — and
7078    /// enqueue it. The server merges the crdt column and leaves every other
7079    /// column untouched (§5.10.3), so a baseVersion-less crdt edit cannot
7080    /// clobber a concurrent edit to another column. A locally absent row
7081    /// records the same partial operation; the server answers per §6.2.
7082    #[cfg(feature = "crdt-yjs")]
7083    fn perform_crdt_push_update(
7084        &mut self,
7085        table: &str,
7086        row_id: &str,
7087        column: &str,
7088        crdt_bytes: &[u8],
7089    ) -> Result<String, ClientError> {
7090        let mut bytes_obj = Map::new();
7091        bytes_obj.insert("$bytes".to_owned(), Value::from(bytes_to_hex(crdt_bytes)));
7092        let mut partial = Map::new();
7093        partial.insert(column.to_owned(), Value::Object(bytes_obj));
7094        self.perform_patch(table, row_id, partial, None)
7095    }
7096
7097    /// Run an arbitrary read-only SQL query against the local database and
7098    /// return each row as a `column-name → JSON value` map. This is the seam
7099    /// the React `useSyncQuery` live-query API needs (it takes app-authored
7100    /// SQL over the visible tables/views, not a fixed table read like
7101    /// [`read_rows`]).
7102    ///
7103    /// Bound `params` are the driver value forms: JSON strings/numbers/bools/
7104    /// null bind directly; a `{"$bytes": hex}` object binds as a BLOB — the
7105    /// same envelope the command surface uses everywhere else. Output BLOB
7106    /// columns come back as `{"$bytes": hex}` to round-trip cleanly.
7107    ///
7108    /// The result column typing is dynamic (SQLite's stored affinity), because
7109    /// arbitrary SQL can alias, join, and compute — there is no schema column
7110    /// to consult per output cell, unlike [`read_rows`].
7111    pub fn query(&self, sql: &str, params: &[QueryValue]) -> Result<Vec<QueryRow>, String> {
7112        query_connection(&self.conn, sql, params).map_err(|failure| failure.to_string())
7113    }
7114
7115    /// Repository benchmark and conformance access, present only with `bench-internals`.
7116    #[cfg(feature = "bench-internals")]
7117    #[doc(hidden)]
7118    pub fn benchmark_connection(&mut self) -> &mut Connection {
7119        &mut self.conn
7120    }
7121
7122    /// Private benchmark intervals; absent from ordinary builds and diagnostics.
7123    #[cfg(feature = "bench-internals")]
7124    #[doc(hidden)]
7125    pub fn benchmark_phases(
7126        &self,
7127        enabled: Option<bool>,
7128        reset: bool,
7129    ) -> Result<Option<Value>, String> {
7130        self.benchmark_phases.configure(enabled, reset)?;
7131        Ok(self.benchmark_phases.snapshot())
7132    }
7133
7134    /// One serialized, read-only authority observation, including durable coverage.
7135    pub fn authority_snapshot(&self) -> Result<Value, String> {
7136        validate_authority_reads(&self.schema, &self.limits.authority_reads)?;
7137        if self.limits.authority_reads.is_empty() {
7138            return Err("client.authority_read_forbidden: no authority reads were declared at client creation".into());
7139        }
7140        self.conn
7141            .execute_batch("SAVEPOINT authority_snapshot")
7142            .map_err(|e| e.to_string())?;
7143        let result = (|| {
7144            let revision = self
7145                .get_meta(LOCAL_REVISION_KEY)?
7146                .ok_or("sync.local_corrupt: authority revision is missing")?;
7147            let parsed_revision = revision
7148                .parse::<u64>()
7149                .map_err(|_| "sync.local_corrupt: authority revision is invalid")?;
7150            if parsed_revision.to_string() != revision {
7151                return Err("sync.local_corrupt: authority revision is invalid".into());
7152            }
7153            let mut tables = Vec::new();
7154            for read in &self.limits.authority_reads {
7155                let table = self
7156                    .schema
7157                    .table(&read.table)
7158                    .ok_or("client.authority_read_forbidden: authority table is not declared")?;
7159                let scalar = |column: &str, value: &Value| -> Result<String, String> {
7160                    let ty = table
7161                        .columns
7162                        .iter()
7163                        .find(|c| c.name == column)
7164                        .expect("validated authority column")
7165                        .ty;
7166                    if ty == ColumnType::Boolean {
7167                        if let Some(value) = value.as_bool() {
7168                            return Ok(value.to_string());
7169                        }
7170                        return match value.as_i64() {
7171                            Some(0) => Ok("false".into()),
7172                            Some(1) => Ok("true".into()),
7173                            _ => Err("sync.local_corrupt: authority boolean is invalid".into()),
7174                        };
7175                    }
7176                    match value {
7177                        Value::String(s) => Ok(s.clone()),
7178                        Value::Number(n) => Ok(n.to_string()),
7179                        Value::Object(o) if ty == ColumnType::Integer && o.len() == 1 => o
7180                            .get("$bigint")
7181                            .and_then(Value::as_str)
7182                            .map(str::to_owned)
7183                            .ok_or("sync.local_corrupt: authority integer is invalid".into()),
7184                        _ => {
7185                            Err("sync.local_corrupt: authority scalar is missing or invalid".into())
7186                        }
7187                    }
7188                };
7189                let matches = |row: &QueryRow| -> Result<bool, String> {
7190                    for (variable, values) in &read.scopes {
7191                        let column = table
7192                            .scope_column(variable)
7193                            .expect("validated authority scope");
7194                        let value = row
7195                            .get(column)
7196                            .ok_or("sync.local_corrupt: authority scope column is missing")?;
7197                        if value.is_null() || !values.contains(&scalar(column, value)?) {
7198                            return Ok(false);
7199                        }
7200                    }
7201                    Ok(true)
7202                };
7203                let selected = read
7204                    .columns
7205                    .iter()
7206                    .map(|c| quote_ident(c))
7207                    .collect::<Vec<_>>()
7208                    .join(",");
7209                let mut intent = BTreeSet::new();
7210                let mut relevant = BTreeSet::new();
7211                for row in query_connection(&self.conn,
7212                    "SELECT id,values_json FROM (SELECT json_extract(value,'$.table') AS tbl,json_extract(value,'$.rowId') AS id,json_extract(value,'$.values') AS values_json FROM _syncular_outbox,json_each(ops_json) UNION ALL SELECT json_extract(value,'$.table'),json_extract(value,'$.rowId'),json_extract(value,'$.values') FROM _syncular_failed_commits,json_each(operations_json) UNION ALL SELECT tbl,id,intent_json FROM _syncular_acked_rows) WHERE tbl=?",
7213                    &[Value::String(read.table.clone())]).map_err(|e| e.to_string())? {
7214                    let id = row.get("id").and_then(Value::as_str).ok_or("sync.local_corrupt: authority intent id is invalid")?.to_owned();
7215                    intent.insert(id.clone());
7216                    if let Some(raw) = row.get("values_json").and_then(Value::as_str) {
7217                        let values: QueryRow = serde_json::from_str(raw).map_err(|_| "sync.local_corrupt: authority intent values are invalid")?;
7218                        if read.scopes.keys().all(|v| values.contains_key(table.scope_column(v).expect("validated scope"))) && matches(&values)? {relevant.insert(id);}
7219                    }
7220                }
7221                let mut rows = BTreeMap::new();
7222                for full in [base_table(&read.table), visible_table(&read.table)] {
7223                    for mut values in query_connection(
7224                        &self.conn,
7225                        &format!(
7226                            "SELECT {selected},_syncular_version AS authority_version FROM {full}"
7227                        ),
7228                        &[],
7229                    )
7230                    .map_err(|e| e.to_string())?
7231                    {
7232                        if !matches(&values)? {
7233                            continue;
7234                        }
7235                        let id = scalar(
7236                            &table.primary_key,
7237                            values
7238                                .get(&table.primary_key)
7239                                .expect("selected primary key"),
7240                        )?;
7241                        relevant.insert(id.clone());
7242                        if full == base_table(&read.table) {
7243                            let version = values
7244                                .remove("authority_version")
7245                                .expect("selected version");
7246                            let version = version
7247                                .as_i64()
7248                                .ok_or("sync.local_corrupt: authority server version is invalid")?;
7249                            if version >= 0 {
7250                                rows.insert(id.clone(), json!({"values": values, "version": version, "hasLocalIntent": intent.contains(&id)}));
7251                            }
7252                        }
7253                    }
7254                }
7255                intent.retain(|id| relevant.contains(id));
7256                let rows = rows.into_values().collect::<Vec<_>>();
7257                // Every unit covering the table, as its (unit, table) pair (§4.4).
7258                let replica = self.stored(replica_cursor(&self.conn))?;
7259                let persisted = self
7260                    .stored(load_units(&self.conn))?
7261                    .into_iter()
7262                    .filter(|unit| covers(table, &unit.scopes))
7263                    .map(|unit| {
7264                        let effective = unit.echo.as_deref().and_then(|echo| {
7265                            pair_scopes(&unit.scopes, &read.table, Some(echo))
7266                        });
7267                        let requested = read
7268                            .scopes
7269                            .keys()
7270                            .map(|v| (v.clone(), json!(unit.scopes.get(v).into_iter().collect::<Vec<_>>())))
7271                            .collect::<Map<_, _>>();
7272                        let complete = effective
7273                            .as_ref()
7274                            .is_some_and(|scopes| scopes.len() == read.scopes.len())
7275                            && unit.status == UnitStatus::Active
7276                            && !unit.bootstrapping();
7277                        let exposed = effective.map(|scopes| {
7278                            read.scopes
7279                                .keys()
7280                                .map(|v| {
7281                                    let held = scopes.iter().find(|(variable, _)| variable == v);
7282                                    (v.clone(), json!(held.map_or(&Vec::new(), |(_, values)| values)))
7283                                })
7284                                .collect::<Map<_, _>>()
7285                        });
7286                        json!({"requestedScopes": requested, "status": unit.status, "cursor": unit.cursor.unwrap_or(replica), "effectiveScopes": exposed, "complete": complete})
7287                    })
7288                    .collect::<Vec<_>>();
7289                fn covered(selectors: &[(&String, &Vec<String>)], candidates: &[&Value]) -> bool {
7290                    match selectors.split_first() {
7291                        None => !candidates.is_empty(),
7292                        Some(((variable, values), rest)) => values.iter().all(|value| {
7293                            covered(
7294                                rest,
7295                                &candidates
7296                                    .iter()
7297                                    .copied()
7298                                    .filter(|p| {
7299                                        p["effectiveScopes"][variable.as_str()]
7300                                            .as_array()
7301                                            .is_some_and(|held| {
7302                                                held.contains(&Value::String(value.clone()))
7303                                                    || held.contains(&Value::String("*".into()))
7304                                            })
7305                                    })
7306                                    .collect::<Vec<_>>(),
7307                            )
7308                        }),
7309                    }
7310                }
7311                let coverage = if covered(
7312                    &read.scopes.iter().collect::<Vec<_>>(),
7313                    &persisted
7314                        .iter()
7315                        .filter(|p| p["complete"] == true)
7316                        .collect::<Vec<_>>(),
7317                ) {
7318                    "complete"
7319                } else if persisted.iter().any(|p| {
7320                    p["status"] == "active"
7321                        && read.scopes.iter().all(|(v, values)| {
7322                            values.iter().any(|value| {
7323                                p["requestedScopes"][v].as_array().is_some_and(|held| {
7324                                    held.contains(&Value::String(value.clone()))
7325                                })
7326                            })
7327                        })
7328                }) {
7329                    "pending"
7330                } else {
7331                    "missing"
7332                };
7333                tables.push(json!({"table": read.table, "rows": rows, "localIntentRowIds": intent, "scopes": read.scopes, "coverage": coverage, "persisted": persisted}));
7334            }
7335            Ok(
7336                json!({"revision": revision, "complete": tables.iter().all(|t| t["coverage"] == "complete"), "tables": tables}),
7337            )
7338        })();
7339        match result {
7340            Ok(snapshot) => {
7341                self.conn
7342                    .execute_batch("RELEASE authority_snapshot")
7343                    .map_err(|e| e.to_string())?;
7344                Ok(snapshot)
7345            }
7346            Err(error) => {
7347                self.conn
7348                    .execute_batch("ROLLBACK TO authority_snapshot; RELEASE authority_snapshot")
7349                    .map_err(|e| format!("{error}; rollback failed: {e}"))?;
7350                Err(error)
7351            }
7352        }
7353    }
7354
7355    /// Rows, coverage, and local revision from one SQLite read snapshot.
7356    /// §7.5 atomic snapshot read. An `owner` records a failure in
7357    /// `diagnostics_snapshot().query_failures` until its next successful read.
7358    pub fn query_snapshot(
7359        &mut self,
7360        sql: &str,
7361        params: &[QueryValue],
7362        coverage: &[WindowCoverage],
7363        owner: Option<QueryOwner<'_>>,
7364    ) -> Result<QuerySnapshot, String> {
7365        let result = snapshot_connection(&self.conn, sql, params, coverage);
7366        if let Some(owner) = owner {
7367            self.record_query_read(owner, result.as_ref().err());
7368        }
7369        result.map_err(|failure| failure.to_string())
7370    }
7371
7372    /// §7.6: record the outcome of one owned snapshot read, including reads a
7373    /// host ran on a separate connection. A success removes the owner's entry;
7374    /// an identical failure leaves it unchanged; a changed failure moves last.
7375    pub fn record_query_read(&mut self, owner: QueryOwner<'_>, failure: Option<&QueryReadFailure>) {
7376        let position = self
7377            .query_failures
7378            .iter()
7379            .position(|entry| entry.id == owner.id);
7380        let Some(failure) = failure else {
7381            if let Some(index) = position {
7382                self.query_failures.remove(index);
7383            }
7384            return;
7385        };
7386        let tables: Vec<String> = BTreeSet::from_iter(owner.tables.iter().copied())
7387            .into_iter()
7388            .map(str::to_owned)
7389            .collect();
7390        let code = failure.code.unwrap_or("client.query_failed").to_owned();
7391        if position.is_some_and(|index| {
7392            let entry = &self.query_failures[index];
7393            entry.tables == tables && entry.code == code && entry.sqlite_code == failure.sqlite_code
7394        }) {
7395            return;
7396        }
7397        if let Some(index) = position {
7398            self.query_failures.remove(index);
7399        }
7400        self.query_failures.push(DiagnosticQueryFailure {
7401            id: owner.id.to_owned(),
7402            tables,
7403            code,
7404            sqlite_code: failure.sqlite_code,
7405            at_ms: self.clock_now_ms(),
7406        });
7407        if self.query_failures.len() > MAX_DIAGNOSTIC_QUERY_FAILURES {
7408            self.query_failures.remove(0);
7409        }
7410    }
7411
7412    // -- request building ---------------------------------------------------------
7413
7414    fn build_request(&self, url_capable: bool) -> Result<(Message, RequestMeta), Box<SyncOutcome>> {
7415        #[cfg(feature = "bench-internals")]
7416        let _phase = self.benchmark_phases.start(Phase::RequestPrepare);
7417        let log_epoch = self.get_meta(LOG_EPOCH_KEY).map_err(|message| {
7418            Box::new(SyncOutcome::Failed {
7419                error_code: "storage.failed".into(),
7420                message,
7421                details: None,
7422                recommended_action: None,
7423            })
7424        })?;
7425        let max_commits = self.limits.max_push_commits_per_request;
7426        let max_ops = self
7427            .limits
7428            .max_push_operations_per_request
7429            .unwrap_or(PUSH_OPS_PER_REQUEST);
7430        let byte_cap = self.limits.max_push_request_bytes;
7431        // §4.2/§5.4: bit 3 is advertised iff the transport can fetch a
7432        // bare URL — capability negotiation, decided per transport.
7433        let accept = self.limits.accept.unwrap_or(if url_capable {
7434            DEFAULT_ACCEPT | ACCEPT_SIGNED_URLS
7435        } else {
7436            DEFAULT_ACCEPT
7437        });
7438        let storage_failed = |message: String| {
7439            Box::new(SyncOutcome::Failed {
7440                error_code: "storage.failed".into(),
7441                message,
7442                details: None,
7443                recommended_action: None,
7444            })
7445        };
7446        let header = Frame::ReqHeader {
7447            client_id: self.client_id.clone(),
7448            schema_version: self.schema.version,
7449            log_epoch: log_epoch.clone(),
7450            outbox_id: self.get_meta(OUTBOX_ID_KEY).map_err(storage_failed)?,
7451        };
7452        // §1.5: the lead frames are REQ_HEADER plus the optional ROUND_FAILURE,
7453        // which stays ahead of every push, pull, unit and progress frame; the
7454        // byte-capacity probes below measure this exact sequence.
7455        let mut lead = vec![header];
7456        lead.extend(
7457            self.round_failure
7458                .iter()
7459                .map(|code| Frame::RoundFailure { code: code.clone() }),
7460        );
7461        let registered: Vec<UnitRecord> = self
7462            .stored(load_units(&self.conn))
7463            .map_err(storage_failed)?
7464            .into_iter()
7465            .filter(|unit| unit.status == UnitStatus::Active)
7466            .collect();
7467        let registration_digest = ssp2::registration::registration_digest(
7468            registered
7469                .iter()
7470                .map(|unit| scope_pairs(&unit.scopes))
7471                .collect::<Vec<_>>()
7472                .iter()
7473                .map(Vec::as_slice),
7474        );
7475        let echo_digest = self
7476            .get_meta(ECHO_DIGEST_KEY)
7477            .map_err(storage_failed)?
7478            .filter(|hex| !hex.is_empty())
7479            .map(|hex| {
7480                crate::values::hex_to_bytes(&hex)
7481                    .ok()
7482                    .and_then(|bytes| <[u8; 32]>::try_from(bytes).ok())
7483                    .ok_or_else(|| {
7484                        "sync.local_corrupt: persisted echo digest is invalid".to_owned()
7485                    })
7486            })
7487            .transpose()
7488            .map_err(storage_failed)?;
7489        let pull = Frame::PullHeader {
7490            limit_commits: self.limits.limit_commits.unwrap_or(0),
7491            limit_snapshot_rows: self.limits.limit_snapshot_rows.unwrap_or(0),
7492            max_snapshot_pages: self.limits.max_snapshot_pages.unwrap_or(0),
7493            accept,
7494            cursor: self
7495                .stored(replica_cursor(&self.conn))
7496                .map_err(storage_failed)?,
7497            registration_digest,
7498            echo_digest,
7499        };
7500        let registration_digest = bytes_to_hex(&registration_digest);
7501        // §4.1: the list travels when it changed since the server stored it,
7502        // after `unregistered` and after a reset (both clear the stored digest).
7503        let travels = self
7504            .get_meta(REGISTERED_DIGEST_KEY)
7505            .map_err(storage_failed)?
7506            .as_deref()
7507            != Some(registration_digest.as_str());
7508        let mut sub_frames: Vec<Frame> = Vec::new();
7509        if travels {
7510            sub_frames.extend(registered.iter().map(|unit| Frame::Unit {
7511                scopes: scope_pairs(&unit.scopes),
7512            }));
7513        }
7514        // §4.3: one PROGRESS per catching-up unit, in registration order.
7515        sub_frames.extend(registered.iter().filter_map(|unit| {
7516            Some(Frame::Progress {
7517                id: unit.id.clone(),
7518                cursor: unit.cursor?,
7519                bootstrap_state: unit.bootstrap_state.clone().map(RawJson),
7520            })
7521        }));
7522        // The fixed frames (order-independent size) and the header+pull probe
7523        // both encode canonically, so the byte budget is the exact wire size.
7524        let fixed_bytes = match byte_cap {
7525            Some(_) => {
7526                let mut frames = lead.clone();
7527                frames.push(pull.clone());
7528                frames.extend(sub_frames.iter().cloned());
7529                encode_message(&Message {
7530                    wire_version: WIRE_VERSION,
7531                    msg_kind: MsgKind::Request,
7532                    frames,
7533                })
7534                .len()
7535            }
7536            None => 0,
7537        };
7538        if let Some(cap) = byte_cap {
7539            if fixed_bytes > cap {
7540                return Err(Self::push_capacity_outcome("bytes", cap, fixed_bytes, None));
7541            }
7542        }
7543        let base_bytes = match byte_cap {
7544            Some(_) => {
7545                let mut frames = lead.clone();
7546                frames.push(pull.clone());
7547                encode_message(&Message {
7548                    wire_version: WIRE_VERSION,
7549                    msg_kind: MsgKind::Request,
7550                    frames,
7551                })
7552                .len()
7553            }
7554            None => 0,
7555        };
7556        let total_outbox = if log_epoch.is_some() {
7557            self.outbox.len()
7558        } else {
7559            0
7560        };
7561        let mut push_frames: Vec<Frame> = Vec::new();
7562        let mut pushed_ids = Vec::new();
7563        let mut ops_in_request = 0usize;
7564        let mut bytes_in_request = 0usize;
7565        let mut deferred_commits = 0usize;
7566        #[cfg(feature = "bench-internals")]
7567        let outbox_phase = self.benchmark_phases.start(Phase::OutboxEncode);
7568        for (index, commit) in self.outbox.iter().take(total_outbox).enumerate() {
7569            // §6.1/§7.1: the commit-count and operation caps are hard. A
7570            // first commit over the operation cap is a typed capacity error
7571            // and stays queued; a later one defers with the complete suffix.
7572            if max_commits.is_some_and(|cap| push_frames.len() >= cap) {
7573                deferred_commits = total_outbox - index;
7574                break;
7575            }
7576            if ops_in_request + commit.ops.len() > max_ops {
7577                if push_frames.is_empty() {
7578                    return Err(Self::push_capacity_outcome(
7579                        "operations",
7580                        max_ops,
7581                        commit.ops.len(),
7582                        Some(commit.client_commit_id.as_str()),
7583                    ));
7584                }
7585                deferred_commits = total_outbox - index;
7586                break;
7587            }
7588            // §7.1 chained base: the commit waits for the change of the commit
7589            // it chains on, which supplies its rebased version.
7590            if self.outbox[..index]
7591                .iter()
7592                .flat_map(|earlier| &earlier.ops)
7593                .any(|earlier| commit.ops.iter().any(|op| chains_on(earlier, op)))
7594            {
7595                deferred_commits = total_outbox - index;
7596                break;
7597            }
7598            let operations = commit
7599                .ops
7600                .iter()
7601                .map(|op| {
7602                    let payload = match self
7603                        .schema
7604                        .table(&op.table)
7605                        .and_then(|table| Some((table, op.presented_values(table)?)))
7606                    {
7607                        Some((table, values)) => {
7608                            // §0: outbox entries encode at send time with the
7609                            // current codec (validated at mutate()). §6.1: the
7610                            // values map's keys are the sparse presence set; an
7611                            // insert-intent operation presents every column (§7.1).
7612                            // §5.11: encrypted columns are encrypted here, with
7613                            // an absent key-id selector read from the stored row.
7614                            // `sync_inner` runs `drop_unencodable_outbox` first,
7615                            // so an encrypt failure cannot reach this encode; the
7616                            // author seam validated every other failure.
7617                            let fallback =
7618                                self.stored_key_fallback(table, &op.row_id)
7619                                    .map_err(|message| {
7620                                        Box::new(SyncOutcome::Failed {
7621                                            error_code: "storage.failed".into(),
7622                                            message,
7623                                            details: None,
7624                                            recommended_action: None,
7625                                        })
7626                                    })?;
7627                            encode_sparse_row_json(
7628                                table,
7629                                &op.row_id,
7630                                &values,
7631                                &self.encryption,
7632                                fallback.as_ref(),
7633                            )
7634                            .ok()
7635                        }
7636                        None => None,
7637                    };
7638                    Ok(ssp2::model::Operation {
7639                        table: op.table.clone(),
7640                        row_id: op.row_id.clone(),
7641                        op: if op.upsert { Op::Upsert } else { Op::Delete },
7642                        base_version: op.base_version,
7643                        payload,
7644                    })
7645                })
7646                .collect::<Result<Vec<_>, Box<SyncOutcome>>>()?;
7647            // §6.3.2, §7.1: the commit's outbox sequence.
7648            let client_seq = self
7649                .conn
7650                .prepare_cached("SELECT seq FROM _syncular_outbox WHERE commit_id = ?1")
7651                .and_then(|mut statement| {
7652                    statement.query_row([&commit.client_commit_id], |row| row.get(0))
7653                })
7654                .map_err(|error| {
7655                    storage_failed(Self::sqlite_failure(&self.storage_failure, error))
7656                })?;
7657            let frame = Frame::PushCommit {
7658                client_commit_id: commit.client_commit_id.clone(),
7659                client_seq,
7660                operations,
7661            };
7662            let frame_bytes = match byte_cap {
7663                Some(_) => {
7664                    let mut frames = lead.clone();
7665                    frames.push(frame.clone());
7666                    frames.push(pull.clone());
7667                    encode_message(&Message {
7668                        wire_version: WIRE_VERSION,
7669                        msg_kind: MsgKind::Request,
7670                        frames,
7671                    })
7672                    .len()
7673                        - base_bytes
7674                }
7675                None => 0,
7676            };
7677            // The head is always encoded so the caller can report the full
7678            // projected request size; a later commit defers once it no longer
7679            // fits.
7680            if let Some(cap) = byte_cap {
7681                if !push_frames.is_empty() && fixed_bytes + bytes_in_request + frame_bytes > cap {
7682                    deferred_commits = total_outbox - index;
7683                    break;
7684                }
7685            }
7686            push_frames.push(frame);
7687            pushed_ids.push(commit.client_commit_id.clone());
7688            ops_in_request += commit.ops.len();
7689            bytes_in_request += frame_bytes;
7690        }
7691        #[cfg(feature = "bench-internals")]
7692        drop(outbox_phase);
7693        if let Some(cap) = byte_cap {
7694            if fixed_bytes + bytes_in_request > cap {
7695                // The head alone does not fit this budget; the commit stays
7696                // queued and no transport round is attempted.
7697                let blocked = if push_frames.is_empty() {
7698                    None
7699                } else {
7700                    self.outbox.first().map(|c| c.client_commit_id.as_str())
7701                };
7702                return Err(Self::push_capacity_outcome(
7703                    "bytes",
7704                    cap,
7705                    fixed_bytes + bytes_in_request,
7706                    blocked,
7707                ));
7708            }
7709        }
7710        let mut frames = lead;
7711        frames.extend(push_frames);
7712        frames.push(pull);
7713        frames.extend(sub_frames);
7714        let message = Message {
7715            wire_version: WIRE_VERSION,
7716            msg_kind: MsgKind::Request,
7717            frames,
7718        };
7719        Ok((
7720            message,
7721            RequestMeta {
7722                pushed_ids,
7723                registration_digest,
7724                accept,
7725                deferred_commits,
7726                applied: BTreeMap::new(),
7727                deferred: BTreeSet::new(),
7728            },
7729        ))
7730    }
7731
7732    /// §7.1: a configured push capacity limit cannot carry this request.
7733    fn push_capacity_outcome(
7734        kind: &str,
7735        limit: usize,
7736        size: usize,
7737        client_commit_id: Option<&str>,
7738    ) -> Box<SyncOutcome> {
7739        let mut details = Map::new();
7740        details.insert("kind".to_owned(), Value::from(kind));
7741        details.insert("limit".to_owned(), Value::from(limit as u64));
7742        details.insert("size".to_owned(), Value::from(size as u64));
7743        if let Some(id) = client_commit_id {
7744            details.insert("clientCommitId".to_owned(), Value::from(id));
7745        }
7746        Box::new(SyncOutcome::Failed {
7747            error_code: "client.push_request_too_large".into(),
7748            message: "the push request exceeds a configured capacity limit".into(),
7749            details: Some(Value::Object(details)),
7750            recommended_action: None,
7751        })
7752    }
7753
7754    // -- sync -------------------------------------------------------------------
7755
7756    /// Cloneable observation handle; no client lock is needed to read it.
7757    pub fn progress(&self) -> ProgressObserver {
7758        self.progress.clone()
7759    }
7760
7761    pub fn sync(&mut self, transport: &mut dyn Transport) -> SyncOutcome {
7762        self.sync_bounded(transport, None)
7763    }
7764
7765    /// One round; `idle_bound` is the idle drive's round bound (§7.8).
7766    fn sync_bounded(
7767        &mut self,
7768        transport: &mut dyn Transport,
7769        idle_bound: Option<(u64, u32)>,
7770    ) -> SyncOutcome {
7771        let mut prepared = match self.prepare_sync_round(transport.supports_url_fetch()) {
7772            Ok(prepared) => prepared,
7773            Err(outcome) => return *outcome,
7774        };
7775        prepared.idle_bound = idle_bound;
7776        // The round's network deadline rides `PreparedSyncRound` and is scoped
7777        // to each exchange, so uploads, continuations, the main request, and
7778        // every segment fetch share one monotonic anchor without leaving
7779        // deadline state on a reused transport.
7780        let mut next = prepared;
7781        loop {
7782            match self.apply_sync_round(next.exchange(transport)) {
7783                crate::AppliedSyncRound::Continue(prepared) => next = prepared,
7784                crate::AppliedSyncRound::Complete {
7785                    outcome, controls, ..
7786                } => {
7787                    if self.transport_enabled {
7788                        for text in controls {
7789                            let _ = transport.realtime_send(&text);
7790                        }
7791                    } else {
7792                        self.disconnect_realtime(transport);
7793                    }
7794                    return outcome;
7795                }
7796            }
7797        }
7798    }
7799
7800    fn finish_sync_round(&mut self, started_at_ms: i64, mut outcome: SyncOutcome) -> SyncOutcome {
7801        if let SyncOutcome::Failed {
7802            error_code,
7803            message,
7804            details,
7805            ..
7806        } = &mut outcome
7807        {
7808            if let Some((failure, code)) = self
7809                .storage_failure
7810                .borrow()
7811                .as_ref()
7812                .and_then(|failure| failure.code.map(|code| (failure, code)))
7813            {
7814                *error_code = code.to_owned();
7815                *message = match code {
7816                    "client.storage_busy" => "local SQLite storage is busy",
7817                    "client.storage_full" => "local SQLite storage is full",
7818                    "client.storage_io" => "local SQLite storage I/O failed",
7819                    "client.storage_corrupt" => "local SQLite storage is corrupt",
7820                    _ => unreachable!("unrecognized classified storage failure"),
7821                }
7822                .to_owned();
7823                *details = failure.details();
7824            }
7825        }
7826        let details = match &outcome {
7827            SyncOutcome::Failed { details, .. } => details.as_ref(),
7828            _ => None,
7829        };
7830        let cause_kind = details
7831            .and_then(|details| details.get("causeKind"))
7832            .and_then(Value::as_str)
7833            .and_then(|cause| match cause {
7834                "network" => Some(crate::TransportCauseKind::Network),
7835                "connect" => Some(crate::TransportCauseKind::Connect),
7836                "tls" => Some(crate::TransportCauseKind::Tls),
7837                "io" => Some(crate::TransportCauseKind::Io),
7838                "request" => Some(crate::TransportCauseKind::Request),
7839                "protocol" => Some(crate::TransportCauseKind::Protocol),
7840                "timeout" => Some(crate::TransportCauseKind::Timeout),
7841                "round_deadline" => Some(crate::TransportCauseKind::RoundDeadline),
7842                "aborted" => Some(crate::TransportCauseKind::Aborted),
7843                "body" => Some(crate::TransportCauseKind::Body),
7844                "status" => Some(crate::TransportCauseKind::Status),
7845                "redirect" => Some(crate::TransportCauseKind::Redirect),
7846                "unknown" => Some(crate::TransportCauseKind::Unknown),
7847                _ => None,
7848            });
7849        let http_status = if matches!(
7850            cause_kind,
7851            Some(
7852                crate::TransportCauseKind::Status
7853                    | crate::TransportCauseKind::Body
7854                    | crate::TransportCauseKind::Redirect
7855            )
7856        ) {
7857            details
7858                .and_then(|details| details.get("httpStatus"))
7859                .and_then(Value::as_f64)
7860                .filter(|status| status.fract() == 0.0 && (100.0..=599.0).contains(status))
7861                .map(|status| status as u16)
7862        } else {
7863            None
7864        };
7865        let retry_delay_ms = self.round_retry_delay_ms;
7866        self.progress.update(|p| match &outcome {
7867            SyncOutcome::Ok(report) | SyncOutcome::BudgetExhausted(report)
7868                if report.failed.is_empty() =>
7869            {
7870                p.state = ProgressState::Complete
7871            }
7872            SyncOutcome::Ok(_) | SyncOutcome::BudgetExhausted(_) => {
7873                p.state = ProgressState::Failed;
7874                p.error_code = Some("sync.scope_revoked".into());
7875            }
7876            SyncOutcome::Failed { error_code, .. } => {
7877                p.state = ProgressState::Failed;
7878                p.error_code = Some(Self::diagnostic_code(error_code));
7879                p.cause_kind = cause_kind;
7880                p.http_status = http_status;
7881                p.retry_delay_ms = retry_delay_ms;
7882            }
7883            SyncOutcome::RealtimeUnavailable { .. } => {
7884                p.state = ProgressState::Failed;
7885                p.error_code = Some(REALTIME_UNAVAILABLE_CODE.to_owned());
7886                p.retry_delay_ms = retry_delay_ms;
7887            }
7888        });
7889        let completed_at_ms = self.clock_now_ms();
7890        self.last_round = Some(match &outcome {
7891            SyncOutcome::Ok(report) | SyncOutcome::BudgetExhausted(report) => DiagnosticLastRound {
7892                status: "succeeded".to_owned(),
7893                started_at_ms,
7894                completed_at_ms,
7895                duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
7896                counters: Some(DiagnosticRoundCounters {
7897                    pushed: report.pushed,
7898                    applied: report.applied.len(),
7899                    rejected: report.rejected.len(),
7900                    retryable: report.retryable.len(),
7901                    conflicts: report.conflicts,
7902                    commits_applied: report.commits_applied,
7903                    segment_rows_applied: report.segment_rows_applied,
7904                    bootstrapping: report.bootstrapping.len(),
7905                    resets: report.resets.len(),
7906                    revoked: report.revoked.len(),
7907                    failed: report.failed.len(),
7908                    deferred_commits: report.deferred_commits,
7909                }),
7910                error_code: None,
7911                cause_kind: None,
7912                http_status: None,
7913            },
7914            SyncOutcome::Failed { error_code, .. } => DiagnosticLastRound {
7915                status: "failed".to_owned(),
7916                started_at_ms,
7917                completed_at_ms,
7918                duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
7919                counters: None,
7920                error_code: Some(Self::diagnostic_code(error_code)),
7921                cause_kind,
7922                http_status,
7923            },
7924            SyncOutcome::RealtimeUnavailable { .. } => DiagnosticLastRound {
7925                status: "failed".to_owned(),
7926                started_at_ms,
7927                completed_at_ms,
7928                duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
7929                counters: None,
7930                error_code: Some(REALTIME_UNAVAILABLE_CODE.to_owned()),
7931                cause_kind: None,
7932                http_status: None,
7933            },
7934        });
7935        outcome
7936    }
7937
7938    /// Capture one request on the client owner. Mutations made after this
7939    /// call remain in the outbox for a later round; no network I/O occurs here.
7940    pub fn prepare_sync_round(
7941        &mut self,
7942        url_capable: bool,
7943    ) -> Result<crate::PreparedSyncRound, Box<SyncOutcome>> {
7944        if !self.transport_enabled {
7945            return Err(Box::new(SyncOutcome::Failed {
7946                error_code: "sync.offline".into(),
7947                message: "transport gate is closed".into(),
7948                details: None,
7949                recommended_action: None,
7950            }));
7951        }
7952        if self.active_round.is_some() {
7953            return Err(Box::new(SyncOutcome::Failed {
7954                error_code: "client.round_in_progress".into(),
7955                message: "a sync round is already in flight".into(),
7956                details: None,
7957                recommended_action: None,
7958            }));
7959        }
7960        self.storage_failure.borrow_mut().take();
7961        self.progress.start();
7962        self.round_retry_delay_ms = None;
7963        let started_at_ms = self.clock_now_ms();
7964        let prepared = (|| {
7965            if self.stopped {
7966                return Err(Box::new(SyncOutcome::Ok(SyncReport {
7967                    schema_floor: self.schema_floor.clone(),
7968                    ..SyncReport::default()
7969                })));
7970            }
7971            if self.security_preflight {
7972                return Err(Box::new(SyncOutcome::Failed {
7973                    error_code: SECURITY_PREFLIGHT_REQUIRED_CODE.into(),
7974                    message: "security preflight must complete before syncing".into(),
7975                    details: None,
7976                    recommended_action: None,
7977                }));
7978            }
7979            self.set_sync_needed(false, false);
7980            self.drain_pending_evictions()
7981                .and_then(|_| self.drop_incompatible_outbox())
7982                .and_then(|_| self.drop_unencodable_outbox(true, None))
7983                .map_err(|message| {
7984                    Box::new(SyncOutcome::Failed {
7985                        error_code: "storage.failed".into(),
7986                        message,
7987                        details: None,
7988                        recommended_action: None,
7989                    })
7990                })?;
7991            // §5.11: reveal rows sealed under a key the keyring gained since.
7992            self.reveal_sealed_rows().map_err(|error| {
7993                Box::new(SyncOutcome::Failed {
7994                    error_code: error.code,
7995                    message: error.message,
7996                    details: error.details,
7997                    recommended_action: None,
7998                })
7999            })?;
8000            let has_log_epoch = self.get_meta(LOG_EPOCH_KEY).map_err(|message| {
8001                Box::new(SyncOutcome::Failed {
8002                    error_code: "storage.failed".into(),
8003                    message,
8004                    details: None,
8005                    recommended_action: None,
8006                })
8007            })?;
8008            let uploads = if has_log_epoch.is_some() && self.schema_has_blobs() {
8009                self.pending_blob_uploads().map_err(|e| {
8010                    Box::new(SyncOutcome::Failed {
8011                        error_code: e.code,
8012                        message: e.message,
8013                        details: e.details,
8014                        recommended_action: None,
8015                    })
8016                })?
8017            } else {
8018                Vec::new()
8019            };
8020            if self.realtime_state != RealtimeState::Connected
8021                && self.realtime_policy == RealtimePolicy::Required
8022            {
8023                let retry_delay_ms = self.schedule_background_retry();
8024                self.realtime_retry_delay_ms = Some(retry_delay_ms);
8025                return Err(Box::new(SyncOutcome::RealtimeUnavailable {
8026                    state: self.realtime_state,
8027                    reason_code: self.realtime_reason_code.clone(),
8028                    retry_delay_ms,
8029                }));
8030            }
8031            let (message, mut meta) = self.build_request(url_capable)?;
8032            let units = self.stored(load_units(&self.conn)).map_err(|message| {
8033                Box::new(SyncOutcome::Failed {
8034                    error_code: "storage.failed".into(),
8035                    message,
8036                    details: None,
8037                    recommended_action: None,
8038                })
8039            })?;
8040            for unit in &units {
8041                let applied =
8042                    self.stored(applied_rows(&self.conn, &unit.id))
8043                        .map_err(|message| {
8044                            Box::new(SyncOutcome::Failed {
8045                                error_code: "storage.failed".into(),
8046                                message,
8047                                details: None,
8048                                recommended_action: None,
8049                            })
8050                        })?;
8051                if !applied.is_empty() {
8052                    meta.applied.insert(unit.id.clone(), applied);
8053                }
8054            }
8055            let id = uuid::Uuid::new_v4();
8056            self.active_round = Some(id);
8057            let phase = if uploads.is_empty() {
8058                crate::round::RoundPhase::Sync
8059            } else {
8060                crate::round::RoundPhase::UploadGrant
8061            };
8062            Ok(crate::PreparedSyncRound {
8063                id,
8064                started_at_ms,
8065                message,
8066                meta,
8067                uploads: uploads.into(),
8068                phase,
8069                encoded: None,
8070                continuation: None,
8071                completed: None,
8072                realtime: self.realtime_state == RealtimeState::Connected,
8073                // §5.5: the owning unit's scope map, each value a one-element list.
8074                scopes: units
8075                    .into_iter()
8076                    .map(|unit| {
8077                        (
8078                            unit.id,
8079                            canonical_scope_json(&unit_scope_lists(&unit.scopes)),
8080                        )
8081                    })
8082                    .collect(),
8083                #[cfg(feature = "bench-internals")]
8084                benchmark_phases: self.benchmark_phases.clone(),
8085                progress: self.progress.clone(),
8086                fixed_now: self
8087                    .now_ms
8088                    .as_ref()
8089                    .map(|clock| clock.load(std::sync::atomic::Ordering::Relaxed)),
8090                round_deadline_at: None,
8091                idle_bound: None,
8092            })
8093        })();
8094        prepared.map_err(|outcome| Box::new(self.finish_sync_round(started_at_ms, *outcome)))
8095    }
8096
8097    /// Discard the reply of an old authorization or registration context. Its
8098    /// submitted commits stay durable and can be retried with the same ids.
8099    pub fn cancel_sync_round(&mut self) {
8100        self.active_round = None;
8101    }
8102
8103    /// Apply captured network results on the owner, preserving all normal
8104    /// version, schema, content-address and cursor checks. No network I/O.
8105    pub fn apply_sync_round(
8106        &mut self,
8107        completed: crate::CompletedSyncRound,
8108    ) -> crate::AppliedSyncRound {
8109        let mut prepared = completed.prepared;
8110        if self.active_round != Some(prepared.id) || self.security_preflight {
8111            return crate::AppliedSyncRound::Complete {
8112                outcome: SyncOutcome::Failed {
8113                    error_code: "client.round_cancelled".into(),
8114                    message: "sync round belongs to a released client context".into(),
8115                    details: None,
8116                    recommended_action: None,
8117                },
8118                controls: Vec::new(),
8119                more: false,
8120                bootstrap_advanced: false,
8121            };
8122        }
8123        self.storage_failure.borrow_mut().take();
8124        let (response, mut downloads, transport_failed) = match completed.exchange {
8125            crate::round::ExchangeResult::Upload { id, result } => {
8126                let result = match result {
8127                    // §5.9.7: a non-retryable server refusal (`blob.too_large`)
8128                    // is the body's final outcome. The commits naming it take
8129                    // the incompatible-commit path with the server's code and
8130                    // leave this round's request; the round continues.
8131                    Err(error)
8132                        if error
8133                            .details
8134                            .as_ref()
8135                            .and_then(|details| details.get("retryable"))
8136                            == Some(&Value::Bool(false)) =>
8137                    {
8138                        self.conn
8139                            .prepare(
8140                                "SELECT commit_id FROM _syncular_blob_commit_refs WHERE blob_id = ?",
8141                            )
8142                            .and_then(|mut statement| {
8143                                statement
8144                                    .query_map(rusqlite::params![id], |row| row.get::<_, String>(0))?
8145                                    .collect::<Result<HashSet<_>, _>>()
8146                            })
8147                            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
8148                            .and_then(|refused| {
8149                                let commits = self
8150                                    .outbox
8151                                    .iter()
8152                                    .filter(|commit| refused.contains(&commit.client_commit_id))
8153                                    .cloned()
8154                                    .collect::<Vec<_>>();
8155                                if !commits.is_empty() {
8156                                    self.drop_outbox_commits(&commits, &error.code, &error.message)?;
8157                                }
8158                                prepared.message.frames.retain(|frame| {
8159                                    !matches!(frame, Frame::PushCommit { client_commit_id, .. }
8160                                        if refused.contains(client_commit_id))
8161                                });
8162                                prepared.meta.pushed_ids.retain(|id| !refused.contains(id));
8163                                prepared.continuation = Some(crate::round::RoundPhase::UploadGrant);
8164                                Ok(())
8165                            })
8166                            .map_err(|message| TransportError::new("client.failed", message))
8167                    }
8168                    result => result,
8169                };
8170                let result = result.and_then(|()| {
8171                    self.conn
8172                        .execute(
8173                            "DELETE FROM _syncular_blob_uploads WHERE blob_id = ?",
8174                            rusqlite::params![id],
8175                        )
8176                        .map(|_| ())
8177                        .map_err(|error| {
8178                            TransportError::new(
8179                                "client.failed",
8180                                Self::sqlite_failure(&self.storage_failure, error),
8181                            )
8182                        })
8183                });
8184                match result {
8185                    Ok(()) => return crate::AppliedSyncRound::Continue(prepared),
8186                    Err(TransportError {
8187                        code,
8188                        message,
8189                        details,
8190                    }) => {
8191                        self.active_round = None;
8192                        if Self::unreachable_transport(&details) {
8193                            // The round's work is still owed; the host's
8194                            // connectivity resume runs it.
8195                            self.set_sync_needed(true, false);
8196                        } else if Self::retryable_failure_code(&code) {
8197                            self.schedule_background_retry();
8198                        }
8199                        let recommended_action = recommended_action_of(&details);
8200                        let outcome = self.finish_sync_round(
8201                            prepared.started_at_ms,
8202                            SyncOutcome::Failed {
8203                                error_code: code,
8204                                message,
8205                                details,
8206                                recommended_action,
8207                            },
8208                        );
8209                        return crate::AppliedSyncRound::Complete {
8210                            outcome,
8211                            controls: Vec::new(),
8212                            more: false,
8213                            bootstrap_advanced: false,
8214                        };
8215                    }
8216                }
8217            }
8218            crate::round::ExchangeResult::Reply {
8219                response,
8220                downloads,
8221                transport_failed,
8222            } => (response, downloads, transport_failed),
8223        };
8224        self.active_round = None;
8225        let before = self.resume_tokens();
8226        let outcome = (|| {
8227            let response = match response {
8228                Ok(response) => {
8229                    // §1.5: a complete, decodable response ends the report, an
8230                    // in-band ERROR included (the server already recorded it).
8231                    self.round_failure = None;
8232                    response
8233                }
8234                Err(TransportError {
8235                    code,
8236                    message,
8237                    details,
8238                }) => {
8239                    // §1.5: the server cannot see a round whose transport
8240                    // failed or whose response never decoded, so its code rides
8241                    // the following requests. A code the frame grammar rejects
8242                    // is not reportable, and the previous code stays; the
8243                    // client-local drive timeout never goes on the wire (§7.8).
8244                    if code != "sync.idle_timeout" && ssp2::model::valid_round_failure_code(&code) {
8245                        self.round_failure = Some(code.clone());
8246                    }
8247                    if transport_failed {
8248                        if prepared.realtime {
8249                            self.set_realtime_state(RealtimeState::Lost, Some(&code));
8250                        }
8251                        if let Err(message) = self.record_lease_error(&code) {
8252                            return SyncOutcome::Failed {
8253                                error_code: "storage.failed".to_owned(),
8254                                message,
8255                                details: None,
8256                                recommended_action: None,
8257                            };
8258                        }
8259                    }
8260                    if Self::retryable_failure_code(&code) && !Self::unreachable_transport(&details)
8261                    {
8262                        let delay = self.schedule_background_retry();
8263                        if prepared.realtime && transport_failed {
8264                            self.realtime_retry_delay_ms = Some(delay);
8265                        }
8266                    }
8267                    let recommended_action = recommended_action_of(&details);
8268                    return SyncOutcome::Failed {
8269                        error_code: code,
8270                        message,
8271                        details,
8272                        recommended_action,
8273                    };
8274                }
8275            };
8276            let mut outcome = self.process_response(&mut downloads, response, &prepared.meta);
8277            if let SyncOutcome::Ok(report) = &mut outcome {
8278                report.deferred_commits = prepared.meta.deferred_commits;
8279            }
8280            match &outcome {
8281                SyncOutcome::Ok(_) => self.reset_background_retry(),
8282                SyncOutcome::Failed {
8283                    error_code,
8284                    details,
8285                    ..
8286                } if Self::retryable_failure_code(error_code)
8287                    && !Self::unreachable_transport(details) =>
8288                {
8289                    self.schedule_background_retry();
8290                }
8291                _ => {}
8292            }
8293            if prepared.meta.deferred_commits > 0 {
8294                self.set_sync_needed(true, true);
8295            }
8296            self.previous_version_lifetime_check();
8297            outcome
8298        })();
8299        match &outcome {
8300            // §7.1: a unit still catching up needs another round, also when the
8301            // drive that started it ends early.
8302            SyncOutcome::Ok(report) if !report.bootstrapping.is_empty() => {
8303                self.set_sync_needed(true, false)
8304            }
8305            // The round's work is still owed; the host's connectivity resume
8306            // runs it.
8307            SyncOutcome::Failed { details, .. } if Self::unreachable_transport(details) => {
8308                self.set_sync_needed(true, false)
8309            }
8310            _ => {}
8311        }
8312        let more = self.transport_enabled
8313            && matches!(&outcome, SyncOutcome::Ok(report)
8314                if report.schema_floor.is_none()
8315                    && (!report.bootstrapping.is_empty()
8316                        || report.commits_applied > 0
8317                        || report.segment_rows_applied > 0
8318                        || !report.resets.is_empty()
8319                        || report.deferred_commits > 0
8320                        || self.sync_needed));
8321        let bootstrap_advanced = matches!(&outcome, SyncOutcome::Ok(report) if report.segment_rows_applied > 0 && !report.bootstrapping.is_empty())
8322            && before != self.resume_tokens();
8323        crate::AppliedSyncRound::Complete {
8324            outcome: self.finish_sync_round(prepared.started_at_ms, outcome),
8325            controls: if self.transport_enabled {
8326                downloads.controls
8327            } else {
8328                Vec::new()
8329            },
8330            more,
8331            bootstrap_advanced,
8332        }
8333    }
8334
8335    /// Every unit's resume token and applied-segment record: a bootstrap
8336    /// that advanced changes one (§4.7), also when it applied images of a
8337    /// section it deferred (§5.3, §5.6).
8338    #[allow(clippy::type_complexity)]
8339    fn resume_tokens(&self) -> Option<Vec<(String, Option<String>, Option<String>)>> {
8340        self.conn
8341            .prepare_cached(
8342                "SELECT id, bootstrap_state, applied FROM _syncular_units ORDER BY rowid",
8343            )
8344            .and_then(|mut stmt| {
8345                stmt.query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))?
8346                    .collect()
8347            })
8348            .ok()
8349    }
8350
8351    /// §7.8: one idle drive's default total deadline, the TS client's value.
8352    pub const SYNC_IDLE_TIMEOUT_MS: u64 = 60_000;
8353
8354    pub fn sync_until_idle(
8355        &mut self,
8356        transport: &mut dyn Transport,
8357        max_rounds: Option<u32>,
8358    ) -> SyncOutcome {
8359        self.sync_until_idle_with_deadline(transport, max_rounds, Some(Self::SYNC_IDLE_TIMEOUT_MS))
8360    }
8361
8362    /// As [`SyncClient::sync_until_idle`], with an explicit total deadline in
8363    /// milliseconds (`None` disables it). The deadline is checked against the
8364    /// client clock between rounds: a drive past it after a completed round
8365    /// ends as `BudgetExhausted`. Each round is bounded by `timeout_ms` on its
8366    /// own, or the transport's tighter whole-round deadline (§1.8); a round
8367    /// that reaches the drive bound fails with `sync.idle_timeout`.
8368    pub fn sync_until_idle_with_deadline(
8369        &mut self,
8370        transport: &mut dyn Transport,
8371        max_rounds: Option<u32>,
8372        timeout_ms: Option<u64>,
8373    ) -> SyncOutcome {
8374        // §7.8: an explicit budget is a strict positive round count. Zero is a
8375        // caller error, not an exhaustion.
8376        if max_rounds == Some(0) {
8377            return SyncOutcome::Failed {
8378                details: None,
8379                error_code: "sync.invalid_request".into(),
8380                message: "maxRounds must be a positive integer".into(),
8381                recommended_action: None,
8382            };
8383        }
8384        if timeout_ms.is_some_and(|ms| ms == 0 || i64::try_from(ms).is_err()) {
8385            return SyncOutcome::Failed {
8386                details: None,
8387                error_code: "sync.invalid_request".into(),
8388                message: "timeoutMs must be a positive integer".into(),
8389                recommended_action: None,
8390            };
8391        }
8392        let started_at_ms = self.clock_now_ms();
8393        let rounds = max_rounds.unwrap_or(20);
8394        let mut aggregate = SyncReport::default();
8395        let mut spent = 0;
8396        let mut completed = false;
8397        while spent < rounds {
8398            // §7.8: a deadline that elapsed across completed rounds ends the
8399            // drive like an exhausted round budget; the next drive resumes.
8400            if completed
8401                && timeout_ms.is_some_and(|ms| {
8402                    self.clock_now_ms() >= started_at_ms.saturating_add(ms as i64)
8403                })
8404            {
8405                return SyncOutcome::BudgetExhausted(aggregate);
8406            }
8407            let idle_bound = timeout_ms.map(|ms| (ms, spent));
8408            spent += 1;
8409            let before = self.resume_tokens();
8410            match self.sync_bounded(transport, idle_bound) {
8411                SyncOutcome::Failed {
8412                    error_code,
8413                    message,
8414                    details,
8415                    recommended_action,
8416                } => {
8417                    return SyncOutcome::Failed {
8418                        details,
8419                        error_code,
8420                        message,
8421                        recommended_action,
8422                    };
8423                }
8424                outcome @ SyncOutcome::RealtimeUnavailable { .. } => return outcome,
8425                SyncOutcome::BudgetExhausted(_) => {
8426                    unreachable!("sync() never exhausts a round budget")
8427                }
8428                SyncOutcome::Ok(report) => {
8429                    completed = true;
8430                    if max_rounds.is_none()
8431                        && report.segment_rows_applied > 0
8432                        && !report.bootstrapping.is_empty()
8433                        && before != self.resume_tokens()
8434                    {
8435                        spent = 0;
8436                    }
8437                    // TS parity: a disabled transport or a schema floor ends
8438                    // the loop. The latest round still merges first.
8439                    let stop = !self.transport_enabled() || report.schema_floor.is_some();
8440                    aggregate.merge(&report);
8441                    if stop {
8442                        return SyncOutcome::Ok(aggregate);
8443                    }
8444                    // §4.5: pull again whenever the response contained
8445                    // commits or segments; resets re-bootstrap; a pending
8446                    // resume token continues paging (§4.7); deferred outbox
8447                    // commits and a raised sync-needed signal push on the next
8448                    // round.
8449                    let more = !report.bootstrapping.is_empty()
8450                        || report.commits_applied > 0
8451                        || report.segment_rows_applied > 0
8452                        || !report.resets.is_empty()
8453                        || report.deferred_commits > 0
8454                        || self.sync_needed;
8455                    if !more {
8456                        return SyncOutcome::Ok(aggregate);
8457                    }
8458                }
8459            }
8460        }
8461        // §7.8: partial success. The aggregate report is retained and the run
8462        // is explicitly not idle.
8463        SyncOutcome::BudgetExhausted(aggregate)
8464    }
8465
8466    fn process_response(
8467        &mut self,
8468        transport: &mut dyn Transport,
8469        response: Message,
8470        meta: &RequestMeta,
8471    ) -> SyncOutcome {
8472        #[cfg(feature = "bench-internals")]
8473        let _phase = self.benchmark_phases.start(Phase::ResponseApply);
8474        let mut report = SyncReport {
8475            pushed: meta.pushed_ids.len() as u32,
8476            ..SyncReport::default()
8477        };
8478        let mut rejection_details_by_commit: HashMap<String, BTreeMap<i32, RejectionDetails>> =
8479            HashMap::new();
8480        let pushed_ids = meta
8481            .pushed_ids
8482            .iter()
8483            .map(String::as_str)
8484            .collect::<HashSet<_>>();
8485        let mut last_final_push_result_id: Option<String> = None;
8486        for frame in &response.frames {
8487            match frame {
8488                Frame::PushResultDetails {
8489                    client_commit_id,
8490                    entries,
8491                } => {
8492                    let details = rejection_details_by_commit
8493                        .entry(client_commit_id.clone())
8494                        .or_default();
8495                    for entry in entries {
8496                        let parsed = match RejectionDetails::parse(&entry.details.0) {
8497                            Ok(value) => value,
8498                            Err(message) => {
8499                                return SyncOutcome::Failed {
8500                                    details: None,
8501                                    error_code: "sync.invalid_request".to_owned(),
8502                                    message,
8503                                    recommended_action: None,
8504                                };
8505                            }
8506                        };
8507                        details.insert(entry.op_index, parsed);
8508                    }
8509                }
8510                Frame::PushResult {
8511                    client_commit_id,
8512                    status,
8513                    results,
8514                    ..
8515                } if pushed_ids.contains(client_commit_id.as_str())
8516                    && Self::push_result_is_final(*status, results) =>
8517                {
8518                    last_final_push_result_id = Some(client_commit_id.clone());
8519                }
8520                _ => {}
8521            }
8522        }
8523        let mut frames = response.frames.into_iter().peekable();
8524        match frames.next() {
8525            Some(Frame::RespHeader {
8526                required_schema_version,
8527                latest_schema_version,
8528                log_epoch,
8529                reset_required,
8530            }) => {
8531                if let Some(required) = required_schema_version {
8532                    // §1.6 schema-floor response: nothing else is processed;
8533                    // stop syncing and surface the upgrade requirement.
8534                    let floor = SchemaFloor {
8535                        required_schema_version: Some(required),
8536                        latest_schema_version,
8537                    };
8538                    if let Err(message) = self.set_schema_floor(Some(floor.clone())) {
8539                        return SyncOutcome::Failed {
8540                            details: None,
8541                            error_code: "storage.failed".to_owned(),
8542                            message,
8543                            recommended_action: None,
8544                        };
8545                    }
8546                    report.schema_floor = Some(floor);
8547                    return SyncOutcome::Ok(report);
8548                }
8549                if reset_required {
8550                    if frames.next().is_some() {
8551                        return SyncOutcome::Failed {
8552                            details: None,
8553                            error_code: "client.invalid_host_response".to_owned(),
8554                            message: "log-epoch reset response contains body frames".to_owned(),
8555                            recommended_action: None,
8556                        };
8557                    }
8558                    match self.run_log_epoch_reset(&log_epoch) {
8559                        Ok(resets) => {
8560                            report.resets = resets;
8561                            return SyncOutcome::Ok(report);
8562                        }
8563                        Err(message) => {
8564                            return SyncOutcome::Failed {
8565                                details: None,
8566                                error_code: "sync.local_corrupt".to_owned(),
8567                                message,
8568                                recommended_action: None,
8569                            };
8570                        }
8571                    }
8572                }
8573                let stored_epoch = match self.get_meta(LOG_EPOCH_KEY) {
8574                    Ok(stored) => stored,
8575                    Err(message) => {
8576                        return SyncOutcome::Failed {
8577                            details: None,
8578                            error_code: "storage.failed".to_owned(),
8579                            message,
8580                            recommended_action: None,
8581                        };
8582                    }
8583                };
8584                if stored_epoch.as_deref() != Some(log_epoch.as_str()) {
8585                    return SyncOutcome::Failed {
8586                        details: None,
8587                        error_code: "client.invalid_host_response".to_owned(),
8588                        message: "server changed logEpoch without requiring a reset".to_owned(),
8589                        recommended_action: None,
8590                    };
8591                }
8592            }
8593            _ => {
8594                return SyncOutcome::Failed {
8595                    details: None,
8596                    error_code: "sync.invalid_request".to_owned(),
8597                    message: "response does not start with RESP_HEADER".to_owned(),
8598                    recommended_action: None,
8599                };
8600            }
8601        }
8602
8603        if let Err(message) = self.begin_coalesced_apply() {
8604            return SyncOutcome::Failed {
8605                details: None,
8606                error_code: "storage.failed".into(),
8607                message,
8608                recommended_action: None,
8609            };
8610        }
8611        let mut failure: Option<(String, String, Option<String>, Option<Value>)> = None;
8612        // §4.5 client application: the units whose persisted echo a live
8613        // change must match, fixed at LIVE_START after every SUB_END persisted.
8614        let mut live: Option<(LiveStatus, String, Vec<UnitRecord>)> = None;
8615        let mut live_ended = false;
8616        while let Some(frame) = frames.next() {
8617            match frame {
8618                frame @ Frame::PushResult { .. } => {
8619                    let successful = matches!(
8620                        &frame,
8621                        Frame::PushResult {
8622                            status: PushStatus::Applied | PushStatus::Cached,
8623                            ..
8624                        }
8625                    );
8626                    let mut results = vec![frame];
8627                    if successful {
8628                        while matches!(
8629                            frames.peek(),
8630                            Some(Frame::PushResult {
8631                                status: PushStatus::Applied | PushStatus::Cached,
8632                                ..
8633                            })
8634                        ) {
8635                            if let Some(next) = frames.next() {
8636                                results.push(next);
8637                            }
8638                        }
8639                    }
8640                    if self
8641                        .handle_push_results(
8642                            &results,
8643                            &pushed_ids,
8644                            &rejection_details_by_commit,
8645                            &mut report,
8646                            last_final_push_result_id.as_deref(),
8647                        )
8648                        .is_err()
8649                    {
8650                        failure = Some((
8651                            "client.outcome_persistence_failed".to_owned(),
8652                            "local commit outcome could not be persisted".to_owned(),
8653                            None,
8654                            None,
8655                        ));
8656                        break;
8657                    }
8658                }
8659                Frame::PushResultDetails { .. } => {}
8660                Frame::SubStart {
8661                    id,
8662                    status,
8663                    reason_code,
8664                    bootstrap,
8665                    fenced,
8666                } => {
8667                    // §1.4 rule 5: an ERROR closes the section; its body still
8668                    // applied, its SUB_END values are never persisted.
8669                    let mut body = Vec::new();
8670                    let mut sub_end: Option<(i64, Option<String>)> = None;
8671                    let mut error: Option<Frame> = None;
8672                    for inner in frames.by_ref() {
8673                        match inner {
8674                            Frame::SubEnd {
8675                                next_cursor,
8676                                bootstrap_state,
8677                            } => {
8678                                sub_end = Some((next_cursor, bootstrap_state.map(|r| r.0)));
8679                                break;
8680                            }
8681                            inner @ Frame::Error { .. } => {
8682                                error = Some(inner);
8683                                break;
8684                            }
8685                            Frame::Unknown { .. } => {}
8686                            other => body.push(other),
8687                        }
8688                    }
8689                    let ended = sub_end.is_some();
8690                    let section = SectionStart {
8691                        id,
8692                        status,
8693                        reason_code,
8694                        bootstrap,
8695                        fenced,
8696                    };
8697                    match self.process_section(
8698                        transport,
8699                        &section,
8700                        body,
8701                        sub_end,
8702                        meta,
8703                        &mut report,
8704                    ) {
8705                        Err(SectionError::Abort(code, message)) => {
8706                            failure = Some((code, message, None, None));
8707                            break;
8708                        }
8709                        Err(SectionError::Transport(error)) => {
8710                            failure = Some((error.code, error.message, None, error.details));
8711                            break;
8712                        }
8713                        _ => {}
8714                    }
8715                    if ended {
8716                        // The counters stay: `complete` and `failed` report them.
8717                        self.progress.update(|p| {
8718                            p.unit_id = None;
8719                            p.table = None;
8720                            p.segment_id = None;
8721                        });
8722                    }
8723                    match error {
8724                        Some(Frame::Error {
8725                            code,
8726                            message,
8727                            recommended_action,
8728                            details,
8729                            ..
8730                        }) => {
8731                            failure = Some(Self::error_frame_failure(
8732                                code,
8733                                message,
8734                                recommended_action,
8735                                details,
8736                            ));
8737                            break;
8738                        }
8739                        Some(_) => unreachable!("only an ERROR closes a section early"),
8740                        None if !ended => {
8741                            failure = Some((
8742                                "sync.invalid_request".to_owned(),
8743                                "unit section without SUB_END".to_owned(),
8744                                None,
8745                                None,
8746                            ));
8747                            break;
8748                        }
8749                        None => {}
8750                    }
8751                }
8752                Frame::LiveStart {
8753                    status,
8754                    reason_code,
8755                } => {
8756                    let receivers = match self.stored(load_units(&self.conn)) {
8757                        Ok(units) => units
8758                            .into_iter()
8759                            .filter(|unit| {
8760                                unit.status == UnitStatus::Active
8761                                    && unit.echo.is_some()
8762                                    && !unit.bootstrapping()
8763                            })
8764                            .collect(),
8765                        Err(message) => {
8766                            failure = Some(("storage.failed".to_owned(), message, None, None));
8767                            break;
8768                        }
8769                    };
8770                    live = Some((status, reason_code, receivers));
8771                }
8772                Frame::Commit {
8773                    commit_seq,
8774                    tables,
8775                    changes,
8776                    ..
8777                } => {
8778                    let Some((LiveStatus::Active, _, receivers)) = &live else {
8779                        continue;
8780                    };
8781                    let receivers = receivers.clone();
8782                    match self.apply_live_commit(
8783                        &receivers,
8784                        &tables,
8785                        changes,
8786                        commit_seq,
8787                        AppliedServerCommitSource::Pull,
8788                    ) {
8789                        Ok(applied) => report.commits_applied += u32::from(applied),
8790                        Err(error) => {
8791                            failure = Some((error.code, error.message, None, error.details));
8792                            break;
8793                        }
8794                    }
8795                }
8796                Frame::LiveEnd {
8797                    through_seq,
8798                    echo_digest,
8799                } => {
8800                    let Some((status, reason_code, _)) = live.take() else {
8801                        continue;
8802                    };
8803                    live_ended = true;
8804                    live = Some((status, String::new(), Vec::new()));
8805                    if let Err(message) = self.finish_live_section(
8806                        status,
8807                        &reason_code,
8808                        through_seq,
8809                        echo_digest,
8810                        &meta.registration_digest,
8811                        &mut report,
8812                    ) {
8813                        failure = Some(("storage.failed".to_owned(), message, None, None));
8814                        break;
8815                    }
8816                }
8817                Frame::Lease {
8818                    lease_id,
8819                    expires_at_ms,
8820                } => {
8821                    // §7.3.5: persist the opaque lease; a fresh lease clears
8822                    // any prior lease error (the outage/revocation is over).
8823                    if let Err(message) = self.set_lease_state(Some(LeaseState {
8824                        lease_id: Some(lease_id),
8825                        expires_at_ms: Some(expires_at_ms),
8826                        error_code: None,
8827                    })) {
8828                        failure = Some(("storage.failed".to_owned(), message, None, None));
8829                        break;
8830                    }
8831                }
8832                Frame::Error {
8833                    code,
8834                    message,
8835                    recommended_action,
8836                    details,
8837                    ..
8838                } => {
8839                    // §1.6: the whole request failed; already-completed
8840                    // units keep their applied data and cursors.
8841                    failure = Some(Self::error_frame_failure(
8842                        code,
8843                        message,
8844                        recommended_action,
8845                        details,
8846                    ));
8847                    break;
8848                }
8849                Frame::Unknown { .. } => {}
8850                _ => {
8851                    failure = Some((
8852                        "sync.invalid_request".to_owned(),
8853                        "unexpected frame in response".to_owned(),
8854                        None,
8855                        None,
8856                    ));
8857                    break;
8858                }
8859            }
8860        }
8861
8862        let applied = match failure {
8863            Some(failure) => Err(failure),
8864            // Local apply boundaries already reconciled their affected overlay.
8865            // Refuse round success if any remaining dirty work fails to
8866            // reconcile. §4.8 E1: the push half may have drained commits that
8867            // pinned rows of a shrunk window unit, so retry deferred evictions.
8868            None => self
8869                .rebuild_overlay_if_dirty()
8870                .and_then(|()| self.drain_pending_evictions())
8871                .map_err(|message| ("storage.failed".to_owned(), message, None, None)),
8872        };
8873        // §1.4 rule 4: frames completed before a failure stay committed; a
8874        // failed COMMIT of the coalesced transaction fails the round.
8875        if let Err(message) = self.finish_coalesced_apply() {
8876            return SyncOutcome::Failed {
8877                details: None,
8878                error_code: "storage.failed".into(),
8879                message,
8880                recommended_action: None,
8881            };
8882        }
8883        if let Err((error_code, message, recommended_action, details)) = applied {
8884            return SyncOutcome::Failed {
8885                details,
8886                error_code,
8887                message,
8888                recommended_action,
8889            };
8890        }
8891        let catching_up = match self.stored(load_units(&self.conn)) {
8892            Ok(units) => units
8893                .into_iter()
8894                .filter(|unit| unit.status == UnitStatus::Active && unit.cursor.is_some())
8895                .map(|unit| unit.id)
8896                .collect::<Vec<_>>(),
8897            Err(message) => {
8898                return SyncOutcome::Failed {
8899                    details: None,
8900                    error_code: "storage.failed".into(),
8901                    message,
8902                    recommended_action: None,
8903                }
8904            }
8905        };
8906        // §7.1: a response without LIVE_END, one answered `unregistered` or
8907        // `reset`, or a unit still catching up leaves sync-needed raised;
8908        // after `unregistered` the next round starts at once.
8909        let status = live.map(|(status, _, _)| status);
8910        if !live_ended || status != Some(LiveStatus::Active) || !catching_up.is_empty() {
8911            self.set_sync_needed(true, status == Some(LiveStatus::Unregistered));
8912        }
8913        self.ack_after_pull(transport);
8914        // §7.4.5: the reset is over once the first post-reset pull round
8915        // leaves no unit catching up: the tables are rebuilt and current.
8916        if self.upgrading && catching_up.is_empty() {
8917            self.set_upgrading(false);
8918        }
8919        report.bootstrapping = catching_up;
8920        SyncOutcome::Ok(report)
8921    }
8922
8923    /// §1.6 / §10.1: the failure an in-band `ERROR` frame reports.
8924    fn error_frame_failure(
8925        code: String,
8926        message: String,
8927        recommended_action: String,
8928        details: Option<RawJson>,
8929    ) -> (String, String, Option<String>, Option<Value>) {
8930        let recommended_action = (!recommended_action.is_empty()).then_some(recommended_action);
8931        let details = details
8932            .and_then(|details| serde_json::from_str::<Value>(&details.0).ok())
8933            .filter(Value::is_object);
8934        (code, message, recommended_action, details)
8935    }
8936
8937    /// §4.5 at `LIVE_END` of a pull: join rule J for an `active` section, the
8938    /// §4.6 replica reset for `reset`, nothing for `unregistered`; then the
8939    /// replica cursor, the echo digest and the digest of the list the server
8940    /// now stores.
8941    fn finish_live_section(
8942        &mut self,
8943        status: LiveStatus,
8944        reason_code: &str,
8945        through_seq: i64,
8946        echo_digest: Option<[u8; 32]>,
8947        registration_digest: &str,
8948        report: &mut SyncReport,
8949    ) -> Result<(), String> {
8950        let rollback = self.capture();
8951        let resets = report.resets.len();
8952        self.begin_observation("syncular_live_end")?;
8953        let finished = (|| {
8954            let mut batch = ChangeAccumulator::default();
8955            let before = self.stored(replica_cursor(&self.conn))?;
8956            let mut cursor = if status == LiveStatus::Unregistered {
8957                before
8958            } else {
8959                -1
8960            };
8961            let joined = before.max(through_seq);
8962            for unit in self.stored(load_units(&self.conn))? {
8963                if unit.status != UnitStatus::Active || unit.bootstrapping() {
8964                    continue;
8965                }
8966                match status {
8967                    // §4.6 replica reset: every live unit bootstraps again;
8968                    // post-bootstrap units keep their cursors.
8969                    LiveStatus::Reset if unit.cursor.is_none() => {
8970                        for (base_key, value) in self.stored(window_refs(&self.conn, &unit.id))? {
8971                            batch.window(&base_key, &value);
8972                        }
8973                        report.resets.push(unit.id.clone());
8974                        self.stored(save_unit(
8975                            &self.conn,
8976                            &UnitRecord {
8977                                reason_code: Some(reason_code.to_owned()),
8978                                cursor: Some(-1),
8979                                seeded: Vec::new(),
8980                                ..unit
8981                            },
8982                        ))?;
8983                    }
8984                    // §4.5 join rule J.
8985                    LiveStatus::Active => {
8986                        let next = unit.cursor.unwrap_or(joined).max(through_seq);
8987                        if next >= joined {
8988                            cursor = joined;
8989                        }
8990                        if unit.cursor.is_some() {
8991                            self.stored(save_unit(
8992                                &self.conn,
8993                                &UnitRecord {
8994                                    cursor: (next < joined).then_some(next),
8995                                    ..unit
8996                                },
8997                            ))?;
8998                        }
8999                    }
9000                    _ => {}
9001                }
9002            }
9003            self.set_meta(REPLICA_CURSOR_KEY, &cursor.to_string())?;
9004            self.set_meta(
9005                ECHO_DIGEST_KEY,
9006                &echo_digest
9007                    .map(|digest| bytes_to_hex(&digest))
9008                    .unwrap_or_default(),
9009            )?;
9010            if status == LiveStatus::Unregistered {
9011                self.delete_meta(REGISTERED_DIGEST_KEY)?;
9012            } else {
9013                self.set_meta(REGISTERED_DIGEST_KEY, registration_digest)?;
9014            }
9015            self.finish_observation("syncular_live_end", batch)
9016        })();
9017        if finished.is_err() {
9018            self.rollback_observation("syncular_live_end");
9019            self.restore(rollback);
9020            report.resets.truncate(resets);
9021        }
9022        finished
9023    }
9024
9025    // -- push results (§6.3, §7.2) ------------------------------------------------
9026
9027    fn handle_push_results(
9028        &mut self,
9029        frames: &[Frame],
9030        pushed_ids: &HashSet<&str>,
9031        rejection_details_by_commit: &HashMap<String, BTreeMap<i32, RejectionDetails>>,
9032        report: &mut SyncReport,
9033        last_final_push_result_id: Option<&str>,
9034    ) -> Result<(), String> {
9035        self.begin_observation("syncular_push_results")?;
9036        let rollback = self.capture();
9037        let conflict_count = self.conflicts.len();
9038        let applied_count = report.applied.len();
9039        let rejected_count = report.rejected.len();
9040        let retryable_count = report.retryable.len();
9041        let mut drained = Vec::new();
9042        let mut batch = ChangeAccumulator::default();
9043        let persisted = (|| {
9044            for frame in frames {
9045                let Frame::PushResult {
9046                    client_commit_id,
9047                    status,
9048                    results,
9049                    commit_seq,
9050                } = frame
9051                else {
9052                    return Err("unexpected acknowledgement frame".to_owned());
9053                };
9054                if !pushed_ids.contains(client_commit_id.as_str()) {
9055                    continue;
9056                }
9057                let Some(index) = self
9058                    .outbox
9059                    .iter()
9060                    .position(|commit| commit.client_commit_id == *client_commit_id)
9061                else {
9062                    continue;
9063                };
9064                let status = *status;
9065                let rejection_details = rejection_details_by_commit.get(client_commit_id);
9066                let operations = self.outbox[index].ops.clone();
9067                match status {
9068                    PushStatus::Applied | PushStatus::Cached => {
9069                        // §7.2: a lost ack replays as `cached` — proceed as if the
9070                        // ack had arrived.
9071                        let sequence =
9072                            commit_seq.ok_or_else(|| "ACK lacks a commit sequence".to_owned())?;
9073                        let affected: HashSet<_> = operations
9074                            .iter()
9075                            .map(|op| (op.table.as_str(), op.row_id.as_str()))
9076                            .collect();
9077                        let prior = self.acknowledged_operations(Some(&affected), None)?;
9078                        for (op_index, operation) in operations.iter().enumerate() {
9079                            let delivered: bool = self.conn.query_row("SELECT EXISTS(SELECT 1 FROM _syncular_row_deliveries WHERE tbl=?1 AND id=?2 AND commit_seq>=?3)", rusqlite::params![operation.table, operation.row_id, sequence], |row| row.get(0))
9080                                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9081                            if delivered {
9082                                continue;
9083                            }
9084                            let table = self
9085                                .schema
9086                                .table(&operation.table)
9087                                .ok_or_else(|| "sync.unknown_table".to_owned())?;
9088                            let mut intended: Map<String, Value> = self
9089                                .stored_row(table, &operation.row_id, true)?
9090                                .map(|(row, _)| {
9091                                    table
9092                                        .columns
9093                                        .iter()
9094                                        .zip(&row)
9095                                        .map(|(column, value)| {
9096                                            (column.name.clone(), column_value_to_json(value))
9097                                        })
9098                                        .collect()
9099                                })
9100                                .unwrap_or_default();
9101                            let mut scope_image = intended.clone();
9102                            for op in prior
9103                                .iter()
9104                                .chain(
9105                                    self.failed_commits
9106                                        .iter()
9107                                        .flat_map(|(commit, _)| &commit.ops),
9108                                )
9109                                .chain(
9110                                    self.outbox
9111                                        .iter()
9112                                        .take(index)
9113                                        .flat_map(|commit| &commit.ops),
9114                                )
9115                                .chain(operations.iter().take(op_index + 1))
9116                            {
9117                                if op.table != operation.table || op.row_id != operation.row_id {
9118                                    continue;
9119                                }
9120                                if let Some(values) = &op.values {
9121                                    intended.extend(values.clone());
9122                                } else {
9123                                    scope_image = intended.clone();
9124                                    intended.clear();
9125                                }
9126                            }
9127                            self.conn.execute("INSERT INTO _syncular_acked_rows(commit_id,idx,tbl,id,commit_seq,op_json,intent_json) VALUES (?1,?2,?3,?4,?5,?6,?7)",
9128                                rusqlite::params![client_commit_id, op_index as i64, operation.table, operation.row_id, sequence,
9129                                    serde_json::to_string(&CommitOperation::from(operation)).map_err(|_| "sync.local_corrupt".to_owned())?,
9130                                    serde_json::to_string(if operation.upsert { &intended } else { &scope_image }).map_err(|_| "sync.local_corrupt".to_owned())?])
9131                                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9132                        }
9133                        let journal_results = results
9134                            .iter()
9135                            .map(|result| {
9136                                let op_index = match result {
9137                                    OpResult::Applied { op_index }
9138                                    | OpResult::Conflict { op_index, .. }
9139                                    | OpResult::Error { op_index, .. } => *op_index,
9140                                };
9141                                CommitOperationOutcome::Applied { op_index }
9142                            })
9143                            .collect::<Vec<_>>();
9144                        let outcome_status = if status == PushStatus::Applied {
9145                            CommitOutcomeStatus::Applied
9146                        } else {
9147                            CommitOutcomeStatus::Cached
9148                        };
9149                        let persisted = self
9150                            .persist_commit_outcome(
9151                                client_commit_id,
9152                                outcome_status,
9153                                &journal_results,
9154                                None,
9155                            )
9156                            .and_then(|()| self.delete_outbox_persisted(client_commit_id));
9157                        persisted?;
9158                        self.sync_needed = true;
9159                        batch.status = true;
9160                        batch.outcomes = true;
9161                    }
9162                    PushStatus::Rejected => {
9163                        if results.iter().any(|result| {
9164                            matches!(
9165                                result,
9166                                OpResult::Error {
9167                                    code,
9168                                    retryable: true,
9169                                    ..
9170                                } if code == "sync.idempotency_cache_miss"
9171                            )
9172                        }) {
9173                            // §6.3/§7.2: a serving failure, not an outcome — keep the
9174                            // exact commit queued for an identical retry.
9175                            report.retryable.push(client_commit_id.to_owned());
9176                            continue;
9177                        }
9178
9179                        let mut journal_results = Vec::with_capacity(results.len());
9180                        let mut conflicts = Vec::new();
9181                        let mut rejections = Vec::new();
9182                        for result in results {
9183                            match result {
9184                                OpResult::Applied { op_index } => {
9185                                    journal_results.push(CommitOperationOutcome::Applied {
9186                                        op_index: *op_index,
9187                                    });
9188                                }
9189                                OpResult::Conflict {
9190                                    op_index,
9191                                    code,
9192                                    message,
9193                                    server_version,
9194                                    server_row,
9195                                    conflict_columns: conflict_columns_bitmap,
9196                                } => {
9197                                    let operation = operations
9198                                        .get(*op_index as usize)
9199                                        .map(CommitOperation::from);
9200                                    let (table, row_id) = operation
9201                                        .as_ref()
9202                                        .map(|op| (op.table.clone(), op.row_id.clone()))
9203                                        .unwrap_or_default();
9204                                    let server_row_json = self
9205                                        .schema
9206                                        .table(&table)
9207                                        .and_then(|t| {
9208                                            decode_row_bytes(t, server_row, &self.encryption)
9209                                                .ok()
9210                                                .map(|(row, _)| (t, row))
9211                                        })
9212                                        .map(|(t, row)| {
9213                                            let mut map = Map::new();
9214                                            for (i, column) in t.columns.iter().enumerate() {
9215                                                map.insert(
9216                                                    column.name.clone(),
9217                                                    column_value_to_json(
9218                                                        row.get(i).unwrap_or(&None),
9219                                                    ),
9220                                                );
9221                                            }
9222                                            map
9223                                        })
9224                                        .unwrap_or_default();
9225                                    // §6.3: decode the presence-layout bitmap
9226                                    // into the marked column names.
9227                                    let conflict_columns = self
9228                                        .schema
9229                                        .table(&table)
9230                                        .map(|t| {
9231                                            t.columns
9232                                                .iter()
9233                                                .enumerate()
9234                                                .filter(|(index, _)| {
9235                                                    conflict_columns_bitmap
9236                                                        .get(index / 8)
9237                                                        .is_some_and(|byte| {
9238                                                            byte & (1 << (index % 8)) != 0
9239                                                        })
9240                                                })
9241                                                .map(|(_, column)| column.name.clone())
9242                                                .collect::<Vec<_>>()
9243                                        })
9244                                        .unwrap_or_default();
9245                                    let conflict = ConflictRecord {
9246                                        client_commit_id: client_commit_id.to_owned(),
9247                                        op_index: *op_index,
9248                                        table,
9249                                        row_id,
9250                                        code: code.clone(),
9251                                        message: message.clone(),
9252                                        server_version: *server_version,
9253                                        server_row: server_row_json,
9254                                        conflict_columns,
9255                                        operation,
9256                                    };
9257                                    journal_results.push(CommitOperationOutcome::Conflict {
9258                                        conflict: conflict.clone(),
9259                                    });
9260                                    conflicts.push(conflict);
9261                                }
9262                                OpResult::Error {
9263                                    op_index,
9264                                    code,
9265                                    message,
9266                                    retryable,
9267                                } => {
9268                                    let rejection = RejectionRecord {
9269                                        client_commit_id: client_commit_id.to_owned(),
9270                                        op_index: *op_index,
9271                                        code: code.clone(),
9272                                        message: message.clone(),
9273                                        retryable: *retryable,
9274                                        details: rejection_details
9275                                            .and_then(|details| details.get(op_index))
9276                                            .cloned(),
9277                                        operation: operations
9278                                            .get(*op_index as usize)
9279                                            .map(CommitOperation::from),
9280                                    };
9281                                    journal_results.push(CommitOperationOutcome::Error {
9282                                        rejection: rejection.clone(),
9283                                    });
9284                                    rejections.push(rejection);
9285                                }
9286                            }
9287                        }
9288                        if self.retain_failed_commits {
9289                            self.retain_failed_commit(client_commit_id, &operations)?;
9290                        }
9291                        let outcome_status = if conflicts.is_empty() {
9292                            CommitOutcomeStatus::Rejected
9293                        } else {
9294                            CommitOutcomeStatus::Conflict
9295                        };
9296                        let persisted = self
9297                            .persist_commit_outcome(
9298                                client_commit_id,
9299                                outcome_status,
9300                                &journal_results,
9301                                Some(&operations),
9302                            )
9303                            .and_then(|()| self.delete_outbox_persisted(client_commit_id));
9304                        persisted?;
9305                        batch.conflicts = !conflicts.is_empty();
9306                        batch.rejections = !rejections.is_empty();
9307                        batch.status = true;
9308                        batch.outcomes = true;
9309                        self.conflicts.extend(conflicts);
9310                        self.rejections.extend(rejections);
9311                    }
9312                }
9313                drained.push(self.outbox.remove(index));
9314                if batch.status {
9315                    for operation in &operations {
9316                        if !self.record_row_scopes(
9317                            &mut batch,
9318                            &operation.table,
9319                            &operation.row_id,
9320                            false,
9321                        ) {
9322                            batch.table(&operation.table);
9323                        }
9324                    }
9325                }
9326                match status {
9327                    PushStatus::Applied | PushStatus::Cached => {
9328                        report.applied.push(client_commit_id.to_owned())
9329                    }
9330                    PushStatus::Rejected => report.rejected.push(client_commit_id.to_owned()),
9331                }
9332            }
9333            if batch.outcomes && frames.iter().any(|frame| matches!(frame, Frame::PushResult { client_commit_id, .. } if Some(client_commit_id.as_str()) == last_final_push_result_id)) {
9334                self.prune_commit_outcomes()?;
9335            }
9336            self.reconcile_rows(&drained)?;
9337            self.rebuild_overlay_if_dirty()?;
9338            self.finish_observation("syncular_push_results", batch)
9339        })();
9340        if let Err(error) = persisted {
9341            self.rollback_observation("syncular_push_results");
9342            self.restore(rollback);
9343            report.applied.truncate(applied_count);
9344            report.rejected.truncate(rejected_count);
9345            report.retryable.truncate(retryable_count);
9346            return Err(error);
9347        }
9348        if !rollback.sync_needed && self.sync_needed {
9349            self.sync_intent_queue.push_back(SyncIntent::Interactive);
9350        }
9351        report.conflicts += (self.conflicts.len() - conflict_count) as u32;
9352        Ok(())
9353    }
9354
9355    fn push_result_is_final(status: PushStatus, results: &[OpResult]) -> bool {
9356        status != PushStatus::Rejected
9357            || !results.iter().any(|result| {
9358                matches!(
9359                    result,
9360                    OpResult::Error {
9361                        code,
9362                        retryable: true,
9363                        ..
9364                    } if code == "sync.idempotency_cache_miss"
9365                )
9366            })
9367    }
9368
9369    // -- unit sections and the live section ------------------------------------------
9370
9371    /// One unit section (§4.4): apply its segments, then persist its
9372    /// `SUB_END` (§4.6 reset, §3.3 revocation and purge, the echo, cursor and
9373    /// resume token). A section of a unit the app released while the round was
9374    /// in flight is skipped; a fail-closed first page fails the unit and skips
9375    /// the rest of its section (§5.6).
9376    fn process_section(
9377        &mut self,
9378        transport: &mut dyn Transport,
9379        start: &SectionStart,
9380        body: Vec<Frame>,
9381        sub_end: Option<(i64, Option<String>)>,
9382        meta: &RequestMeta,
9383        report: &mut SyncReport,
9384    ) -> Result<(), SectionError> {
9385        let storage = |message| SectionError::Abort("storage.failed".to_owned(), message);
9386        let unit = self
9387            .stored(get_unit(&self.conn, &start.id))
9388            .map_err(storage)?
9389            .filter(|unit| unit.status == UnitStatus::Active);
9390        // §7.7: a section boundary ends the previous payload.
9391        self.progress.update(|p| {
9392            p.phase = ProgressPhase::Import;
9393            p.unit_id = unit.as_ref().map(|unit| unit.id.clone());
9394            p.table = None;
9395            p.segment_id = None;
9396            p.bytes_received = 0;
9397            p.bytes_total = None;
9398            p.rows_processed = 0;
9399            p.rows_total = None;
9400            p.import_write_ms = None;
9401            p.import_commit_ms = None;
9402        });
9403        let Some(unit) = unit else {
9404            return Ok(());
9405        };
9406        let echo: Vec<EchoRecord> = start
9407            .fenced
9408            .iter()
9409            .map(|record| EchoRecord {
9410                table: record.table.clone(),
9411                status: record.status.name().to_owned(),
9412                scopes: record.scopes.iter().cloned().collect(),
9413            })
9414            .collect();
9415        // §5.6: rows earlier attempts at this position applied.
9416        let applied_record = self
9417            .stored(applied_rows(&self.conn, &unit.id))
9418            .map_err(storage)?;
9419        // §5.6: a resumed bootstrap keeps the tables it already seeded.
9420        let mut seeded: BTreeSet<String> = if unit.bootstrap_state.is_none() {
9421            BTreeSet::new()
9422        } else {
9423            unit.seeded.iter().cloned().collect()
9424        };
9425        let applied = (|| {
9426            for frame in body {
9427                match frame {
9428                    Frame::SegmentInline { payload } => {
9429                        let segment = decode_rows_segment(&payload).map_err(|e| {
9430                            SectionError::Abort(e.code.as_str().to_owned(), e.detail)
9431                        })?;
9432                        self.progress.update(|p| {
9433                            p.phase = ProgressPhase::Import;
9434                            p.unit_id = Some(unit.id.clone());
9435                            p.table = Some(segment.table.clone());
9436                            p.segment_id = None;
9437                            p.bytes_received = payload.len() as u64;
9438                            p.bytes_total = Some(payload.len() as u64);
9439                            p.rows_processed = 0;
9440                            p.rows_total =
9441                                Some(segment.blocks.iter().map(|block| block.len() as u64).sum());
9442                            p.import_write_ms = None;
9443                            p.import_commit_ms = None;
9444                        });
9445                        let (pair, clear, effective) =
9446                            self.segment_pair(&unit, &echo, &seeded, &segment.table)?;
9447                        report.segment_rows_applied +=
9448                            self.apply_segment(&pair, &segment, clear, &effective, &unit.id)?;
9449                        seeded.insert(segment.table);
9450                    }
9451                    Frame::SegmentRef {
9452                        segment_id,
9453                        byte_length,
9454                        media_type,
9455                        table,
9456                        row_count,
9457                        as_of_commit_seq,
9458                        scope_digest,
9459                        row_cursor,
9460                        next_row_cursor,
9461                        url,
9462                        url_expires_at_ms,
9463                        ..
9464                    } => {
9465                        // §4.2: reject a descriptor whose mediaType was not
9466                        // advertised — never skip or guess.
9467                        let advertised = match media_type {
9468                            MediaType::Rows => {
9469                                meta.accept & ACCEPT_EXTERNAL_ROWS != 0
9470                                    || meta.accept & ACCEPT_INLINE_ROWS != 0
9471                            }
9472                            MediaType::Sqlite => meta.accept & ACCEPT_SQLITE != 0,
9473                        };
9474                        if !advertised {
9475                            return Err(SectionError::Abort(
9476                                "sync.invalid_request".to_owned(),
9477                                format!(
9478                                    "SEGMENT_REF mediaType {} was not advertised in accept (§4.2)",
9479                                    media_type.name()
9480                                ),
9481                            ));
9482                        }
9483                        // §5.6 applied-segment rule: a partly applied image
9484                        // already ran its table's first-page clear.
9485                        let resumed = applied_record.get(&segment_id).copied();
9486                        if resumed.is_some() {
9487                            seeded.insert(table.clone());
9488                        }
9489                        if resumed == Some(row_count) {
9490                            continue;
9491                        }
9492                        if meta
9493                            .deferred
9494                            .contains(&(unit.id.clone(), segment_id.clone()))
9495                        {
9496                            return Err(SectionError::Deferred);
9497                        }
9498                        self.progress.update(|p| {
9499                            p.phase = ProgressPhase::Download;
9500                            p.unit_id = Some(unit.id.clone());
9501                            p.table = Some(table.clone());
9502                            p.segment_id = Some(segment_id.clone());
9503                            p.bytes_received = 0;
9504                            p.bytes_total = u64::try_from(byte_length).ok();
9505                            p.rows_processed = 0;
9506                            p.rows_total = u64::try_from(row_count).ok();
9507                            p.import_write_ms = None;
9508                            p.import_commit_ms = None;
9509                        });
9510                        let observer = self.progress.clone();
9511                        let mut on_progress = |bytes| observer.update(|p| p.bytes_received = bytes);
9512                        let bytes = if let Some(url) = url {
9513                            // §5.4: a url-carrying descriptor MUST be fetched
9514                            // from that URL; failure invalidates the whole
9515                            // descriptor (no fall-through to §5.5 — re-pull
9516                            // recovers, §1.4 rule 5).
9517                            if meta.accept & ACCEPT_SIGNED_URLS == 0 {
9518                                return Err(SectionError::Abort(
9519                                    "sync.invalid_request".to_owned(),
9520                                    "SEGMENT_REF carries a url but accept bit 3 was not advertised (§5.4)"
9521                                        .to_owned(),
9522                                ));
9523                            }
9524                            // §5.4: MUST NOT start a fetch at/past expiry.
9525                            if url_expires_at_ms.is_some_and(|exp| {
9526                                exp <= transport
9527                                    .segment_fetch_started_at(&url)
9528                                    .unwrap_or_else(|| self.clock_now_ms())
9529                            }) {
9530                                return Err(SectionError::Abort(
9531                                    "sync.segment_expired".to_owned(),
9532                                    format!(
9533                                        "signed URL for segment {segment_id} expired before fetch — re-pull mints fresh descriptors (§5.4)"
9534                                    ),
9535                                ));
9536                            }
9537                            transport
9538                                .fetch_url(
9539                                    &url,
9540                                    u64::try_from(byte_length).unwrap_or(0),
9541                                    &mut on_progress,
9542                                )
9543                                .map_err(|e| self.segment_fetch_failed(e))?
9544                        } else {
9545                            // §5.5: the owning unit's scope map, each value a one-element list.
9546                            let requested_scopes_json =
9547                                canonical_scope_json(&unit_scope_lists(&unit.scopes));
9548                            transport
9549                                .download_segment(
9550                                    &SegmentRequest {
9551                                        segment_id: segment_id.clone(),
9552                                        table: table.clone(),
9553                                        requested_scopes_json,
9554                                        byte_length: u64::try_from(byte_length).unwrap_or(0),
9555                                    },
9556                                    &mut on_progress,
9557                                )
9558                                .map_err(|e| {
9559                                    // §5.5: access changed since the response;
9560                                    // the next round's SUB_START settles the unit.
9561                                    if e.code == "sync.forbidden" {
9562                                        SectionError::Refused
9563                                    } else {
9564                                        self.segment_fetch_failed(e)
9565                                    }
9566                                })?
9567                        };
9568                        // §5.4: the descriptor's byteLength caps the body before
9569                        // §5.1 verifies the content address; the client discards
9570                        // the segment and re-pulls on either failure.
9571                        if bytes.len() as i64 > byte_length {
9572                            self.schedule_background_retry();
9573                            return Err(SectionError::Abort(
9574                                "sync.invalid_request".to_owned(),
9575                                "segment exceeds its byteLength".to_owned(),
9576                            ));
9577                        }
9578                        on_progress(bytes.len() as u64);
9579                        let digest = Sha256::digest(&bytes);
9580                        let expected = segment_id
9581                            .strip_prefix("sha256:")
9582                            .unwrap_or(segment_id.as_str());
9583                        if bytes_to_hex(&digest) != expected {
9584                            // §5.1: the client discards the segment and re-pulls.
9585                            self.schedule_background_retry();
9586                            return Err(SectionError::Abort(
9587                                "sync.invalid_request".to_owned(),
9588                                "segment bytes do not match the content address (§5.1)".to_owned(),
9589                            ));
9590                        }
9591                        self.progress.update(|p| p.phase = ProgressPhase::Import);
9592                        if media_type == MediaType::Sqlite {
9593                            // §5.3: a range image keeps its position in the resume
9594                            // token; a descriptor row cursor is invalid.
9595                            if row_cursor.is_some() || next_row_cursor.is_some() {
9596                                return Err(SectionError::Abort(
9597                                    "sync.invalid_request".to_owned(),
9598                                    "sqlite segments are whole-table: rowCursor/nextRowCursor must be absent (§5.3)"
9599                                        .to_owned(),
9600                                ));
9601                            }
9602                            let (pair, clear, effective) =
9603                                self.segment_pair(&unit, &echo, &seeded, &table)?;
9604                            report.segment_rows_applied += self.apply_sqlite_segment(
9605                                &pair,
9606                                &bytes,
9607                                clear,
9608                                &effective,
9609                                &unit.id,
9610                                &segment_id,
9611                                resumed.unwrap_or(0),
9612                                row_count,
9613                                as_of_commit_seq,
9614                                &scope_digest,
9615                            )?;
9616                            seeded.insert(table);
9617                        } else {
9618                            let segment = decode_rows_segment(&bytes).map_err(|e| {
9619                                SectionError::Abort(e.code.as_str().to_owned(), e.detail)
9620                            })?;
9621                            let (pair, clear, effective) =
9622                                self.segment_pair(&unit, &echo, &seeded, &segment.table)?;
9623                            report.segment_rows_applied +=
9624                                self.apply_segment(&pair, &segment, clear, &effective, &unit.id)?;
9625                            seeded.insert(segment.table);
9626                            // An image records its rows with each chunk.
9627                            self.stored(record_applied_rows(
9628                                &self.conn,
9629                                &unit.id,
9630                                &segment_id,
9631                                row_count,
9632                            ))
9633                            .map_err(storage)?;
9634                        }
9635                    }
9636                    _ => {
9637                        return Err(SectionError::Abort(
9638                            "sync.invalid_request".to_owned(),
9639                            "unexpected frame inside a unit section".to_owned(),
9640                        ));
9641                    }
9642                }
9643            }
9644            Ok(())
9645        })();
9646        match applied {
9647            Ok(()) => {}
9648            // The unit keeps catching up with its cursor unpersisted.
9649            Err(SectionError::Refused | SectionError::Deferred) => return Ok(()),
9650            Err(SectionError::FailClosed) => {
9651                // §5.6: the failure is unit-local; the rest of the response
9652                // still applies and the failed unit leaves the list.
9653                self.begin_observation("syncular_section_failure")
9654                    .map_err(storage)?;
9655                let failed = (|| {
9656                    let mut batch = ChangeAccumulator::default();
9657                    for (base_key, value) in self.stored(window_refs(&self.conn, &unit.id))? {
9658                        batch.window(&base_key, &value);
9659                    }
9660                    batch.status = true;
9661                    self.stored(save_unit(
9662                        &self.conn,
9663                        &UnitRecord {
9664                            status: UnitStatus::Failed,
9665                            reason_code: Some("sync.scope_revoked".to_owned()),
9666                            ..unit.clone()
9667                        },
9668                    ))?;
9669                    self.finish_observation("syncular_section_failure", batch)
9670                })();
9671                if let Err(message) = failed {
9672                    self.rollback_observation("syncular_section_failure");
9673                    return Err(storage(message));
9674                }
9675                report.failed.push(unit.id);
9676                return Ok(());
9677            }
9678            Err(error) => return Err(error),
9679        }
9680        let Some((next_cursor, bootstrap_state)) = sub_end else {
9681            return Ok(());
9682        };
9683        let rollback = self.capture();
9684        let counts = (
9685            report.resets.len(),
9686            report.revoked.len(),
9687            report.failed.len(),
9688        );
9689        self.begin_observation("syncular_sub_end")
9690            .map_err(storage)?;
9691        let finished = self
9692            .finish_section(start, echo, next_cursor, bootstrap_state, seeded, report)
9693            .and_then(|batch| {
9694                self.rebuild_overlay_if_dirty()?;
9695                self.finish_observation("syncular_sub_end", batch)
9696            });
9697        if let Err(message) = finished {
9698            self.rollback_observation("syncular_sub_end");
9699            self.restore(rollback);
9700            report.resets.truncate(counts.0);
9701            report.revoked.truncate(counts.1);
9702            report.failed.truncate(counts.2);
9703            return Err(storage(message));
9704        }
9705        Ok(())
9706    }
9707
9708    /// §5.6: the table, first-page flag and effective scopes of one segment
9709    /// of `unit`'s section.
9710    #[allow(clippy::type_complexity)]
9711    fn segment_pair(
9712        &self,
9713        unit: &UnitRecord,
9714        echo: &[EchoRecord],
9715        seeded: &BTreeSet<String>,
9716        name: &str,
9717    ) -> Result<(TableSchema, bool, Vec<(String, Vec<String>)>), SectionError> {
9718        let table = self.schema.table(name).cloned().ok_or_else(|| {
9719            SectionError::Abort(
9720                "sync.schema_mismatch".to_owned(),
9721                format!("segment table {name:?} is not in the client schema"),
9722            )
9723        })?;
9724        let effective = covers(&table, &unit.scopes)
9725            .then(|| pair_scopes(&unit.scopes, name, Some(echo)))
9726            .flatten()
9727            .ok_or_else(|| {
9728                SectionError::Abort(
9729                    "sync.invalid_request".to_owned(),
9730                    "a segment names no active pair of its unit".to_owned(),
9731                )
9732            })?;
9733        Ok((table, !seeded.contains(name), effective))
9734    }
9735
9736    /// §4.4 `SUB_END` of an active, revoked or reset section, inside the
9737    /// caller's observation: a reset discards cursor, resume token and
9738    /// first-page record and keeps the echo; a revoked pair purges by its last
9739    /// echo (§3.3); otherwise the echo replaces the persisted one, a revoked
9740    /// fenced record that turns active again re-bootstraps the unit, and a
9741    /// bootstrap section persists its pin, resume token and seeded tables.
9742    fn finish_section(
9743        &mut self,
9744        start: &SectionStart,
9745        echo: Vec<EchoRecord>,
9746        next_cursor: i64,
9747        bootstrap_state: Option<String>,
9748        seeded: BTreeSet<String>,
9749        report: &mut SyncReport,
9750    ) -> Result<ChangeAccumulator, String> {
9751        #[cfg(feature = "bench-internals")]
9752        let _phase = self.benchmark_phases.start(Phase::CursorPersist);
9753        let mut batch = ChangeAccumulator::default();
9754        // §4.4: SUB_END writes only into a unit that is still registered.
9755        let Some(unit) = self
9756            .stored(get_unit(&self.conn, &start.id))?
9757            .filter(|unit| unit.status == UnitStatus::Active)
9758        else {
9759            return Ok(batch);
9760        };
9761        for (base_key, value) in self.stored(window_refs(&self.conn, &unit.id))? {
9762            batch.window(&base_key, &value);
9763        }
9764        if start.status == SubStatus::Reset {
9765            // §4.6: staleness, never a purge; the rows stay until the
9766            // bootstrap replaces them.
9767            report.resets.push(unit.id.clone());
9768            self.stored(save_unit(
9769                &self.conn,
9770                &UnitRecord {
9771                    reason_code: Some(start.reason_code.clone()),
9772                    cursor: Some(-1),
9773                    bootstrap_state: None,
9774                    seeded: Vec::new(),
9775                    ..unit
9776                },
9777            ))?;
9778            return Ok(batch);
9779        }
9780        let revoked = start.status == SubStatus::Revoked;
9781        let mut units = None;
9782        let mut failed = false;
9783        let mut rearmed = false;
9784        for table in self.schema.tables.clone() {
9785            if !covers(&table, &unit.scopes) {
9786                continue;
9787            }
9788            let last = unit
9789                .echo
9790                .as_deref()
9791                .and_then(|last| pair_scopes(&unit.scopes, &table.name, Some(last)));
9792            let next = (!revoked)
9793                .then(|| pair_scopes(&unit.scopes, &table.name, Some(&echo)))
9794                .flatten();
9795            // §3.3: a revoked fenced record that turns active again re-bootstraps.
9796            rearmed |= unit.echo.is_some() && last.is_none() && next.is_some();
9797            // §3.3: purge a pair the response revokes, keyed on the echo the
9798            // client persisted while the pair was active.
9799            if let (Some(last), None) = (&last, &next) {
9800                let units = match units {
9801                    Some(ref units) => units,
9802                    None => units.insert(self.stored(load_units(&self.conn))?),
9803                };
9804                let owners = unit_owners(units, &table, &unit.id);
9805                failed |= !self.purge_pair(&mut batch, &table, last, &owners)?;
9806            }
9807        }
9808        if revoked || failed {
9809            if revoked {
9810                report.revoked.push(unit.id.clone());
9811            }
9812            if failed {
9813                report.failed.push(unit.id.clone());
9814            }
9815            batch.status = true;
9816            self.stored(save_unit(
9817                &self.conn,
9818                &UnitRecord {
9819                    status: if failed {
9820                        UnitStatus::Failed
9821                    } else {
9822                        UnitStatus::Revoked
9823                    },
9824                    reason_code: Some(if failed {
9825                        "sync.scope_revoked".to_owned()
9826                    } else {
9827                        start.reason_code.clone()
9828                    }),
9829                    cursor: Some(-1),
9830                    bootstrap_state: None,
9831                    seeded: Vec::new(),
9832                    echo: None,
9833                    ..unit
9834                },
9835            ))?;
9836            return Ok(batch);
9837        }
9838        let completed = start.bootstrap && !rearmed && bootstrap_state.is_none();
9839        let (cursor, bootstrap_state, seeded) = if rearmed {
9840            (Some(-1), None, Vec::new())
9841        } else if start.bootstrap {
9842            let seeded = if bootstrap_state.is_none() {
9843                Vec::new()
9844            } else {
9845                seeded.into_iter().collect()
9846            };
9847            (Some(next_cursor), bootstrap_state, seeded)
9848        } else {
9849            (
9850                unit.cursor,
9851                unit.bootstrap_state.clone(),
9852                unit.seeded.clone(),
9853            )
9854        };
9855        self.stored(save_unit(
9856            &self.conn,
9857            &UnitRecord {
9858                reason_code: None,
9859                echo: Some(echo.clone()),
9860                cursor,
9861                bootstrap_state,
9862                seeded,
9863                ..unit.clone()
9864            },
9865        ))?;
9866        if !completed {
9867            return Ok(batch);
9868        }
9869        // §7.1: a completed bootstrap covers acknowledged intent of its pairs
9870        // at or below its pin.
9871        for table in self.schema.tables.clone() {
9872            let Some(effective) = covers(&table, &unit.scopes)
9873                .then(|| pair_scopes(&unit.scopes, &table.name, Some(&echo)))
9874                .flatten()
9875            else {
9876                continue;
9877            };
9878            let mut statement = self
9879                .conn
9880                .prepare("SELECT id,intent_json FROM _syncular_acked_rows WHERE tbl=?1 AND commit_seq<=?2")
9881                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9882            let rows = statement
9883                .query_map(rusqlite::params![table.name, next_cursor], |row| {
9884                    Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
9885                })
9886                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9887            let mut covered = HashSet::new();
9888            for row in rows {
9889                let (id, intent) =
9890                    row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9891                let values: Map<String, Value> =
9892                    serde_json::from_str(&intent).map_err(|_| "sync.local_corrupt".to_owned())?;
9893                if owns_local_row(&table, &effective, &[], &values) {
9894                    covered.insert(id);
9895                }
9896            }
9897            drop(statement);
9898            for id in covered {
9899                self.conn
9900                    .execute(
9901                        "DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2 AND commit_seq<=?3",
9902                        rusqlite::params![table.name, id, next_cursor],
9903                    )
9904                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9905                self.overlay_dirty.table(&table.name);
9906                batch.table(&table.name);
9907            }
9908        }
9909        Ok(batch)
9910    }
9911
9912    /// §3.3: purge one revoked pair inside the caller's observation: delete
9913    /// the rows matching its last effective scopes that no owner holds, drop
9914    /// the retained intent and every pending commit that provably writes into
9915    /// them, and delete now-unauthorized blob bodies. `Ok(false)` when the
9916    /// table has no local scope-column mapping (fail closed: nothing deleted).
9917    fn purge_pair(
9918        &mut self,
9919        batch: &mut ChangeAccumulator,
9920        table: &TableSchema,
9921        last: &[(String, Vec<String>)],
9922        owners: &[Vec<(String, Vec<String>)>],
9923    ) -> Result<bool, String> {
9924        if local_scope_predicate(table, last, owners).is_err() {
9925            return Ok(false);
9926        }
9927        for commit in self.drop_failed_commits(|name, values| {
9928            name == table.name && owns_local_row(table, last, owners, values)
9929        })? {
9930            batch.conflicts = true;
9931            batch.rejections = true;
9932            batch.outcomes = true;
9933            for operation in commit.ops {
9934                batch.table(&operation.table);
9935            }
9936        }
9937        if !self.purge_scope_rows(&table.name, last, owners)? {
9938            return Err("storage.failed: the revoked scope purge failed".to_owned());
9939        }
9940        self.record_scope_map(batch, &table.name, last);
9941        if self.drop_doomed_outbox(table, last, owners)? {
9942            batch.status = true;
9943            batch.rejections = true;
9944            batch.outcomes = true;
9945        }
9946        // §5.9.7 B2: revocation deletes now-unauthorized blob bodies.
9947        self.delete_unreferenced_cached_blobs()?;
9948        Ok(true)
9949    }
9950
9951    /// §4.5 client application: apply one live-section or delta `COMMIT`
9952    /// with only the changes that match the persisted echo of a receiving
9953    /// unit; every applied change then goes through the version guard.
9954    /// Returns whether any change of the frame applied.
9955    fn apply_live_commit(
9956        &mut self,
9957        receivers: &[UnitRecord],
9958        tables: &[String],
9959        changes: Vec<ssp2::model::Change>,
9960        commit_seq: i64,
9961        source: AppliedServerCommitSource,
9962    ) -> Result<bool, TransportError> {
9963        let changes: Vec<ssp2::model::Change> = changes
9964            .into_iter()
9965            .filter(|change| {
9966                // An unknown table fails loud in the apply below.
9967                let Some(table) = tables
9968                    .get(change.table_index as usize)
9969                    .and_then(|name| self.schema.table(name))
9970                else {
9971                    return true;
9972                };
9973                receivers.iter().any(|unit| {
9974                    covers(table, &unit.scopes)
9975                        && pair_scopes(&unit.scopes, &table.name, unit.echo.as_deref()).is_some_and(
9976                            |scopes| {
9977                                scopes.iter().all(|(variable, values)| {
9978                                    change
9979                                        .scopes
9980                                        .iter()
9981                                        .find(|(key, _)| key == variable)
9982                                        .is_some_and(|(_, value)| values.contains(value))
9983                                })
9984                            },
9985                        )
9986                })
9987            })
9988            .collect();
9989        if changes.is_empty() {
9990            return Ok(false);
9991        }
9992        let affected_tables = changes
9993            .iter()
9994            .filter_map(|change| tables.get(change.table_index as usize))
9995            .cloned()
9996            .collect::<BTreeSet<_>>();
9997        let previous_batches = self.change_queue.len();
9998        self.apply_commit_frame(tables, &changes, commit_seq)?;
9999        if self.change_queue.len() > previous_batches {
10000            if let Some(batch) = self.change_queue.back_mut() {
10001                batch
10002                    .applied_commits
10003                    .extend(
10004                        affected_tables
10005                            .into_iter()
10006                            .map(|table| AppliedServerCommit {
10007                                table,
10008                                commit_seq,
10009                                source,
10010                            }),
10011                    );
10012            }
10013        }
10014        Ok(true)
10015    }
10016
10017    // Both pull and realtime frames use the same durable observation boundary.
10018    fn apply_commit_frame(
10019        &mut self,
10020        tables: &[String],
10021        changes: &[ssp2::model::Change],
10022        sequence: i64,
10023    ) -> Result<(), TransportError> {
10024        #[cfg(feature = "bench-internals")]
10025        let _phase = self.benchmark_phases.start(Phase::CommitApply);
10026        let rollback = self.capture();
10027        self.coalesce_unit(changes.len())
10028            .and_then(|()| self.begin_observation("syncular_commit"))
10029            .map_err(|message| TransportError::new("storage.failed", message))?;
10030        let applied = (|| {
10031            let mut batch = ChangeAccumulator::default();
10032            self.record_commit_changes(&mut batch, tables, changes);
10033            // §7.1 chained base: a change names the version an acknowledged own
10034            // commit wrote; pending upserts chained on it move to that version.
10035            for change in changes {
10036                let Some(table) = tables.get(change.table_index as usize) else {
10037                    continue;
10038                };
10039                let acknowledged = self
10040                    .conn
10041                    .prepare_cached("SELECT op_json FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2 AND commit_seq=?3")
10042                    .and_then(|mut statement| {
10043                        statement
10044                            .query_map(rusqlite::params![table, change.row_id, sequence], |row| row.get::<_, String>(0))?
10045                            .collect::<Result<Vec<_>, _>>()
10046                    })
10047                    .map_err(|error| TransportError::new("storage.failed", Self::sqlite_failure(&self.storage_failure, error)))?
10048                    .iter()
10049                    .map(|op| serde_json::from_str::<CommitOperation>(op).map(OutboxOp::from))
10050                    .collect::<Result<Vec<_>, _>>()
10051                    .map_err(|_| TransportError::new("storage.failed", "sync.local_corrupt: invalid acknowledged operation".to_owned()))?;
10052                for index in 0..self.outbox.len() {
10053                    if !self.outbox[index]
10054                        .ops
10055                        .iter()
10056                        .any(|op| acknowledged.iter().any(|earlier| chains_on(earlier, op)))
10057                    {
10058                        continue;
10059                    }
10060                    let commit = Arc::make_mut(&mut self.outbox[index]);
10061                    for op in &mut commit.ops {
10062                        if acknowledged.iter().any(|earlier| chains_on(earlier, op)) {
10063                            op.base_version = Some(change.row_version);
10064                        }
10065                    }
10066                    let ops = serde_json::to_string(
10067                        &commit
10068                            .ops
10069                            .iter()
10070                            .map(CommitOperation::from)
10071                            .collect::<Vec<_>>(),
10072                    )
10073                    .map_err(|_| {
10074                        TransportError::new(
10075                            "storage.failed",
10076                            "sync.local_corrupt: unencodable outbox".to_owned(),
10077                        )
10078                    })?;
10079                    let id = commit.client_commit_id.clone();
10080                    self.conn
10081                        .execute(
10082                            "UPDATE _syncular_outbox SET ops_json=?1 WHERE commit_id=?2",
10083                            rusqlite::params![ops, id],
10084                        )
10085                        .map_err(|error| {
10086                            TransportError::new(
10087                                "storage.failed",
10088                                Self::sqlite_failure(&self.storage_failure, error),
10089                            )
10090                        })?;
10091                }
10092            }
10093            {
10094                let storage_failed = |error| {
10095                    TransportError::new(
10096                        "storage.failed",
10097                        Self::sqlite_failure(&self.storage_failure, error),
10098                    )
10099                };
10100                let pinned: HashSet<(&str, &str)> = self
10101                    .outbox
10102                    .iter()
10103                    .flat_map(|commit| &commit.ops)
10104                    .map(|op| (op.table.as_str(), op.row_id.as_str()))
10105                    .collect();
10106                for change in changes {
10107                    let table = tables.get(change.table_index as usize).ok_or_else(|| {
10108                        TransportError::new(
10109                            "sync.invalid_request",
10110                            "change tableIndex out of range".to_owned(),
10111                        )
10112                    })?;
10113                    if pinned.contains(&(table.as_str(), change.row_id.as_str())) {
10114                        self.conn.prepare_cached("INSERT INTO _syncular_row_deliveries(tbl,id,commit_seq) VALUES (?1,?2,?3) ON CONFLICT(tbl,id) DO UPDATE SET commit_seq=MAX(commit_seq,excluded.commit_seq)")
10115                            .and_then(|mut statement| statement.execute(rusqlite::params![table, change.row_id, sequence]))
10116                            .map_err(storage_failed)?;
10117                    }
10118                    self.conn.prepare_cached("DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2 AND commit_seq<=?3")
10119                        .and_then(|mut statement| statement.execute(rusqlite::params![table, change.row_id, sequence]))
10120                        .map_err(storage_failed)?;
10121                }
10122            }
10123            self.apply_commit_changes(tables, changes)?;
10124            self.rebuild_overlay_if_dirty()
10125                .map_err(|message| TransportError::new("storage.failed", message))?;
10126            self.finish_observation("syncular_commit", batch)
10127                .map_err(|message| TransportError::new("storage.failed", message))
10128        })();
10129        if applied.is_err() {
10130            self.rollback_observation("syncular_commit");
10131            self.restore(rollback);
10132        }
10133        applied
10134    }
10135
10136    fn apply_commit_changes(
10137        &mut self,
10138        tables: &[String],
10139        changes: &[ssp2::model::Change],
10140    ) -> Result<(), TransportError> {
10141        // A clean overlay can reconcile changed primary keys independently
10142        // when their tables have no secondary unique constraints. Such a
10143        // constraint can change whether another row's write succeeds, so a
10144        // table with one and any local intent takes the table rebuild.
10145        let changed_tables: BTreeSet<&str> = changes
10146            .iter()
10147            .filter_map(|change| tables.get(change.table_index as usize))
10148            .map(String::as_str)
10149            .collect();
10150        let mut mirror_visible = !self.overlay_dirty.get();
10151        for name in &changed_tables {
10152            if mirror_visible
10153                && self
10154                    .schema
10155                    .table(name)
10156                    .is_some_and(|table| table.indexes.iter().any(|index| index.unique))
10157            {
10158                let (pending, protected) = self
10159                    .table_intent(name)
10160                    .map_err(|message| TransportError::new("storage.failed", message))?;
10161                mirror_visible = !pending && !protected;
10162            }
10163        }
10164        for change in changes {
10165            let table_name = tables.get(change.table_index as usize).ok_or_else(|| {
10166                TransportError::new(
10167                    "sync.invalid_request",
10168                    "change tableIndex out of range".to_owned(),
10169                )
10170            })?;
10171            let table = self.schema.table(table_name).ok_or_else(|| {
10172                TransportError::new(
10173                    "sync.schema_mismatch",
10174                    format!("change targets unknown table {table_name:?}"),
10175                )
10176            })?;
10177            let landed = match change.op {
10178                Op::Upsert => {
10179                    let payload = change.row.as_ref().ok_or_else(|| {
10180                        TransportError::new(
10181                            "sync.invalid_request",
10182                            "upsert change without row payload".to_owned(),
10183                        )
10184                    })?;
10185                    // §5.11: decrypt encrypted columns on apply, or seal the row.
10186                    let (row, sealed) = {
10187                        #[cfg(feature = "bench-internals")]
10188                        let _phase = self.benchmark_phases.start(Phase::RowDecode);
10189                        // §5.11: a decrypt failure keeps its own identity.
10190                        decode_row_bytes(table, payload, &self.encryption)?
10191                    };
10192                    let table_name = table.name.clone();
10193                    // §4.5 version guard: only a newer upsert lands.
10194                    let landed = self
10195                        .write_base_row(&table_name, &row, change.row_version, sealed.as_ref())
10196                        .map_err(|m| TransportError::new("sync.invalid_request", m))?;
10197                    if mirror_visible && landed {
10198                        self.write_row(
10199                            &visible_table(&table_name),
10200                            &table_name,
10201                            &row,
10202                            change.row_version,
10203                            sealed.as_ref(),
10204                        )
10205                        .map_err(|m| TransportError::new("sync.invalid_request", m))?;
10206                    }
10207                    landed
10208                }
10209                // §4.5 version guard: a delete at or below the local version
10210                // is skipped, a delete of an absent row is a no-op.
10211                Op::Delete => self
10212                    .delete_base_row(table_name, &change.row_id, change.row_version)
10213                    .map_err(|m| TransportError::new("sync.invalid_request", m))?,
10214            };
10215            // The change dropped the row's acknowledged intent either way
10216            // (§7.1), so the overlay of a skipped change rebuilds from the
10217            // unchanged base.
10218            self.overlay_dirty.table(table_name);
10219            if mirror_visible && !(landed && change.op == Op::Upsert) {
10220                let predicate = row_id_predicate(table);
10221                let visible = visible_table(table_name);
10222                let mut statements = vec![format!("DELETE FROM {visible} WHERE {predicate}")];
10223                if !landed {
10224                    statements.push(format!(
10225                        "INSERT INTO {visible} SELECT * FROM {} WHERE {predicate}",
10226                        base_table(table_name)
10227                    ));
10228                }
10229                for statement in &statements {
10230                    self.conn
10231                        .prepare_cached(statement)
10232                        .and_then(|mut statement| {
10233                            statement.execute(rusqlite::params![change.row_id])
10234                        })
10235                        .map_err(|error| {
10236                            TransportError::new("sync.invalid_request", error.to_string())
10237                        })?;
10238                }
10239            }
10240        }
10241        if mirror_visible {
10242            let changed_rows: HashSet<(&str, &str)> = changes
10243                .iter()
10244                .map(|change| {
10245                    (
10246                        tables[change.table_index as usize].as_str(),
10247                        change.row_id.as_str(),
10248                    )
10249                })
10250                .collect();
10251            self.replay_overlay(&changed_tables, Some(&changed_rows))
10252                .map_err(|message| TransportError::new("storage.failed", message))?;
10253            self.overlay_dirty.set(false);
10254        }
10255        Ok(())
10256    }
10257
10258    /// §5.6 segment application: validate against the generated schema,
10259    /// clear the pair's scope on the table's first page (fail closed
10260    /// without a mapping), then upsert each row with its segment-carried
10261    /// server version under the §4.5 version guard (§5.2).
10262    fn apply_segment(
10263        &mut self,
10264        table: &TableSchema,
10265        segment: &RowsSegment,
10266        first_page: bool,
10267        effective: &[(String, Vec<String>)],
10268        unit_id: &str,
10269    ) -> Result<u32, SectionError> {
10270        // §5.2: the column table validates against the generated schema —
10271        // order, names, types, nullability; mismatch is fatal. §5.11: the
10272        // server sends the WIRE types (bytes for an encrypted column), so
10273        // validate against wire_columns.
10274        let column = if segment.columns.len() != table.wire_columns.len() {
10275            Some(Value::from("count"))
10276        } else {
10277            segment
10278                .columns
10279                .iter()
10280                .zip(table.wire_columns.iter())
10281                .position(|(a, b)| a.name != b.name || a.ty != b.ty || a.nullable != b.nullable)
10282                .map(Value::from)
10283        };
10284        if segment.table != table.name
10285            || segment.schema_version != self.schema.version
10286            || column.is_some()
10287        {
10288            let mut details = Map::new();
10289            details.insert("table".to_owned(), Value::from(table.name.as_str()));
10290            if let Some(column) = column {
10291                details.insert("column".to_owned(), column);
10292            }
10293            return Err(SectionError::Transport(TransportError {
10294                code: "sync.schema_mismatch".to_owned(),
10295                message: "rows segment columns do not match the generated schema (§5.2)".to_owned(),
10296                details: Some(Value::Object(details)),
10297            }));
10298        }
10299        let mut applied = 0u32;
10300        let empty = Vec::new();
10301        let blocks = if segment.blocks.is_empty() {
10302            std::slice::from_ref(&empty)
10303        } else {
10304            &segment.blocks
10305        };
10306        for (index, block) in blocks.iter().enumerate() {
10307            let clear = first_page && index == 0;
10308            if !clear && block.is_empty() {
10309                continue;
10310            }
10311            let rollback = self.capture();
10312            let (pending, protected) = self
10313                .table_intent(&table.name)
10314                .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10315            let reconcile_pending =
10316                !protected && pending && table.indexes.iter().any(|index| index.unique);
10317            let mirror_visible = !protected && !reconcile_pending;
10318            self.coalesce_unit(block.len())
10319                .and_then(|()| self.begin_observation("syncular_segment_block"))
10320                .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10321            let outcome = (|| {
10322                self.rebuild_overlay_if_dirty()
10323                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10324                let mut changed_keys = Vec::new();
10325                let mut batch = ChangeAccumulator::default();
10326                if clear && self.clear_scope(table, effective, unit_id)? || !block.is_empty() {
10327                    batch.table(&table.name);
10328                }
10329                for (row_index, row) in block.iter().enumerate() {
10330                    let decrypted;
10331                    let mut sealed = None;
10332                    let values = if table.has_encrypted_columns() {
10333                        let mut values = row.values.clone();
10334                        sealed = crate::values::decrypt_segment_row(
10335                            table,
10336                            &mut values,
10337                            &self.encryption,
10338                        )
10339                        .map_err(SectionError::Transport)?;
10340                        decrypted = values;
10341                        &decrypted
10342                    } else {
10343                        &row.values
10344                    };
10345                    if !self
10346                        .write_base_row(&table.name, values, row.server_version, sealed.as_ref())
10347                        .map_err(|message| {
10348                            SectionError::Abort("sync.invalid_request".into(), message)
10349                        })?
10350                    {
10351                        continue;
10352                    }
10353                    if reconcile_pending {
10354                        changed_keys.push(
10355                            owned_sql_value(RowParam::Cell(&values[table.pk_index])).map_err(
10356                                |message| SectionError::Abort("storage.failed".into(), message),
10357                            )?,
10358                        );
10359                    }
10360                    if mirror_visible {
10361                        self.write_row(
10362                            &visible_table(&table.name),
10363                            &table.name,
10364                            values,
10365                            row.server_version,
10366                            sealed.as_ref(),
10367                        )
10368                        .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10369                    }
10370                    let processed = u64::from(applied) + row_index as u64 + 1;
10371                    if processed.is_multiple_of(1024) {
10372                        self.progress.update(|p| p.rows_processed = processed);
10373                    }
10374                }
10375                self.reconcile_import(table, &mut changed_keys, protected, reconcile_pending)
10376                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10377                self.finish_observation("syncular_segment_block", batch)
10378                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))
10379            })();
10380            if let Err(error) = outcome {
10381                self.rollback_observation("syncular_segment_block");
10382                self.restore(rollback);
10383                return Err(error);
10384            }
10385            applied += block.len() as u32;
10386            self.progress
10387                .update(|p| p.rows_processed = u64::from(applied));
10388            std::thread::yield_now();
10389        }
10390        self.progress
10391            .update(|p| p.rows_processed = u64::from(applied));
10392        Ok(applied)
10393    }
10394
10395    /// §5.3 sqlite-image application: validate the in-file metadata
10396    /// against the descriptor, validate column names/order against the
10397    /// generated schema, run the §5.6 first-page clear when fresh, then
10398    /// replace-or-upsert every image row with its `_syncular_version`.
10399    /// Mechanics: the image lands in a temp file read through a second
10400    /// rusqlite connection, equivalent to ATTACH + INSERT…SELECT.
10401    /// Each image chunk commits its writes and revision before readers continue.
10402    #[allow(clippy::too_many_arguments)]
10403    fn apply_sqlite_segment(
10404        &mut self,
10405        table: &TableSchema,
10406        bytes: &[u8],
10407        first_page: bool,
10408        effective: &[(String, Vec<String>)],
10409        unit_id: &str,
10410        segment_id: &str,
10411        resumed: i64,
10412        row_count: i64,
10413        as_of_commit_seq: i64,
10414        scope_digest: &str,
10415    ) -> Result<u32, SectionError> {
10416        let invalid = |detail: &str| {
10417            SectionError::Abort(
10418                "sync.invalid_request".to_owned(),
10419                format!("sqlite segment rejected: {detail} (§5.3)"),
10420            )
10421        };
10422        // §5.11: an image bypasses the decrypt seam, so a server cannot serve
10423        // one for a table with encrypted columns.
10424        if table.has_encrypted_columns() {
10425            return Err(invalid("table has encrypted columns"));
10426        }
10427
10428        // The image lands beside the replica so it shares the replica's
10429        // directory permissions and volume; an in-memory replica has no
10430        // directory and uses the system temp dir. The file is owner-only.
10431        let directory = self
10432            .previous_version_replica_path()
10433            .and_then(|replica| std::path::Path::new(&replica).parent().map(Into::into))
10434            .unwrap_or_else(std::env::temp_dir);
10435        let path = directory.join(format!(".syncular-image-{}.tmp", uuid::Uuid::new_v4()));
10436        let outcome = (|| {
10437            let mut options = std::fs::OpenOptions::new();
10438            options.write(true).create_new(true);
10439            #[cfg(unix)]
10440            std::os::unix::fs::OpenOptionsExt::mode(&mut options, 0o600);
10441            std::io::Write::write_all(
10442                &mut options
10443                    .open(&path)
10444                    .map_err(|_| invalid("image temp file write failed"))?,
10445                bytes,
10446            )
10447            .map_err(|_| invalid("image temp file write failed"))?;
10448            let img = rusqlite::Connection::open_with_flags(
10449                &path,
10450                rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY,
10451            )
10452            .map_err(|_| invalid("bytes do not open as a SQLite database"))?;
10453            self.apply_sqlite_image(
10454                &img,
10455                table,
10456                first_page,
10457                effective,
10458                unit_id,
10459                segment_id,
10460                resumed,
10461                row_count,
10462                as_of_commit_seq,
10463                scope_digest,
10464            )
10465        })();
10466        let _ = std::fs::remove_file(&path);
10467        outcome
10468    }
10469
10470    #[allow(clippy::too_many_arguments)]
10471    fn apply_sqlite_image(
10472        &mut self,
10473        img: &rusqlite::Connection,
10474        table: &crate::schema::TableSchema,
10475        first_fresh_page: bool,
10476        effective: &[(String, Vec<String>)],
10477        unit_id: &str,
10478        segment_id: &str,
10479        resumed: i64,
10480        row_count: i64,
10481        as_of_commit_seq: i64,
10482        scope_digest: &str,
10483    ) -> Result<u32, SectionError> {
10484        let invalid = |detail: String| {
10485            SectionError::Abort(
10486                "sync.invalid_request".to_owned(),
10487                format!("sqlite segment rejected: {detail} (§5.3)"),
10488            )
10489        };
10490
10491        // 1. Metadata vs descriptor + client state (§5.3 rule 2; exactly
10492        //    one row).
10493        type MetaRow = (i64, String, i64, i64, String, i64, i64);
10494        let meta: MetaRow = img
10495            .query_row(
10496                "SELECT format, \"table\", \"schemaVersion\", \"asOfCommitSeq\",
10497                        \"scopeDigest\", \"rowCount\",
10498                        (SELECT count(*) FROM _syncular_segment)
10499                 FROM _syncular_segment",
10500                [],
10501                |row| {
10502                    Ok((
10503                        row.get(0)?,
10504                        row.get(1)?,
10505                        row.get(2)?,
10506                        row.get(3)?,
10507                        row.get(4)?,
10508                        row.get(5)?,
10509                        row.get(6)?,
10510                    ))
10511                },
10512            )
10513            .map_err(|_| invalid("missing or unreadable _syncular_segment metadata".to_owned()))?;
10514        let (format, meta_table, schema_version, pin, digest, meta_rows, meta_count) = meta;
10515        if meta_count != 1 {
10516            return Err(invalid(format!(
10517                "_syncular_segment must contain exactly one row, found {meta_count}"
10518            )));
10519        }
10520        if format != 1 {
10521            return Err(invalid(format!("format {format}")));
10522        }
10523        if meta_table != table.name {
10524            return Err(invalid(format!("image table {meta_table:?}")));
10525        }
10526        if schema_version != i64::from(self.schema.version) {
10527            return Err(invalid(format!("schemaVersion {schema_version}")));
10528        }
10529        if pin != as_of_commit_seq {
10530            return Err(invalid(format!("asOfCommitSeq {pin}")));
10531        }
10532        if digest != scope_digest {
10533            return Err(invalid("scopeDigest mismatch".to_owned()));
10534        }
10535        if meta_rows != row_count {
10536            return Err(invalid(format!("rowCount {meta_rows}")));
10537        }
10538
10539        // 2. Column names and order vs the generated schema (§5.3 rule 3).
10540        let mut names: Vec<String> = Vec::new();
10541        let mut primary_keys: Vec<String> = Vec::new();
10542        {
10543            let mut stmt = img
10544                .prepare(&format!("PRAGMA table_info({})", quote_ident(&table.name)))
10545                .map_err(|_| invalid("image data table missing".to_owned()))?;
10546            let mut rows = stmt
10547                .query([])
10548                .map_err(|_| invalid("image data table unreadable".to_owned()))?;
10549            while let Some(row) = rows
10550                .next()
10551                .map_err(|_| invalid("image data table unreadable".to_owned()))?
10552            {
10553                if row
10554                    .get::<_, i64>(5)
10555                    .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?
10556                    > 0
10557                {
10558                    primary_keys.push(row.get(1).map_err(|error| {
10559                        invalid(Self::sqlite_failure(&self.storage_failure, error))
10560                    })?);
10561                }
10562                names.push(
10563                    row.get::<_, String>(1)
10564                        .map_err(|_| invalid("image data table unreadable".to_owned()))?,
10565                );
10566            }
10567        }
10568        let mut expected: Vec<&str> = table.columns.iter().map(|c| c.name.as_str()).collect();
10569        expected.push("_syncular_version");
10570        if names.len() != expected.len() || names.iter().zip(expected.iter()).any(|(a, b)| a != b) {
10571            return Err(SectionError::Transport(TransportError {
10572                code: "sync.schema_mismatch".to_owned(),
10573                message: "sqlite segment columns do not match the generated schema (§5.3)"
10574                    .to_owned(),
10575                details: Some(serde_json::json!({ "table": table.name })),
10576            }));
10577        }
10578
10579        if primary_keys != vec![table.primary_key.clone()] {
10580            return Err(invalid(
10581                "image primary key does not match generated schema".into(),
10582            ));
10583        }
10584        let actual_count: i64 = img
10585            .query_row(
10586                &format!("SELECT count(*) FROM {}", quote_ident(&table.name)),
10587                [],
10588                |row| row.get(0),
10589            )
10590            .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
10591        if actual_count != row_count {
10592            return Err(invalid("image row count does not match descriptor".into()));
10593        }
10594        let insert = self.insert_row_sql(&base_table(&table.name), table);
10595        let visible_insert = self.insert_row_sql(&visible_table(&table.name), table);
10596        let column_list = names
10597            .iter()
10598            .map(|n| quote_ident(n))
10599            .collect::<Vec<_>>()
10600            .join(", ");
10601        // §5.6: an earlier attempt at this position committed the first
10602        // `resumed` rows in primary-key order; the content address pins them.
10603        let mut stmt = img
10604            .prepare(&format!(
10605                "SELECT {column_list} FROM {} ORDER BY {} LIMIT -1 OFFSET ?1",
10606                quote_ident(&table.name),
10607                quote_ident(&table.primary_key)
10608            ))
10609            .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
10610        let mut rows = stmt
10611            .query([resumed])
10612            .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
10613        let resumed = u32::try_from(resumed)
10614            .ok()
10615            .filter(|rows| i64::from(*rows) <= actual_count)
10616            .ok_or_else(|| invalid("applied-segment record exceeds the image".into()))?;
10617        let mut applied = resumed;
10618        let mut limit = if first_fresh_page {
10619            1024
10620        } else {
10621            self.image_chunk_rows
10622                .get(&table.name)
10623                .copied()
10624                .unwrap_or(1024)
10625        };
10626        let (mut write, mut commit) = (std::time::Duration::ZERO, std::time::Duration::ZERO);
10627        loop {
10628            let clear = applied == 0 && first_fresh_page;
10629            let rollback = self.capture();
10630            let (pending, protected) = self
10631                .table_intent(&table.name)
10632                .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10633            let reconcile_pending =
10634                !protected && pending && table.indexes.iter().any(|index| index.unique);
10635            let mirror_visible = !protected && !reconcile_pending;
10636            // §5.3: a table's first chunk holds 1,024 rows; a later range image
10637            // starts at the size its previous range reached. A later chunk holds at
10638            // least 1,024 rows, at most twice the rows of the one before, at
10639            // most 65,536, and at most the rows the one before applied per
10640            // 250 ms from its start to its COMMIT; a chunk that replays local
10641            // writes holds at most 1,024.
10642            let chunk = if pending || protected {
10643                limit.min(1024)
10644            } else {
10645                limit
10646            };
10647            self.coalesce_unit(chunk as usize)
10648                .and_then(|()| self.begin_observation("syncular_image_chunk"))
10649                .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10650            let started = std::time::Instant::now();
10651            let result = (|| {
10652                self.rebuild_overlay_if_dirty()
10653                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10654                let mut changed_keys = Vec::new();
10655                let mut batch = ChangeAccumulator::default();
10656                if clear && self.clear_scope(table, effective, unit_id)? {
10657                    batch.table(&table.name);
10658                }
10659                let mut processed = 0u32;
10660                {
10661                    let mut ins = self.conn.prepare_cached(&insert).map_err(|error| {
10662                        invalid(Self::sqlite_failure(&self.storage_failure, error))
10663                    })?;
10664                    let mut mirror = if mirror_visible {
10665                        Some(self.conn.prepare_cached(&visible_insert).map_err(|error| {
10666                            invalid(Self::sqlite_failure(&self.storage_failure, error))
10667                        })?)
10668                    } else {
10669                        None
10670                    };
10671                    for _ in 0..chunk {
10672                        let Some(row) = rows.next().map_err(|error| {
10673                            invalid(Self::sqlite_failure(&self.storage_failure, error))
10674                        })?
10675                        else {
10676                            break;
10677                        };
10678                        for (i, column) in table.columns.iter().enumerate() {
10679                            let cell = row.get_ref(i).map_err(|error| {
10680                                invalid(Self::sqlite_failure(&self.storage_failure, error))
10681                            })?;
10682                            let param = image_cell_param(column, cell).map_err(invalid)?;
10683                            ins.raw_bind_parameter(i + 1, &param).map_err(|error| {
10684                                invalid(Self::sqlite_failure(&self.storage_failure, error))
10685                            })?;
10686                            if let Some(mirror) = &mut mirror {
10687                                mirror.raw_bind_parameter(i + 1, &param).map_err(|error| {
10688                                    invalid(Self::sqlite_failure(&self.storage_failure, error))
10689                                })?;
10690                            }
10691                        }
10692                        let version_index = table.columns.len();
10693                        let version: i64 = row.get(version_index).map_err(|error| {
10694                            invalid(Self::sqlite_failure(&self.storage_failure, error))
10695                        })?;
10696                        if version < 1 {
10697                            return Err(invalid("image row version must be positive".into()));
10698                        }
10699                        ins.raw_bind_parameter(version_index + 1, version)
10700                            .map_err(|error| {
10701                                invalid(Self::sqlite_failure(&self.storage_failure, error))
10702                            })?;
10703                        // §4.5 version guard: the base upsert decides.
10704                        let landed = ins.raw_execute().map_err(|error| {
10705                            invalid(Self::sqlite_failure(&self.storage_failure, error))
10706                        })? > 0;
10707                        processed += 1;
10708                        if processed.is_multiple_of(1024) && processed < chunk {
10709                            self.progress
10710                                .update(|p| p.rows_processed = u64::from(applied + processed));
10711                        }
10712                        if !landed {
10713                            continue;
10714                        }
10715                        if let Some(mirror) = &mut mirror {
10716                            mirror
10717                                .raw_bind_parameter(version_index + 1, version)
10718                                .map_err(|error| {
10719                                    invalid(Self::sqlite_failure(&self.storage_failure, error))
10720                                })?;
10721                            mirror.raw_execute().map_err(|error| {
10722                                invalid(Self::sqlite_failure(&self.storage_failure, error))
10723                            })?;
10724                        }
10725                        if reconcile_pending {
10726                            changed_keys.push(row.get::<_, SqlValue>(table.pk_index).map_err(
10727                                |error| invalid(Self::sqlite_failure(&self.storage_failure, error)),
10728                            )?);
10729                        }
10730                    }
10731                }
10732                if processed > 0 {
10733                    batch.table(&table.name);
10734                    self.overlay_dirty.table(&table.name);
10735                }
10736                self.stored(record_applied_rows(
10737                    &self.conn,
10738                    unit_id,
10739                    segment_id,
10740                    i64::from(applied + processed),
10741                ))
10742                .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10743                self.reconcile_import(table, &mut changed_keys, protected, reconcile_pending)
10744                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10745                let written = std::time::Instant::now();
10746                self.finish_observation("syncular_image_chunk", batch)
10747                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10748                Ok((processed, written))
10749            })();
10750            let (processed, written) = match result {
10751                Ok(done) => done,
10752                Err(error) => {
10753                    self.rollback_observation("syncular_image_chunk");
10754                    self.restore(rollback);
10755                    return Err(error);
10756                }
10757            };
10758            // The chunk commits before the yield, also inside a coalesced apply.
10759            let coalesced = self.coalesced.is_some();
10760            if coalesced {
10761                self.commit_coalesced_apply()
10762                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10763            }
10764            write += written - started;
10765            commit += written.elapsed();
10766            applied += processed;
10767            let last = processed < chunk || i64::from(applied) == actual_count;
10768            // A file replica's WAL keeps its high-water size until truncated.
10769            if last && self.conn.is_autocommit() {
10770                self.conn
10771                    .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))
10772                    .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
10773            }
10774            if coalesced {
10775                self.begin_coalesced_apply()
10776                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10777            }
10778            self.progress.update(|p| {
10779                p.rows_processed = u64::from(applied);
10780                p.import_write_ms = u64::try_from(write.as_millis()).ok();
10781                p.import_commit_ms = u64::try_from(commit.as_millis()).ok();
10782            });
10783            let duration = started.elapsed().as_secs_f64() * 1000.0;
10784            #[cfg(test)]
10785            let duration = self.image_chunk_ms.get().unwrap_or(duration);
10786            std::thread::yield_now();
10787            if last {
10788                break;
10789            }
10790            // A zero duration saturates the cast and leaves the doubling.
10791            limit = (processed * 2)
10792                .min(65_536)
10793                .min((f64::from(processed) * 250.0 / duration) as u32)
10794                .max(1024);
10795            self.image_chunk_rows.insert(table.name.clone(), limit);
10796        }
10797        if i64::from(applied) != actual_count {
10798            return Err(invalid(format!(
10799                "image yielded {applied} rows, expected {actual_count}"
10800            )));
10801        }
10802        Ok(applied - resumed)
10803    }
10804
10805    /// §7.1 keyed reconciliation of the rows `drained` commits touch: restore
10806    /// them from the base and replay their remaining intent. A table with a
10807    /// secondary unique index restores and replays every row with intent,
10808    /// since one row's value can admit or defer another row's write.
10809    fn reconcile_rows(&self, drained: &[Arc<OutboxCommit>]) -> Result<(), String> {
10810        let names: BTreeSet<&str> = drained
10811            .iter()
10812            .flat_map(|commit| &commit.ops)
10813            .map(|op| op.table.as_str())
10814            .collect();
10815        for name in names {
10816            let Some(table) = self.schema.table(name) else {
10817                continue;
10818            };
10819            let ops = drained.iter().flat_map(|commit| &commit.ops);
10820            let tables = BTreeSet::from([name]);
10821            let rows: HashSet<_> = ops
10822                .clone()
10823                .map(|op| (op.table.as_str(), op.row_id.as_str()))
10824                .collect();
10825            let unique = table.indexes.iter().any(|index| index.unique);
10826            let acknowledged = if unique {
10827                self.acknowledged_operations(None, Some(&tables))?
10828            } else {
10829                Vec::new()
10830            };
10831            let peers = acknowledged
10832                .iter()
10833                .chain(
10834                    self.failed_commits
10835                        .iter()
10836                        .flat_map(|(commit, _)| &commit.ops),
10837                )
10838                .chain(self.outbox.iter().flat_map(|commit| &commit.ops))
10839                .filter(|_| unique);
10840            self.restore_base_rows(table, &mut Vec::new(), peers.chain(ops.clone()))?;
10841            self.replay_overlay(&tables, (!unique).then_some(&rows))?;
10842        }
10843        Ok(())
10844    }
10845
10846    /// Whether `table` has pending outbox intent, and whether it has protected
10847    /// (acknowledged or retained failed) intent.
10848    fn table_intent(&self, table: &str) -> Result<(bool, bool), String> {
10849        let touches = |op: &OutboxOp| op.table == table;
10850        Ok((
10851            self.outbox
10852                .iter()
10853                .flat_map(|commit| &commit.ops)
10854                .any(touches),
10855            self.failed_commits
10856                .iter()
10857                .flat_map(|(commit, _)| &commit.ops)
10858                .any(touches)
10859                || self
10860                    .conn
10861                    .query_row(
10862                        "SELECT EXISTS(SELECT 1 FROM _syncular_acked_rows WHERE tbl=?1)",
10863                        [table],
10864                        |row| row.get(0),
10865                    )
10866                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
10867        ))
10868    }
10869
10870    /// §7.1 reconciliation after an import block or chunk. Keyed replay covers
10871    /// only pending intent, so a table with protected intent rebuilds; a
10872    /// `restore` restores the imported `keys` and every pending row first.
10873    fn reconcile_import(
10874        &mut self,
10875        table: &TableSchema,
10876        keys: &mut Vec<SqlValue>,
10877        protected: bool,
10878        restore: bool,
10879    ) -> Result<(), String> {
10880        if protected {
10881            return self.rebuild_overlay_if_dirty();
10882        }
10883        let pending = self.outbox.iter().flat_map(|commit| &commit.ops);
10884        if restore {
10885            self.restore_base_rows(table, keys, pending.clone())?;
10886        }
10887        self.apply_outbox_ops(pending.filter(|op| op.table == table.name))?;
10888        self.overlay_dirty.set(false);
10889        Ok(())
10890    }
10891
10892    /// Restore the visible rows of `keys` and of the rows `ops` touch in
10893    /// `table` from the base before the caller replays their intent. The
10894    /// caller owns the transaction and publishes its revision after replay.
10895    fn restore_base_rows<'a>(
10896        &self,
10897        table: &TableSchema,
10898        keys: &mut Vec<SqlValue>,
10899        ops: impl IntoIterator<Item = &'a OutboxOp>,
10900    ) -> Result<(), String> {
10901        let visible = visible_table(&table.name);
10902        let base = base_table(&table.name);
10903        let pk = quote_ident(&table.primary_key);
10904        let predicate = row_id_predicate(table);
10905        let mut lookup = self.conn.prepare_cached(&format!(
10906            "SELECT {pk} FROM {base} WHERE {predicate} UNION SELECT {pk} FROM {visible} WHERE {predicate}"
10907        )).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
10908        let mut deleted = HashSet::new();
10909        for op in ops.into_iter().filter(|op| op.table == table.name) {
10910            if op.upsert {
10911                if let Some(values) = &op.values {
10912                    let key = json_to_column_value(
10913                        &table.columns[table.pk_index],
10914                        values.get(&table.primary_key),
10915                    )?;
10916                    keys.push(owned_sql_value(RowParam::Cell(&key))?);
10917                }
10918            } else if deleted.insert(&op.row_id) {
10919                keys.extend(
10920                    lookup
10921                        .query_map([&op.row_id], |row| row.get::<_, SqlValue>(0))
10922                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
10923                        .collect::<Result<Vec<_>, _>>()
10924                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
10925                );
10926            }
10927        }
10928        // Delete the complete affected set before restoring any base value, so
10929        // old optimistic unique values cannot collide with the new server rows.
10930        for chunk in keys.chunks(400) {
10931            let holes = vec!["?"; chunk.len()].join(",");
10932            self.conn
10933                .prepare_cached(&format!("DELETE FROM {visible} WHERE {pk} IN ({holes})"))
10934                .and_then(|mut statement| statement.execute(rusqlite::params_from_iter(chunk)))
10935                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
10936        }
10937        let updates = table
10938            .columns
10939            .iter()
10940            .map(|column| column.name.as_str())
10941            .chain(std::iter::once("_syncular_version"))
10942            .chain(table.sealed_columns().iter().copied())
10943            .filter(|name| *name != table.primary_key)
10944            .map(|name| {
10945                let name = quote_ident(name);
10946                format!("{name}=excluded.{name}")
10947            })
10948            .collect::<Vec<_>>()
10949            .join(",");
10950        for chunk in keys.chunks(400) {
10951            let holes = vec!["?"; chunk.len()].join(",");
10952            self.conn.prepare_cached(&format!("INSERT INTO {visible} SELECT * FROM {base} WHERE {pk} IN ({holes}) ON CONFLICT ({pk}) DO UPDATE SET {updates}"))
10953                .and_then(|mut statement| statement.execute(rusqlite::params_from_iter(chunk)))
10954                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
10955        }
10956        Ok(())
10957    }
10958
10959    // -- application-authorized local purge ---------------------------------------
10960
10961    fn compile_local_data_purge(
10962        &self,
10963        input: &LocalDataPurgeInput,
10964    ) -> Result<(Vec<CompiledLocalDataPurgeTarget>, String), String> {
10965        let invalid = |message: String| format!("sync.invalid_request: {message}");
10966        if input.purge_id.is_empty()
10967            || input.purge_id.len() > 128
10968            || !is_local_operation_code_like(&input.purge_id)
10969        {
10970            return Err(invalid(
10971                "local purge purgeId must be a 1–128 character code-like identifier".to_owned(),
10972            ));
10973        }
10974        if input.targets.is_empty() || input.targets.len() > MAX_LOCAL_PURGE_TARGETS {
10975            return Err(invalid(format!(
10976                "local purge needs between 1 and {MAX_LOCAL_PURGE_TARGETS} targets"
10977            )));
10978        }
10979
10980        let mut deduplicated: BTreeMap<
10981            String,
10982            (CompiledLocalDataPurgeTarget, LocalDataPurgeTarget),
10983        > = BTreeMap::new();
10984        for target in &input.targets {
10985            let table = self.schema.table(&target.table).ok_or_else(|| {
10986                invalid(format!(
10987                    "local purge names unknown table {:?}",
10988                    target.table
10989                ))
10990            })?;
10991            if target.selectors.is_empty() || target.selectors.len() > MAX_LOCAL_PURGE_SELECTORS {
10992                return Err(invalid(format!(
10993                    "local purge target {:?} needs between 1 and {MAX_LOCAL_PURGE_SELECTORS} selectors",
10994                    target.table
10995                )));
10996            }
10997            let mut selectors = Vec::with_capacity(target.selectors.len());
10998            let mut canonical_selectors = BTreeMap::new();
10999            for (column_name, raw_values) in &target.selectors {
11000                let Some((column_index, column)) = table
11001                    .columns
11002                    .iter()
11003                    .enumerate()
11004                    .find(|(_, column)| column.name == *column_name)
11005                else {
11006                    return Err(invalid(format!(
11007                        "local purge target {:?} names unknown column {:?}",
11008                        target.table, column_name
11009                    )));
11010                };
11011                let encrypted = table
11012                    .encrypted_columns
11013                    .iter()
11014                    .any(|candidate| candidate.index == column_index);
11015                if column.ty != ColumnType::String || encrypted {
11016                    return Err(invalid(format!(
11017                        "local purge selector {:?}.{:?} must be a plaintext string column",
11018                        target.table, column_name
11019                    )));
11020                }
11021                if raw_values.is_empty() || raw_values.len() > MAX_LOCAL_PURGE_VALUES {
11022                    return Err(invalid(format!(
11023                        "local purge selector {:?}.{:?} needs between 1 and {MAX_LOCAL_PURGE_VALUES} values",
11024                        target.table, column_name
11025                    )));
11026                }
11027                let mut values = raw_values.clone();
11028                values.sort();
11029                values.dedup();
11030                if values.iter().any(|value| {
11031                    value.is_empty()
11032                        || value.len() > MAX_LOCAL_PURGE_VALUE_LENGTH
11033                        || !is_local_operation_code_like(value)
11034                }) {
11035                    return Err(invalid(format!(
11036                        "local purge selector values must be 1–{MAX_LOCAL_PURGE_VALUE_LENGTH} character code-like identifiers"
11037                    )));
11038                }
11039                selectors.push((column_name.clone(), values.clone()));
11040                canonical_selectors.insert(column_name.clone(), values);
11041            }
11042            let canonical = LocalDataPurgeTarget {
11043                table: target.table.clone(),
11044                selectors: canonical_selectors,
11045            };
11046            let key = serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
11047            deduplicated.insert(
11048                key,
11049                (
11050                    CompiledLocalDataPurgeTarget {
11051                        table: target.table.clone(),
11052                        selectors,
11053                    },
11054                    canonical,
11055                ),
11056            );
11057        }
11058        let targets = deduplicated
11059            .values()
11060            .map(|(compiled, _)| compiled.clone())
11061            .collect::<Vec<_>>();
11062        let canonical = deduplicated
11063            .values()
11064            .map(|(_, target)| target.clone())
11065            .collect::<Vec<_>>();
11066        let canonical_plan =
11067            serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
11068        Ok((targets, canonical_plan))
11069    }
11070
11071    fn local_purge_base_row_ids(
11072        &self,
11073        targets: &[CompiledLocalDataPurgeTarget],
11074    ) -> Result<BTreeMap<String, BTreeSet<String>>, String> {
11075        let mut by_table: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
11076        for target in targets {
11077            let table = self
11078                .schema
11079                .table(&target.table)
11080                .ok_or_else(|| format!("sync.invalid_request: unknown table {:?}", target.table))?;
11081            let mut clauses = Vec::with_capacity(target.selectors.len());
11082            let mut params = Vec::new();
11083            for (column, values) in &target.selectors {
11084                clauses.push(format!(
11085                    "{} IN ({})",
11086                    quote_ident(column),
11087                    values.iter().map(|_| "?").collect::<Vec<_>>().join(", ")
11088                ));
11089                params.extend(values.iter().cloned().map(SqlValue::Text));
11090            }
11091            let sql = format!(
11092                "SELECT CAST({} AS TEXT) FROM {} WHERE {}",
11093                quote_ident(&table.primary_key),
11094                base_table(&target.table),
11095                clauses.join(" AND ")
11096            );
11097            let mut statement = self
11098                .conn
11099                .prepare(&sql)
11100                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11101            let rows = statement
11102                .query_map(rusqlite::params_from_iter(params), |row| {
11103                    let value = sql_ref_to_json(&table.columns[table.pk_index], row.get_ref(0)?);
11104                    render_row_id_json(Some(&value))
11105                        .map_err(|message| rusqlite::Error::ToSqlConversionFailure(message.into()))
11106                })
11107                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11108            let ids = by_table.entry(target.table.clone()).or_default();
11109            for row in rows {
11110                ids.insert(
11111                    row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
11112                );
11113            }
11114        }
11115        Ok(by_table)
11116    }
11117
11118    fn local_purge_values_match(
11119        target: &CompiledLocalDataPurgeTarget,
11120        values: &Map<String, Value>,
11121    ) -> bool {
11122        target.selectors.iter().all(|(column, allowed)| {
11123            values
11124                .get(column)
11125                .and_then(Value::as_str)
11126                .is_some_and(|value| allowed.iter().any(|candidate| candidate == value))
11127        })
11128    }
11129
11130    /// Apply one host-authorized local security purge. The host MUST gate the
11131    /// corresponding subscriptions first; this primitive owns local SQLite
11132    /// cleanup and never grants/revokes server authority by itself.
11133    pub fn purge_local_data(
11134        &mut self,
11135        input: &LocalDataPurgeInput,
11136    ) -> Result<LocalDataPurgeResult, String> {
11137        let (targets, canonical_plan) = self.compile_local_data_purge(input)?;
11138        let meta_key = format!("localPurge:{}", input.purge_id);
11139        let applied_plan = self
11140            .conn
11141            .query_row(
11142                "SELECT value FROM _syncular_meta WHERE key = ?1",
11143                rusqlite::params![meta_key],
11144                |row| row.get::<_, String>(0),
11145            )
11146            .optional()
11147            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11148        if let Some(applied_plan) = applied_plan {
11149            if applied_plan != canonical_plan {
11150                return Err(format!(
11151                    "sync.invalid_request: local purge id {:?} was already used with a different plan",
11152                    input.purge_id
11153                ));
11154            }
11155            // A retry after a failed rewrite finishes it.
11156            self.erase_purged_pages()?;
11157            return Ok(LocalDataPurgeResult {
11158                already_applied: true,
11159                purged_rows: 0,
11160                dropped_commits: 0,
11161            });
11162        }
11163
11164        // RFC 0005 D7: the container file cannot be a purge target (its name is
11165        // not in the running schema), so purge drops it as a fixed step, before
11166        // any mirror row is touched and unconditionally.
11167        let _ = self.drop_previous_version();
11168
11169        let rollback = self.capture();
11170        self.begin_observation("syncular_local_purge")?;
11171        let applied = (|| -> Result<(ChangeAccumulator, LocalDataPurgeResult), String> {
11172            let row_ids = self.local_purge_base_row_ids(&targets)?;
11173            let retained_dropped = self.drop_failed_commits(|table, values| {
11174                targets.iter().any(|target| {
11175                    target.table == table && Self::local_purge_values_match(target, values)
11176                })
11177            })?;
11178            let doomed = self
11179                .outbox
11180                .iter()
11181                .filter(|commit| {
11182                    commit.ops.iter().any(|operation| {
11183                        let matching_targets = targets
11184                            .iter()
11185                            .filter(|target| target.table == operation.table)
11186                            .collect::<Vec<_>>();
11187                        if matching_targets.is_empty() {
11188                            return false;
11189                        }
11190                        if row_ids
11191                            .get(&operation.table)
11192                            .is_some_and(|ids| ids.contains(&operation.row_id))
11193                        {
11194                            return true;
11195                        }
11196                        operation.values.as_ref().is_some_and(|values| {
11197                            matching_targets
11198                                .iter()
11199                                .any(|target| Self::local_purge_values_match(target, values))
11200                        })
11201                    })
11202                })
11203                .cloned()
11204                .collect::<Vec<_>>();
11205            let doomed_ids = doomed
11206                .iter()
11207                .map(|commit| commit.client_commit_id.clone())
11208                .collect::<BTreeSet<_>>();
11209            let mut batch = ChangeAccumulator::default();
11210            for commit in &retained_dropped {
11211                batch.conflicts = true;
11212                batch.rejections = true;
11213                batch.outcomes = true;
11214                for operation in &commit.ops {
11215                    batch.table(&operation.table);
11216                }
11217            }
11218            let mut rejections = Vec::new();
11219            // §5.11 purge erasure: the journal keeps a dropped commit's identity and
11220            // codes, never its operations, so no purged value survives in it.
11221            for commit in &doomed {
11222                for operation in &commit.ops {
11223                    batch.table(&operation.table);
11224                    self.overlay_dirty.table(&operation.table);
11225                }
11226                let results = (0..commit.ops.len())
11227                    .map(|op_index| {
11228                        let rejection = RejectionRecord {
11229                            client_commit_id: commit.client_commit_id.clone(),
11230                            op_index: op_index as i32,
11231                            code: "client.local_data_purged".to_owned(),
11232                            message: "the commit was dropped by an application-authorized local data purge".to_owned(),
11233                            retryable: false,
11234                            details: None,
11235                            operation: None,
11236                        };
11237                        rejections.push(rejection.clone());
11238                        CommitOperationOutcome::Error { rejection }
11239                    })
11240                    .collect::<Vec<_>>();
11241                self.persist_commit_outcome(
11242                    &commit.client_commit_id,
11243                    CommitOutcomeStatus::Rejected,
11244                    &results,
11245                    None,
11246                )?;
11247                self.delete_outbox_persisted(&commit.client_commit_id)?;
11248            }
11249            if !doomed.is_empty() {
11250                self.outbox
11251                    .retain(|commit| !doomed_ids.contains(&commit.client_commit_id));
11252                self.rejections.extend(rejections);
11253                self.prune_commit_outcomes()?;
11254                batch.status = true;
11255                batch.rejections = true;
11256                batch.outcomes = true;
11257            }
11258
11259            let mut purged_rows = 0usize;
11260            for (table, ids) in &row_ids {
11261                if ids.is_empty() {
11262                    continue;
11263                }
11264                batch.table(table);
11265                self.overlay_dirty.table(table);
11266                for row_id in ids {
11267                    self.delete_base_row(table, row_id, i64::MAX)?;
11268                    purged_rows += 1;
11269                }
11270            }
11271            // §5.11 purge erasure: a journaled outcome with an operation or
11272            // conflict row the purge reaches keeps its identity, status, ids
11273            // and codes only.
11274            let touches = |table: &str, row_id: &str, values: Option<&Map<String, Value>>| {
11275                targets.iter().any(|target| {
11276                    target.table == table
11277                        && (row_ids.get(table).is_some_and(|ids| ids.contains(row_id))
11278                            || values.is_some_and(|values| {
11279                                Self::local_purge_values_match(target, values)
11280                            }))
11281                })
11282            };
11283            let entries = self
11284                .conn
11285                .prepare("SELECT client_commit_id, results_json, operations_json FROM _syncular_commit_outcomes")
11286                .and_then(|mut statement| {
11287                    statement
11288                        .query_map([], |row| {
11289                            Ok((
11290                                row.get::<_, String>(0)?,
11291                                row.get::<_, String>(1)?,
11292                                row.get::<_, Option<String>>(2)?,
11293                            ))
11294                        })?
11295                        .collect::<Result<Vec<_>, _>>()
11296                })
11297                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11298            for (client_commit_id, results_json, operations_json) in entries {
11299                let corrupt = |_| "sync.local_corrupt: persisted commit outcome is invalid".to_owned();
11300                let mut results: Vec<CommitOperationOutcome> =
11301                    serde_json::from_str(&results_json).map_err(corrupt)?;
11302                let operations: Vec<CommitOperation> = operations_json
11303                    .map(|json| serde_json::from_str(&json))
11304                    .transpose()
11305                    .map_err(corrupt)?
11306                    .unwrap_or_default();
11307                let named = operations
11308                    .iter()
11309                    .chain(results.iter().filter_map(|result| match result {
11310                        CommitOperationOutcome::Conflict { conflict } => conflict.operation.as_ref(),
11311                        CommitOperationOutcome::Error { rejection } => rejection.operation.as_ref(),
11312                        CommitOperationOutcome::Applied { .. } => None,
11313                    }))
11314                    .any(|operation| touches(&operation.table, &operation.row_id, operation.values.as_ref()))
11315                    || results.iter().any(|result| {
11316                        matches!(result, CommitOperationOutcome::Conflict { conflict }
11317                            if touches(&conflict.table, &conflict.row_id, Some(&conflict.server_row)))
11318                    });
11319                if !named {
11320                    continue;
11321                }
11322                for result in &mut results {
11323                    match result {
11324                        CommitOperationOutcome::Conflict { conflict } => {
11325                            conflict.server_row = Map::new();
11326                            conflict.conflict_columns.clear();
11327                            conflict.operation = None;
11328                        }
11329                        CommitOperationOutcome::Error { rejection } => {
11330                            rejection.details = None;
11331                            rejection.operation = None;
11332                        }
11333                        CommitOperationOutcome::Applied { .. } => {}
11334                    }
11335                }
11336                let results_json = serde_json::to_string(&results).map_err(|error| error.to_string())?;
11337                self.conn
11338                    .execute(
11339                        "UPDATE _syncular_commit_outcomes SET results_json = ?1, operations_json = NULL WHERE client_commit_id = ?2",
11340                        rusqlite::params![results_json, client_commit_id],
11341                    )
11342                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11343                batch.outcomes = true;
11344            }
11345            self.rebuild_overlay_if_dirty()?;
11346            self.delete_unreferenced_cached_blobs()?;
11347            self.conn
11348                .execute(
11349                    "INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
11350                    rusqlite::params![meta_key, canonical_plan],
11351                )
11352                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11353            Ok((
11354                batch,
11355                LocalDataPurgeResult {
11356                    already_applied: false,
11357                    purged_rows,
11358                    dropped_commits: doomed.len(),
11359                },
11360            ))
11361        })()
11362        .and_then(|(batch, result)| {
11363            self.finish_observation("syncular_local_purge", batch)
11364                .map(|()| result)
11365        });
11366        let result = match applied {
11367            Ok(result) => result,
11368            Err(error) => {
11369                self.rollback_observation("syncular_local_purge");
11370                self.restore(rollback);
11371                return Err(error);
11372            }
11373        };
11374        self.cancel_sync_round();
11375        self.erase_purged_pages()?;
11376        Ok(result)
11377    }
11378
11379    /// §5.11 purge erasure: SQLite leaves a deleted value in the freed pages of the
11380    /// database file, and the WAL keeps the frames written before the purge,
11381    /// until either is rewritten. VACUUM rebuilds the file from the remaining
11382    /// rows, and the checkpoint truncates the WAL.
11383    fn erase_purged_pages(&self) -> Result<(), String> {
11384        let erased = self
11385            .conn
11386            .execute_batch("PRAGMA secure_delete = ON; VACUUM;")
11387            .and_then(|()| {
11388                self.conn
11389                    .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))
11390            });
11391        let reset = self.conn.execute_batch("PRAGMA secure_delete = OFF");
11392        erased
11393            .and(reset)
11394            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
11395    }
11396
11397    /// Application-authorized recovery of the replicated projection. Unlike a
11398    /// local security purge, this retains the entire outbox, device identity,
11399    /// lease, outcomes, unit registrations, and protected bookkeeping.
11400    /// The projection reset, unit rewind, optimistic replay, and
11401    /// idempotency marker share one savepoint for interruption safety.
11402    pub fn rebootstrap_local_data(
11403        &mut self,
11404        input: &LocalDataRebootstrapInput,
11405    ) -> Result<LocalDataRebootstrapResult, String> {
11406        if input.rebootstrap_id.is_empty()
11407            || input.rebootstrap_id.len() > 128
11408            || !is_local_operation_code_like(&input.rebootstrap_id)
11409        {
11410            return Err(
11411                "sync.invalid_request: local rebootstrap rebootstrapId must be a 1–128 character code-like identifier"
11412                    .to_owned(),
11413            );
11414        }
11415        let meta_key = format!("localRebootstrap:{}", input.rebootstrap_id);
11416        if let Some(persisted) = meta_get(&self.conn, &meta_key).map_err(|_| {
11417            "sync.local_corrupt: persisted local rebootstrap receipt is unreadable".to_owned()
11418        })? {
11419            let (retained_commits, reset_subscriptions) =
11420                decode_local_rebootstrap_receipt(&persisted)?;
11421            return Ok(LocalDataRebootstrapResult {
11422                already_applied: true,
11423                retained_commits,
11424                reset_subscriptions,
11425            });
11426        }
11427        if self.stopped || self.schema_floor.is_some() {
11428            return Err(
11429                "sync.invalid_request: local rebootstrap cannot bypass an active schema-floor stop; update the application first"
11430                    .to_owned(),
11431            );
11432        }
11433
11434        let retained_commits = self.outbox.len();
11435        let reset_subscriptions = self.stored(load_units(&self.conn))?.len();
11436        let receipt = encode_local_rebootstrap_receipt(retained_commits, reset_subscriptions)?;
11437        let rollback = self.capture();
11438        self.begin_observation("syncular_local_rebootstrap")?;
11439        let mut batch = ChangeAccumulator::default();
11440        let applied = (|| -> Result<(), String> {
11441            self.run_schema_reset_observed(&mut batch, false)?;
11442            self.conn
11443                .execute(
11444                    "INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
11445                    rusqlite::params![meta_key, receipt],
11446                )
11447                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11448            self.sync_needed = true;
11449            self.sync_intent_queue.push_back(SyncIntent::Interactive);
11450            batch.status = true;
11451            Ok(())
11452        })()
11453        .and_then(|()| self.finish_observation("syncular_local_rebootstrap", batch));
11454        if let Err(error) = applied {
11455            self.rollback_observation("syncular_local_rebootstrap");
11456            self.restore(rollback);
11457            return Err(error);
11458        }
11459
11460        Ok(LocalDataRebootstrapResult {
11461            already_applied: false,
11462            retained_commits,
11463            reset_subscriptions,
11464        })
11465    }
11466
11467    // -- scope purge + doomed outbox (§3.3) ----------------------------------------
11468
11469    /// Delete base and visible rows owned by the effective scopes
11470    /// ([`local_scope_predicate`]); `Ok(false)` = no local scope-column
11471    /// mapping for a key (the fail-closed case). A SQLite failure is an error.
11472    fn purge_scope_rows(
11473        &mut self,
11474        table_name: &str,
11475        effective: &[(String, Vec<String>)],
11476        owners: &[Vec<(String, Vec<String>)>],
11477    ) -> Result<bool, String> {
11478        let Some(table) = self.schema.table(table_name) else {
11479            return Ok(false);
11480        };
11481        let Ok(predicate) = local_scope_predicate(table, effective, owners) else {
11482            return Ok(false);
11483        };
11484        let Some((predicate, params)) = predicate else {
11485            return Ok(true);
11486        };
11487        self.overlay_dirty.table(table_name);
11488        for source in [base_table(table_name), visible_table(table_name)] {
11489            self.conn
11490                .execute(
11491                    &format!("DELETE FROM {source} WHERE {predicate}"),
11492                    rusqlite::params_from_iter(&params),
11493                )
11494                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11495        }
11496        Ok(true)
11497    }
11498
11499    /// §3.3: drop pending commits whose upserts provably land in the
11500    /// revoked effective scopes and no other owner covers (row ownership)
11501    /// — whole-commit, never per-operation.
11502    fn drop_doomed_outbox(
11503        &mut self,
11504        table: &TableSchema,
11505        effective: &[(String, Vec<String>)],
11506        owners: &[Vec<(String, Vec<String>)>],
11507    ) -> Result<bool, String> {
11508        let doomed: Vec<_> =
11509            self.outbox
11510                .iter()
11511                .filter(|commit| {
11512                    commit.ops.iter().any(|op| {
11513                        op.upsert
11514                            && op.table == table.name
11515                            && op.values.as_ref().is_some_and(|values| {
11516                                owns_local_row(table, effective, owners, values)
11517                            })
11518                    })
11519                })
11520                .cloned()
11521                .collect();
11522        if doomed.is_empty() {
11523            return Ok(false);
11524        }
11525        let mut rejections = Vec::new();
11526        for commit in &doomed {
11527            let results = commit
11528                .ops
11529                .iter()
11530                .enumerate()
11531                .map(|(op_index, operation)| {
11532                    self.overlay_dirty.table(&operation.table);
11533                    let rejection = RejectionRecord {
11534                        client_commit_id: commit.client_commit_id.clone(),
11535                        op_index: op_index as i32,
11536                        code: "sync.scope_revoked".to_owned(),
11537                        message: "the commit was dropped because its effective scope was revoked"
11538                            .to_owned(),
11539                        retryable: false,
11540                        details: None,
11541                        operation: Some(CommitOperation::from(operation)),
11542                    };
11543                    rejections.push(rejection.clone());
11544                    CommitOperationOutcome::Error { rejection }
11545                })
11546                .collect::<Vec<_>>();
11547            self.persist_commit_outcome(
11548                &commit.client_commit_id,
11549                CommitOutcomeStatus::Rejected,
11550                &results,
11551                Some(&commit.ops),
11552            )?;
11553            self.delete_outbox_persisted(&commit.client_commit_id)?;
11554        }
11555        self.prune_commit_outcomes()?;
11556        let doomed_ids = doomed
11557            .iter()
11558            .map(|commit| commit.client_commit_id.as_str())
11559            .collect::<BTreeSet<_>>();
11560        self.outbox
11561            .retain(|commit| !doomed_ids.contains(commit.client_commit_id.as_str()));
11562        self.rejections.extend(rejections);
11563        Ok(true)
11564    }
11565
11566    // -- blobs (§5.9) ----------------------------------------------------------------
11567
11568    /// §5.9.7: hash bytes into the content address, cache them, queue the
11569    /// upload (flushed before the next push, B4). Returns the canonical
11570    /// BlobRef JSON `{blobId, byteLength, mediaType?}` for a `blob_ref`
11571    /// column value.
11572    pub fn upload_blob(
11573        &mut self,
11574        bytes: &[u8],
11575        media_type: Option<String>,
11576        name: Option<String>,
11577    ) -> Result<Value, String> {
11578        let blob_id = blob_id_for(bytes);
11579        let now = self.clock_now_ms();
11580        let tx = self
11581            .conn
11582            .transaction()
11583            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11584        tx.execute(
11585            "INSERT INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES (?,?,?,?,?)
11586             ON CONFLICT(blob_id) DO NOTHING",
11587            rusqlite::params![blob_id, bytes, bytes.len() as i64, media_type, now],
11588        )
11589        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11590        tx.execute(
11591            "INSERT OR IGNORE INTO _syncular_blob_uploads(blob_id, media_type, created_at_ms) VALUES (?,?,?)",
11592            rusqlite::params![blob_id, media_type, now],
11593        )
11594        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11595        tx.commit()
11596            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11597        self.enforce_blob_cache_cap();
11598        let mut obj = Map::new();
11599        obj.insert("blobId".to_owned(), Value::from(blob_id));
11600        obj.insert("byteLength".to_owned(), Value::from(bytes.len() as i64));
11601        if let Some(mt) = media_type {
11602            obj.insert("mediaType".to_owned(), Value::from(mt));
11603        }
11604        if let Some(n) = name {
11605            obj.insert("name".to_owned(), Value::from(n));
11606        }
11607        Ok(Value::Object(obj))
11608    }
11609
11610    /// §5.9.7: resolve an owned blob body without driver encoding. A
11611    /// content-addressed cache hit serves with no fetch (B1); a miss downloads
11612    /// (§5.9.5), verifies the address, caches the bytes, and returns them.
11613    pub fn fetch_blob_bytes(
11614        &mut self,
11615        transport: &mut dyn Transport,
11616        blob_id_or_ref: &str,
11617    ) -> Result<FetchedBlob, (String, String)> {
11618        let simple = |m: String| ("client.failed".to_owned(), m);
11619        // A BlobRef declares the size that caps the download (§5.9.5); a bare
11620        // blobId declares none.
11621        let (blob_id, byte_length) = if blob_id_or_ref.starts_with("sha256:") {
11622            (blob_id_or_ref.to_owned(), None)
11623        } else {
11624            let value: Value = serde_json::from_str(blob_id_or_ref)
11625                .map_err(|_| simple("blob ref is not JSON".to_owned()))?;
11626            (
11627                value
11628                    .get("blobId")
11629                    .and_then(Value::as_str)
11630                    .ok_or_else(|| simple("blob ref has no blobId".to_owned()))?
11631                    .to_owned(),
11632                value.get("byteLength").and_then(Value::as_u64),
11633            )
11634        };
11635        if let Some(cached) = self.get_cached_blob_bytes(&blob_id).map_err(simple)? {
11636            return Ok(cached);
11637        }
11638        if !self.transport_enabled {
11639            return Err(("sync.offline".into(), "transport gate is closed".into()));
11640        }
11641        // §5.9.5: propagate the server's blob.* code (blob.forbidden /
11642        // blob.not_found) verbatim so the harness can assert on it. The
11643        // authorized endpoint serves bytes inline OR (always-issue, presign
11644        // configured) a signed url the client fetches directly — no host auth,
11645        // no fall-through: failure => re-request (the caller's next fetch_blob_bytes).
11646        #[cfg(feature = "bench-internals")]
11647        let download_phase = self.benchmark_phases.start(Phase::BlobDownload);
11648        let bytes = match transport
11649            .blob_download(&blob_id, byte_length)
11650            .map_err(|e| (e.code, e.message))?
11651        {
11652            BlobDownload::Bytes(bytes) => bytes,
11653            BlobDownload::Url {
11654                url,
11655                url_expires_at_ms,
11656            } => {
11657                // §5.9.5: MUST NOT start a fetch at/past expiry.
11658                if url_expires_at_ms.is_some_and(|exp| exp <= self.clock_now_ms()) {
11659                    return Err((
11660                        "sync.segment_expired".to_owned(),
11661                        format!(
11662                            "blob url for {blob_id} expired before fetch — re-request mints a fresh url (§5.9.5)"
11663                        ),
11664                    ));
11665                }
11666                transport
11667                    .fetch_blob_url(&url, byte_length)
11668                    .map_err(|e| (e.code, e.message))?
11669            }
11670        };
11671        #[cfg(feature = "bench-internals")]
11672        drop(download_phase);
11673        #[cfg(feature = "bench-internals")]
11674        let validate_phase = self.benchmark_phases.start(Phase::BlobValidate);
11675        // §5.9.5 inherits §5.1: the declared size caps the body before the
11676        // content address is verified.
11677        if byte_length.is_some_and(|cap| bytes.len() as u64 > cap) {
11678            return Err((
11679                "sync.invalid_request".to_owned(),
11680                "blob exceeds its byteLength".to_owned(),
11681            ));
11682        }
11683        if blob_id_for(&bytes) != blob_id {
11684            return Err(simple(format!(
11685                "blob content address mismatch for {blob_id}"
11686            )));
11687        }
11688        #[cfg(feature = "bench-internals")]
11689        drop(validate_phase);
11690        #[cfg(feature = "bench-internals")]
11691        let insert_phase = self.benchmark_phases.start(Phase::BlobCacheInsert);
11692        let now = self.clock_now_ms();
11693        self.conn
11694            .execute(
11695                "INSERT OR IGNORE INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES (?,?,?,NULL,?)",
11696                rusqlite::params![blob_id, bytes, bytes.len() as i64, now],
11697            )
11698            .map_err(|e| simple(e.to_string()))?;
11699        self.conn
11700            .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))
11701            .map_err(|e| simple(e.to_string()))?;
11702        self.enforce_blob_cache_cap();
11703        #[cfg(feature = "bench-internals")]
11704        drop(insert_phase);
11705        self.get_cached_blob_bytes(&blob_id)
11706            .map_err(simple)?
11707            .ok_or_else(|| simple("blob cache write failed".to_owned()))
11708    }
11709
11710    fn get_cached_blob_bytes(&self, blob_id: &str) -> Result<Option<FetchedBlob>, String> {
11711        #[cfg(feature = "bench-internals")]
11712        let _phase = self.benchmark_phases.start(Phase::BlobCacheRead);
11713        let mut stmt = self
11714            .conn
11715            .prepare("SELECT bytes, byte_length, media_type FROM _syncular_blobs WHERE blob_id = ?")
11716            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11717        let mut rows = stmt
11718            .query(rusqlite::params![blob_id])
11719            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11720        if let Some(row) = rows
11721            .next()
11722            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
11723        {
11724            let bytes: Vec<u8> = row
11725                .get(0)
11726                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11727            let byte_length: i64 = row
11728                .get(1)
11729                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11730            let media_type: Option<String> = row
11731                .get(2)
11732                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11733            return Ok(Some(FetchedBlob {
11734                blob_id: blob_id.to_owned(),
11735                byte_length,
11736                bytes,
11737                media_type,
11738            }));
11739        }
11740        Ok(None)
11741    }
11742
11743    /// §5.9.7 B4: upload every queued blob before push.
11744    fn pending_blob_uploads(&self) -> Result<Vec<crate::round::PendingUpload>, TransportError> {
11745        let pending: Vec<(String, Option<String>)> = {
11746            let mut stmt = self
11747                .conn
11748                .prepare(
11749                    "SELECT blob_id, media_type FROM (
11750                      SELECT blob_id, media_type, created_at_ms FROM _syncular_blob_uploads
11751                      UNION ALL
11752                      SELECT DISTINCT r.blob_id, b.media_type, 9223372036854775807 AS created_at_ms
11753                      FROM _syncular_blob_commit_refs r LEFT JOIN _syncular_blobs b ON b.blob_id = r.blob_id
11754                      WHERE NOT EXISTS (SELECT 1 FROM _syncular_blob_uploads u WHERE u.blob_id = r.blob_id)
11755                    ) ORDER BY created_at_ms, blob_id",
11756                )
11757                .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
11758            let rows = stmt
11759                .query_map([], |row| {
11760                    Ok((row.get::<_, String>(0)?, row.get::<_, Option<String>>(1)?))
11761                })
11762                .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
11763            rows.collect::<Result<_, _>>()
11764                .map_err(|error| match error {
11765                    rusqlite::Error::InvalidColumnType(..)
11766                    | rusqlite::Error::FromSqlConversionFailure(..) => TransportError::new(
11767                        "sync.local_corrupt",
11768                        "Pending blob upload metadata is invalid",
11769                    ),
11770                    _ => TransportError::new("client.failed", error.to_string()),
11771                })?
11772        };
11773        let mut uploads = Vec::new();
11774        for (blob_id, media_type) in pending {
11775            let bytes = (|| {
11776                let stored: Option<(SqlValue, SqlValue)> = self
11777                    .conn
11778                    .query_row(
11779                        "SELECT bytes, byte_length FROM _syncular_blobs WHERE blob_id = ?",
11780                        rusqlite::params![blob_id],
11781                        |row| Ok((row.get(0)?, row.get(1)?)),
11782                    )
11783                    .optional()
11784                    .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
11785                let Some((SqlValue::Blob(bytes), SqlValue::Integer(byte_length))) = stored else {
11786                    return Err(TransportError::new(
11787                        "sync.local_corrupt",
11788                        "Pending blob upload body is missing or corrupt",
11789                    ));
11790                };
11791                if byte_length != bytes.len() as i64 || blob_id_for(&bytes) != blob_id {
11792                    return Err(TransportError::new(
11793                        "sync.local_corrupt",
11794                        "Pending blob upload body is missing or corrupt",
11795                    ));
11796                }
11797                Ok(bytes)
11798            })();
11799            let failed = bytes.is_err();
11800            uploads.push(crate::round::PendingUpload {
11801                id: blob_id,
11802                bytes,
11803                media_type,
11804            });
11805            if failed {
11806                break;
11807            }
11808        }
11809        Ok(uploads)
11810    }
11811
11812    fn visible_blob_ids_sql(&self) -> String {
11813        let mut selects = Vec::new();
11814        for table in &self.schema.tables {
11815            for column in table
11816                .columns
11817                .iter()
11818                .filter(|column| column.ty == ColumnType::BlobRef)
11819            {
11820                let identifier = quote_ident(&column.name);
11821                selects.push(format!(
11822                    "SELECT json_extract({identifier}, '$.blobId') AS blob_id FROM {} \
11823                     WHERE typeof({identifier}) = 'text' AND json_valid({identifier}) \
11824                     AND json_type({identifier}, '$.blobId') = 'text'",
11825                    quote_ident(&table.name),
11826                ));
11827            }
11828        }
11829        if selects.is_empty() {
11830            "SELECT NULL AS blob_id WHERE 0".to_owned()
11831        } else {
11832            selects.join(" UNION ")
11833        }
11834    }
11835
11836    /// §5.9.7 B1 size cap. Visible rows, pending commits, and
11837    /// staged uploads pin bodies directly; no derived refcount is stored.
11838    fn enforce_blob_cache_cap(&self) {
11839        let Some(max_bytes) = self.limits.blob_cache_max_bytes else {
11840            return;
11841        };
11842        let mut total: i64 = self
11843            .conn
11844            .query_row(
11845                "SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs",
11846                [],
11847                |row| row.get(0),
11848            )
11849            .unwrap_or(0);
11850        if total <= max_bytes {
11851            return;
11852        }
11853        let query = format!(
11854            "SELECT blob_id, byte_length FROM _syncular_blobs
11855             WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
11856               AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
11857               AND blob_id NOT IN ({})
11858             ORDER BY created_at_ms ASC, blob_id ASC",
11859            self.visible_blob_ids_sql(),
11860        );
11861        let candidates: Vec<(String, i64)> = {
11862            let Ok(mut stmt) = self.conn.prepare(&query) else {
11863                return;
11864            };
11865            let Ok(rows) = stmt.query_map([], |row| {
11866                Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?))
11867            }) else {
11868                return;
11869            };
11870            rows.filter_map(Result::ok).collect()
11871        };
11872        for (blob_id, byte_length) in candidates {
11873            if total <= max_bytes {
11874                break;
11875            }
11876            let _ = self.conn.execute(
11877                "DELETE FROM _syncular_blobs WHERE blob_id = ?",
11878                rusqlite::params![blob_id],
11879            );
11880            total -= byte_length;
11881        }
11882    }
11883
11884    fn delete_unreferenced_cached_blobs(&mut self) -> Result<(), String> {
11885        #[cfg(feature = "bench-internals")]
11886        let _phase = self.benchmark_phases.start(Phase::BlobRetention);
11887        if !self.schema_has_blobs() {
11888            return Ok(());
11889        }
11890        self.rebuild_overlay_if_dirty()?;
11891        self.conn
11892            .execute(
11893                &format!(
11894                    "DELETE FROM _syncular_blobs
11895                 WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
11896                   AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
11897                   AND blob_id NOT IN ({})",
11898                    self.visible_blob_ids_sql(),
11899                ),
11900                [],
11901            )
11902            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11903        Ok(())
11904    }
11905
11906    // -- local row storage ----------------------------------------------------------
11907
11908    /// The cached per-table primary-key upsert SQL for `full_table` (see
11909    /// the `insert_sql` field: built once, reused per row).
11910    fn insert_row_sql(&self, full_table: &str, table: &crate::schema::TableSchema) -> String {
11911        if let Some(sql) = self.insert_sql.borrow().get(full_table) {
11912            return sql.clone();
11913        }
11914        let mut columns: Vec<String> = table.columns.iter().map(|c| quote_ident(&c.name)).collect();
11915        columns.push(quote_ident("_syncular_version"));
11916        columns.extend(table.sealed_columns().iter().map(|name| quote_ident(name)));
11917        let placeholders: Vec<&str> = columns.iter().map(|_| "?").collect();
11918        let primary_key = quote_ident(&table.primary_key);
11919        let updates = columns
11920            .iter()
11921            .filter(|column| **column != primary_key)
11922            .map(|column| format!("{column}=excluded.{column}"))
11923            .collect::<Vec<_>>()
11924            .join(", ");
11925        // §4.5 version guard: server state lands in the base table only above
11926        // the stored version; the visible overlay takes every write.
11927        let guard = if full_table == base_table(&table.name) {
11928            format!(" WHERE excluded._syncular_version > {full_table}._syncular_version")
11929        } else {
11930            String::new()
11931        };
11932        let sql = format!(
11933            "INSERT INTO {full_table} ({}) VALUES ({}) ON CONFLICT ({primary_key}) DO UPDATE SET {updates}{guard}",
11934            columns.join(", "),
11935            placeholders.join(", ")
11936        );
11937        self.insert_sql
11938            .borrow_mut()
11939            .insert(full_table.to_owned(), sql.clone());
11940        sql
11941    }
11942
11943    /// §4.5: write a server row under the version guard; returns whether it
11944    /// landed. `sealed` is its §5.11 sealed state, `None` for an unsealed row.
11945    fn write_base_row(
11946        &self,
11947        table_name: &str,
11948        row: &Row,
11949        version: i64,
11950        sealed: Option<&SealedRow>,
11951    ) -> Result<bool, String> {
11952        let landed = self.write_row(&base_table(table_name), table_name, row, version, sealed)?;
11953        if landed {
11954            self.overlay_dirty.table(table_name);
11955        }
11956        Ok(landed)
11957    }
11958
11959    fn write_row(
11960        &self,
11961        full_table: &str,
11962        table_name: &str,
11963        row: &Row,
11964        version: i64,
11965        sealed: Option<&SealedRow>,
11966    ) -> Result<bool, String> {
11967        #[cfg(feature = "bench-internals")]
11968        let _phase = self.benchmark_phases.start(Phase::RowWrite);
11969        let table = self
11970            .schema
11971            .table(table_name)
11972            .ok_or_else(|| format!("unknown table {table_name:?}"))?;
11973        let sql = self.insert_row_sql(full_table, table);
11974        let mut stmt = self
11975            .conn
11976            .prepare_cached(&sql)
11977            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11978        let params = row
11979            .iter()
11980            .map(RowParam::Cell)
11981            .chain(std::iter::once(RowParam::Version(version)))
11982            .chain(
11983                [
11984                    sealed.map(|sealed| sealed.key_id.as_str()),
11985                    sealed.map(|sealed| sealed.envelopes.as_str()),
11986                ]
11987                .into_iter()
11988                .take(table.sealed_columns().len())
11989                .map(RowParam::Text),
11990            );
11991        stmt.execute(rusqlite::params_from_iter(params))
11992            .map(|changed| changed > 0)
11993            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
11994    }
11995
11996    /// §4.5: delete a server row whose stored version lies below `version`;
11997    /// returns whether a row left.
11998    fn delete_base_row(
11999        &self,
12000        table_name: &str,
12001        row_id: &str,
12002        version: i64,
12003    ) -> Result<bool, String> {
12004        let table = self
12005            .schema
12006            .table(table_name)
12007            .ok_or_else(|| format!("unknown table {table_name:?}"))?;
12008        let sql = format!(
12009            "DELETE FROM {} WHERE {} AND _syncular_version < ?2",
12010            base_table(table_name),
12011            row_id_predicate(table)
12012        );
12013        let mut stmt = self
12014            .conn
12015            .prepare_cached(&sql)
12016            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12017        let landed = stmt
12018            .execute(rusqlite::params![row_id, version])
12019            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
12020            > 0;
12021        if landed {
12022            self.overlay_dirty.table(table_name);
12023        }
12024        Ok(landed)
12025    }
12026
12027    fn has_acknowledged_rows(&self) -> Result<bool, String> {
12028        self.conn
12029            .query_row(
12030                "SELECT EXISTS(SELECT 1 FROM _syncular_acked_rows)",
12031                [],
12032                |row| row.get(0),
12033            )
12034            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
12035    }
12036
12037    fn acknowledged_operations(
12038        &self,
12039        rows: Option<&HashSet<(&str, &str)>>,
12040        tables: Option<&BTreeSet<&str>>,
12041    ) -> Result<Vec<OutboxOp>, String> {
12042        let keys = rows.map(|rows| serde_json::to_string(&rows).expect("row keys"));
12043        let mut statement = self
12044            .conn
12045            .prepare_cached(if keys.is_some() {
12046                ACKED_ROWS_BY_ROW
12047            } else {
12048                "SELECT op_json,intent_json FROM _syncular_acked_rows WHERE ?1 IS NULL AND (?2 IS NULL OR tbl IN(SELECT value FROM json_each(?2))) ORDER BY commit_seq,idx"
12049            })
12050            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12051        let rows = statement
12052            .query_map(
12053                rusqlite::params![
12054                    keys,
12055                    tables.map(|tables| serde_json::to_string(tables).expect("table keys"))
12056                ],
12057                |row| {
12058                    #[cfg(test)]
12059                    self.acknowledged_replay_count
12060                        .set(self.acknowledged_replay_count.get() + 1);
12061                    Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
12062                },
12063            )
12064            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12065        rows.map(|row| {
12066            let (op, intent) =
12067                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12068            let op: CommitOperation =
12069                serde_json::from_str(&op).map_err(|_| "sync.local_corrupt".to_owned())?;
12070            let table = self
12071                .schema
12072                .table(&op.table)
12073                .ok_or_else(|| "sync.unknown_table".to_owned())?;
12074            let values = if op.op == "upsert" && self.visible_row(table, &op.row_id)?.is_none() {
12075                Some(serde_json::from_str(&intent).map_err(|_| "sync.local_corrupt".to_owned())?)
12076            } else {
12077                op.values.clone()
12078            };
12079            Ok(OutboxOp {
12080                values,
12081                ..op.into()
12082            })
12083        })
12084        .collect()
12085    }
12086
12087    fn drop_acknowledged_rows(
12088        &self,
12089        matches: impl Fn(&str, &Map<String, Value>) -> bool,
12090    ) -> Result<Vec<OutboxCommit>, String> {
12091        let mut statement = self
12092            .conn
12093            .prepare("SELECT commit_id,tbl,intent_json,id FROM _syncular_acked_rows")
12094            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12095        let rows = statement
12096            .query_map([], |row| {
12097                Ok((
12098                    row.get::<_, String>(0)?,
12099                    row.get::<_, String>(1)?,
12100                    row.get::<_, String>(2)?,
12101                    row.get::<_, String>(3)?,
12102                ))
12103            })
12104            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12105        let mut doomed = HashSet::new();
12106        for row in rows {
12107            let (id, table, intent, row_id) =
12108                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12109            let values =
12110                serde_json::from_str(&intent).map_err(|_| "sync.local_corrupt".to_owned())?;
12111            let base_matches = self
12112                .schema
12113                .table(&table)
12114                .map(|schema| self.stored_row(schema, &row_id, true))
12115                .transpose()?
12116                .flatten()
12117                .is_some_and(|(base, _)| {
12118                    let schema = self.schema.table(&table).expect("acknowledged table");
12119                    let values = schema
12120                        .columns
12121                        .iter()
12122                        .zip(base.iter())
12123                        .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
12124                        .collect();
12125                    matches(&table, &values)
12126                });
12127            if matches(&table, &values) || base_matches {
12128                doomed.insert(id);
12129            }
12130        }
12131        let mut dropped = Vec::new();
12132        for id in doomed {
12133            let mut statement = self
12134                .conn
12135                .prepare("SELECT op_json FROM _syncular_acked_rows WHERE commit_id=?1 ORDER BY idx")
12136                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12137            let operations = statement
12138                .query_map([&id], |row| row.get::<_, String>(0))
12139                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12140            let mut ops = Vec::new();
12141            for operation in operations {
12142                let json = operation
12143                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12144                let op: CommitOperation =
12145                    serde_json::from_str(&json).map_err(|_| "sync.local_corrupt".to_owned())?;
12146                self.overlay_dirty.table(&op.table);
12147                ops.push(OutboxOp {
12148                    table: op.table,
12149                    row_id: op.row_id,
12150                    upsert: op.op == "upsert",
12151                    base_version: op.base_version,
12152                    values: op.values,
12153                    insert: op.insert,
12154                });
12155            }
12156            dropped.push(OutboxCommit {
12157                client_commit_id: id.clone(),
12158                ops,
12159            });
12160            self.conn
12161                .execute("DELETE FROM _syncular_acked_rows WHERE commit_id=?1", [id])
12162                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12163        }
12164        Ok(dropped)
12165    }
12166
12167    /// [`Self::rebuild_overlay`], skipped when neither the base tables nor
12168    /// the outbox changed since the last rebuild (the no-op sync round).
12169    fn rebuild_overlay_if_dirty(&mut self) -> Result<(), String> {
12170        if self.overlay_dirty.get() {
12171            self.rebuild_overlay()?;
12172        }
12173        Ok(())
12174    }
12175
12176    /// §7.1: local reads see outbox state applied optimistically — rebuild
12177    /// every visible table as (base server state) + (pending outbox replay
12178    /// on top). Optimistic rows carry version `-1`.
12179    fn rebuild_overlay(&mut self) -> Result<(), String> {
12180        #[cfg(feature = "bench-internals")]
12181        let _phase = self.benchmark_phases.start(Phase::OverlayRebuild);
12182        #[cfg(test)]
12183        self.overlay_rebuild_count
12184            .set(self.overlay_rebuild_count.get() + 1);
12185        let dirty = self.overlay_dirty.snapshot();
12186        let tables: Vec<_> = self
12187            .schema
12188            .tables
12189            .iter()
12190            .filter(|table| {
12191                dirty
12192                    .as_ref()
12193                    .is_none_or(|names| names.is_empty() || names.contains(&table.name))
12194            })
12195            .cloned()
12196            .collect();
12197        let names: BTreeSet<_> = tables.iter().map(|table| table.name.as_str()).collect();
12198        self.begin_observation("syncular_overlay")?;
12199        let rebuilt = (|| {
12200            for table in &tables {
12201                for index in &table.fts_indexes {
12202                    self.drop_fts_triggers(index)?;
12203                }
12204                let visible = visible_table(&table.name);
12205                let base = base_table(&table.name);
12206                self.conn
12207                    .execute_batch(&format!(
12208                        "DELETE FROM {visible}; INSERT INTO {visible} SELECT * FROM {base}"
12209                    ))
12210                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12211            }
12212            self.replay_overlay(&names, None)?;
12213            for table in &tables {
12214                for index in &table.fts_indexes {
12215                    self.rebuild_fts_projection(table, index)?;
12216                    self.create_fts_triggers(table, index)?;
12217                }
12218            }
12219            self.conn
12220                .execute_batch("RELEASE syncular_overlay")
12221                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12222            Ok(())
12223        })();
12224        if rebuilt.is_err() {
12225            self.conn
12226                .execute_batch("ROLLBACK TO syncular_overlay; RELEASE syncular_overlay")
12227                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12228        } else {
12229            self.overlay_dirty.set(false);
12230        }
12231        rebuilt
12232    }
12233
12234    /// §7.1 replay of acknowledged, retained failed and pending intent, in
12235    /// that order, for the operations of `tables` on `rows` (every row for
12236    /// `None`) over their current visible state.
12237    fn replay_overlay(
12238        &self,
12239        tables: &BTreeSet<&str>,
12240        rows: Option<&HashSet<(&str, &str)>>,
12241    ) -> Result<(), String> {
12242        let replays = |op: &&OutboxOp| {
12243            tables.contains(op.table.as_str())
12244                && rows.is_none_or(|rows| rows.contains(&(op.table.as_str(), op.row_id.as_str())))
12245        };
12246        self.apply_outbox_ops(&self.acknowledged_operations(rows, Some(tables))?)?;
12247        for (commit, initial) in self.failed_commits.iter() {
12248            for operation in commit.ops.iter().filter(replays) {
12249                let Some(table) = self.schema.table(&operation.table) else {
12250                    continue;
12251                };
12252                if operation.upsert
12253                    && self.visible_row(table, &operation.row_id)?.is_none()
12254                    && operation.presented_values(table).is_some_and(|values| {
12255                        table
12256                            .columns
12257                            .iter()
12258                            .all(|column| values.contains_key(&column.name))
12259                    })
12260                {
12261                    let key = serde_json::to_string(&(&operation.table, &operation.row_id))
12262                        .expect("row key serialization");
12263                    if let Some(Value::Object(values)) = initial.get(&key) {
12264                        self.apply_outbox_ops(&[OutboxOp {
12265                            values: Some(values.clone()),
12266                            ..operation.clone()
12267                        }])?;
12268                    }
12269                }
12270            }
12271            self.apply_outbox_ops(commit.ops.iter().filter(replays))?;
12272        }
12273        self.apply_outbox_ops(
12274            self.outbox
12275                .iter()
12276                .flat_map(|commit| &commit.ops)
12277                .filter(replays),
12278        )
12279    }
12280
12281    /// §7.1 replay: every operation is applied over the current visible
12282    /// state. An operation that no longer applies (a server row now holds its
12283    /// unique value) stays pending and invisible until a later replay admits
12284    /// it or the server answers its push.
12285    fn apply_outbox_ops<'a>(
12286        &self,
12287        ops: impl IntoIterator<Item = &'a OutboxOp>,
12288    ) -> Result<(), String> {
12289        #[cfg(feature = "bench-internals")]
12290        let mut phase = self.benchmark_phases.start(Phase::PendingReplay);
12291        for op in ops {
12292            #[cfg(feature = "bench-internals")]
12293            if let Some(phase) = &mut phase {
12294                phase.unit();
12295            }
12296            if op.upsert {
12297                if let Some((table, values)) = self
12298                    .schema
12299                    .table(&op.table)
12300                    .filter(|table| table.indexes.iter().any(|index| index.unique))
12301                    .and_then(|table| Some((table, op.presented_values(table)?)))
12302                {
12303                    let mut intended: Map<String, Value> = self
12304                        .visible_row(table, &op.row_id)?
12305                        .map(|(row, _)| {
12306                            table
12307                                .columns
12308                                .iter()
12309                                .zip(&row)
12310                                .map(|(column, value)| {
12311                                    (column.name.clone(), column_value_to_json(value))
12312                                })
12313                                .collect()
12314                        })
12315                        .unwrap_or_default();
12316                    intended.extend(values.into_owned());
12317                    if !self.unique_conflicts(table, &intended, false)?.is_empty() {
12318                        continue;
12319                    }
12320                }
12321            }
12322            self.apply_outbox_op(op)?;
12323        }
12324        Ok(())
12325    }
12326
12327    fn apply_outbox_op(&self, op: &OutboxOp) -> Result<(), String> {
12328        let Some(table) = self.schema.table(&op.table) else {
12329            return Ok(());
12330        };
12331        let visible = visible_table(&table.name);
12332        if !op.upsert {
12333            let sql = format!("DELETE FROM {visible} WHERE {}", row_id_predicate(table));
12334            self.conn
12335                .prepare_cached(&sql)
12336                .and_then(|mut statement| statement.execute(rusqlite::params![op.row_id]))
12337                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12338            return Ok(());
12339        }
12340        // §7.1: an insert-intent operation that lacks a non-nullable column
12341        // cannot encode; it stays invisible until the send-time prepass drops
12342        // it as sync.outbox_incompatible (§7.4.4).
12343        let Some(values) = op.presented_values(table) else {
12344            return Ok(());
12345        };
12346        // §7.1: the overlay applies the operation's PRESENT columns
12347        // over the current local row. A partial operation over an
12348        // absent local row leaves it absent — the server answers it
12349        // with sync.row_deleted or sync.row_missing (§6.2). An
12350        // insert-intent operation presents every column.
12351        let full = table
12352            .columns
12353            .iter()
12354            .all(|column| values.contains_key(&column.name));
12355        let local = self.visible_row(table, &op.row_id)?;
12356        if local.is_none() && !full {
12357            return Ok(());
12358        }
12359        let mut row: Row = Vec::with_capacity(table.columns.len());
12360        for (index, column) in table.columns.iter().enumerate() {
12361            let incoming = match values.get(&column.name) {
12362                Some(raw) => json_to_column_value(
12363                    if raw.get("$bytes").is_some()
12364                        && table
12365                            .encrypted_columns
12366                            .iter()
12367                            .any(|encrypted| encrypted.index == index)
12368                    {
12369                        &table.wire_columns[index]
12370                    } else {
12371                        column
12372                    },
12373                    Some(raw),
12374                ),
12375                None => Ok(local
12376                    .as_ref()
12377                    .and_then(|(values, _)| values.get(index).cloned())
12378                    .flatten()),
12379            };
12380            row.push(incoming.map_err(|_| {
12381                "sync.local_corrupt: persisted optimistic row value is invalid".to_owned()
12382            })?);
12383        }
12384        let version = local.as_ref().map_or(-1, |(_, version)| *version);
12385        self.write_row(&visible, &table.name, &row, version, None)
12386            .map(|_| ())
12387            .map_err(|error| {
12388                if error.starts_with("UNIQUE constraint failed:") {
12389                    "sync.constraint_violation: local write violates a unique constraint".to_owned()
12390                } else {
12391                    error
12392                }
12393            })
12394    }
12395
12396    /// Whether the row is present in the overlay. Existence reads only the
12397    /// key: locked encrypted values stay opaque, and decoding the complete row
12398    /// would reject a valid plain write.
12399    fn visible_row_exists(&self, table: &TableSchema, row_id: &str) -> Result<bool, String> {
12400        self.conn
12401            .query_row(
12402                &format!(
12403                    "SELECT EXISTS(SELECT 1 FROM {} WHERE {})",
12404                    visible_table(&table.name),
12405                    row_id_predicate(table)
12406                ),
12407                rusqlite::params![row_id],
12408                |row| row.get(0),
12409            )
12410            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
12411    }
12412
12413    /// §5.11: a sealed row is read-only; a write could otherwise send NULL
12414    /// over ciphertext this device cannot see.
12415    fn refuse_sealed_row(&self, table: &TableSchema, row_id: &str) -> Result<(), ClientError> {
12416        if !table.has_encrypted_columns() {
12417            return Ok(());
12418        }
12419        let key_id: Option<String> = self
12420            .conn
12421            .query_row(
12422                &format!(
12423                    "SELECT \"_sync_sealed_key\" FROM {} WHERE {}",
12424                    visible_table(&table.name),
12425                    row_id_predicate(table)
12426                ),
12427                rusqlite::params![row_id],
12428                |row| row.get(0),
12429            )
12430            .optional()
12431            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
12432            .flatten();
12433        match key_id {
12434            None => Ok(()),
12435            Some(key_id) => Err(ClientError {
12436                code: "client.encryption_key_missing".to_owned(),
12437                message: "the row is sealed under a key this device does not hold".to_owned(),
12438                details: Some(json!({ "table": table.name, "keyId": key_id })),
12439                retryable: false,
12440            }),
12441        }
12442    }
12443
12444    /// §5.11 reveal: decrypt every sealed base row whose envelope keys the
12445    /// keyring now holds into its columns and clear both reserved columns,
12446    /// one change batch per table. `client.decrypt_failed` fails the caller.
12447    fn reveal_sealed_rows(&mut self) -> Result<(), TransportError> {
12448        let storage = |message: String| TransportError::new("storage.failed", message);
12449        let tables: Vec<TableSchema> = self
12450            .schema
12451            .tables
12452            .iter()
12453            .filter(|table| table.has_encrypted_columns())
12454            .cloned()
12455            .collect();
12456        for table in &tables {
12457            let base = base_table(&table.name);
12458            let pk = quote_ident(&table.primary_key);
12459            let sealed: Vec<(SqlValue, i64, String)> = self
12460                .conn
12461                .prepare_cached(&format!(
12462                    "SELECT {pk}, \"_syncular_version\", \"_sync_sealed\" FROM {base} WHERE \"_sync_sealed_key\" IS NOT NULL"
12463                ))
12464                .and_then(|mut statement| {
12465                    statement
12466                        .query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))?
12467                        .collect()
12468                })
12469                .map_err(|error| storage(Self::sqlite_failure(&self.storage_failure, error)))?;
12470            let mut revealed = Vec::new();
12471            for (key, version, envelopes) in sealed {
12472                if let Some(row) =
12473                    crate::values::reveal_sealed_row(table, &envelopes, &self.encryption)?
12474                {
12475                    revealed.push((key, version, row));
12476                }
12477            }
12478            if revealed.is_empty() {
12479                continue;
12480            }
12481            self.begin_observation("syncular_reveal").map_err(storage)?;
12482            let outcome = (|| {
12483                let assignments = table
12484                    .encrypted_columns
12485                    .iter()
12486                    .map(|enc| format!("{}=?", quote_ident(&table.columns[enc.index].name)))
12487                    .collect::<Vec<_>>()
12488                    .join(",");
12489                let mut statement = self
12490                    .conn
12491                    .prepare_cached(&format!(
12492                        "UPDATE {base} SET {assignments},\"_sync_sealed_key\"=NULL,\"_sync_sealed\"=NULL WHERE {pk}=? AND \"_syncular_version\"=?"
12493                    ))
12494                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12495                for (key, version, row) in &revealed {
12496                    let mut params = table
12497                        .encrypted_columns
12498                        .iter()
12499                        .map(|enc| owned_sql_value(RowParam::Cell(&row[enc.index])))
12500                        .collect::<Result<Vec<_>, _>>()?;
12501                    params.push(key.clone());
12502                    params.push(SqlValue::Integer(*version));
12503                    statement
12504                        .execute(rusqlite::params_from_iter(params))
12505                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12506                }
12507                drop(statement);
12508                self.overlay_dirty.table(&table.name);
12509                self.rebuild_overlay()?;
12510                let mut batch = ChangeAccumulator::default();
12511                batch.table(&table.name);
12512                self.finish_observation("syncular_reveal", batch)
12513            })();
12514            if outcome.is_err() {
12515                self.rollback_observation("syncular_reveal");
12516            }
12517            outcome.map_err(storage)?;
12518        }
12519        Ok(())
12520    }
12521
12522    /// The visible (optimistic) row's values plus version, or `None` when the
12523    /// row is absent from the overlay.
12524    fn visible_row(&self, table: &TableSchema, row_id: &str) -> Result<Option<(Row, i64)>, String> {
12525        self.stored_row(table, row_id, false)
12526    }
12527
12528    fn stored_row(
12529        &self,
12530        table: &TableSchema,
12531        row_id: &str,
12532        base: bool,
12533    ) -> Result<Option<(Row, i64)>, String> {
12534        let sql = format!(
12535            "SELECT * FROM {} WHERE {}",
12536            if base {
12537                base_table(&table.name)
12538            } else {
12539                visible_table(&table.name)
12540            },
12541            row_id_predicate(table)
12542        );
12543        let mut stmt = self
12544            .conn
12545            .prepare_cached(&sql)
12546            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12547        let mut rows = stmt
12548            .query(rusqlite::params![row_id])
12549            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12550        let Some(row) = rows
12551            .next()
12552            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
12553        else {
12554            return Ok(None);
12555        };
12556        let mut values = Vec::with_capacity(table.columns.len());
12557        for (index, column) in table.columns.iter().enumerate() {
12558            let raw = row
12559                .get_ref(index)
12560                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12561            if let rusqlite::types::ValueRef::Blob(bytes) = raw {
12562                if table
12563                    .encrypted_columns
12564                    .iter()
12565                    .any(|encrypted| encrypted.index == index)
12566                {
12567                    values.push(Some(ColumnValue::Bytes(bytes.to_vec())));
12568                    continue;
12569                }
12570            }
12571            let json = sql_ref_to_json(column, raw);
12572            values.push(json_to_column_value(column, Some(&json)).map_err(|_| {
12573                "sync.local_corrupt: persisted local row value is invalid".to_owned()
12574            })?);
12575        }
12576        let version: i64 = row
12577            .get(table.columns.len())
12578            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12579        Ok(Some((values, version)))
12580    }
12581
12582    // -- realtime (§8) ---------------------------------------------------------------
12583
12584    /// §8.8: the configured realtime policy.
12585    #[must_use]
12586    pub fn realtime_policy(&self) -> RealtimePolicy {
12587        self.realtime_policy
12588    }
12589
12590    /// §8.8: the explicit realtime availability state.
12591    #[must_use]
12592    pub fn realtime_state(&self) -> RealtimeState {
12593        self.realtime_state
12594    }
12595
12596    /// §8.8: set the realtime policy. Refused while the socket is connected:
12597    /// a policy change never silently re-routes an ownership state.
12598    pub fn set_realtime_policy(&mut self, policy: RealtimePolicy) -> Result<(), String> {
12599        if self.realtime_state == RealtimeState::Connected && policy != self.realtime_policy {
12600            return Err(
12601                "sync.invalid_request: disconnect realtime before changing the realtime policy"
12602                    .to_owned(),
12603            );
12604        }
12605        self.realtime_policy = policy;
12606        match policy {
12607            RealtimePolicy::Off => self.set_realtime_state(RealtimeState::Disabled, None),
12608            RealtimePolicy::Required | RealtimePolicy::Optional
12609                if self.realtime_state == RealtimeState::Disabled =>
12610            {
12611                self.set_realtime_state(RealtimeState::Disconnected, None);
12612            }
12613            RealtimePolicy::Required | RealtimePolicy::Optional => {}
12614        }
12615        Ok(())
12616    }
12617
12618    /// Record an availability transition, keeping the reason code bounded and
12619    /// code-like (§7.6) and clearing it where it cannot apply.
12620    fn set_realtime_state(&mut self, state: RealtimeState, reason_code: Option<&str>) {
12621        self.realtime_state = state;
12622        self.realtime_reason_code = reason_code.map(Self::diagnostic_code);
12623        if matches!(state, RealtimeState::Connected | RealtimeState::Disabled) {
12624            self.realtime_reason_code = None;
12625            self.realtime_retry_delay_ms = None;
12626        }
12627    }
12628
12629    #[must_use]
12630    fn realtime_connected(&self) -> bool {
12631        self.realtime_state == RealtimeState::Connected
12632    }
12633
12634    pub fn connect_realtime(&mut self, transport: &mut dyn Transport) -> Result<(), String> {
12635        if !self.transport_enabled {
12636            return Err("sync.offline: transport gate is closed".into());
12637        }
12638        if self.realtime_policy == RealtimePolicy::Off {
12639            return Err(
12640                "sync.invalid_request: realtime is off; realtime connect is refused".to_owned(),
12641            );
12642        }
12643        if self.realtime_connected() {
12644            return Ok(());
12645        }
12646        self.realtime_state = RealtimeState::Connecting;
12647        match transport.realtime_connect_for_client(&self.client_id) {
12648            Ok(()) => {
12649                self.set_realtime_state(RealtimeState::Connected, None);
12650                Ok(())
12651            }
12652            Err(error) => {
12653                self.set_realtime_state(RealtimeState::Refused, Some(&error.code));
12654                Err(format!("{}: {}", error.code, error.message))
12655            }
12656        }
12657    }
12658
12659    pub fn disconnect_realtime(&mut self, transport: &mut dyn Transport) {
12660        self.cancel_sync_round();
12661        // A deliberately disconnected or policy-disabled client has nothing
12662        // to release; a `lost` socket still does (the host owns the handle).
12663        if self.realtime_state == RealtimeState::Disconnected
12664            || self.realtime_state == RealtimeState::Disabled
12665        {
12666            return;
12667        }
12668        let _ = transport.realtime_close();
12669        self.set_realtime_state(
12670            if self.realtime_policy == RealtimePolicy::Off {
12671                RealtimeState::Disabled
12672            } else {
12673                RealtimeState::Disconnected
12674            },
12675            None,
12676        );
12677        self.presence.clear(); // §8.6.1: presence is per-connection
12678    }
12679
12680    /// §8.8: the host observed the socket end without disconnecting it (a
12681    /// peer close, an I/O error, a failed send; the native transport reports
12682    /// these as `Inbound::Disconnected`). Releases the transport, moves a
12683    /// `Connected` state to `Lost` with the transport's code as the reason, and schedules
12684    /// one background retry intent so the host reconnects and catches up.
12685    pub fn realtime_lost(&mut self, transport: &mut dyn Transport, error: &TransportError) {
12686        if self.realtime_state != RealtimeState::Connected {
12687            return;
12688        }
12689        self.cancel_sync_round();
12690        let _ = transport.realtime_close();
12691        self.set_realtime_state(RealtimeState::Lost, Some(&error.code));
12692        self.realtime_retry_delay_ms = Some(self.schedule_background_retry());
12693        self.presence.clear();
12694    }
12695
12696    /// §8.6.2: publish (or clear, `doc: None`) this client's presence
12697    /// document for `scope_key`. Requires a live socket; the document is
12698    /// ephemeral (lost on disconnect). Authorization is the connection's
12699    /// registration (§8.6.3) — an unheld key is rejected loudly by the
12700    /// server with `presence.forbidden`.
12701    pub fn set_presence(
12702        &mut self,
12703        transport: &mut dyn Transport,
12704        scope_key: &str,
12705        doc: Option<&Value>,
12706    ) -> Result<(), String> {
12707        if !self.transport_enabled {
12708            return Err("sync.offline: transport gate is closed".into());
12709        }
12710        if !self.realtime_connected() {
12711            return Err("setPresence requires a connected realtime socket (§8.6)".to_string());
12712        }
12713        let text = encode_presence_publish(scope_key, doc);
12714        transport
12715            .realtime_send(&text)
12716            .map_err(|e| format!("{}: {}", e.code, e.message))
12717    }
12718
12719    /// §8.6: the peers currently present on a scope key (ephemeral).
12720    pub fn presence(&self, scope_key: &str) -> Vec<PresencePeer> {
12721        self.presence
12722            .get(scope_key)
12723            .map(|peers| peers.values().cloned().collect())
12724            .unwrap_or_default()
12725    }
12726
12727    /// §8.6 apply an inbound presence fanout to the local map.
12728    fn apply_presence(
12729        &mut self,
12730        scope_key: String,
12731        kind: Option<PresenceKind>,
12732        actor_id: Option<String>,
12733        client_id: Option<String>,
12734        doc: Option<Value>,
12735        error: Option<String>,
12736    ) {
12737        // A refused publish (§8.6.2) changes no peer; the native host surfaces
12738        // it as a `presence` event carrying the server's `error` code.
12739        if error.is_some() {
12740            return;
12741        }
12742        let (Some(kind), Some(actor_id), Some(client_id)) = (kind, actor_id, client_id) else {
12743            return;
12744        };
12745        let peer_key = format!("{actor_id} {client_id}");
12746        match kind {
12747            PresenceKind::Leave => {
12748                if let Some(peers) = self.presence.get_mut(&scope_key) {
12749                    peers.remove(&peer_key);
12750                    if peers.is_empty() {
12751                        self.presence.remove(&scope_key);
12752                    }
12753                }
12754            }
12755            _ => {
12756                let doc = match doc {
12757                    Some(Value::Object(_)) => doc.unwrap(),
12758                    _ => return,
12759                };
12760                self.presence.entry(scope_key).or_default().insert(
12761                    peer_key,
12762                    PresencePeer {
12763                        actor_id,
12764                        client_id,
12765                        doc,
12766                    },
12767                );
12768            }
12769        }
12770    }
12771
12772    /// Inbound JSON control message (§8.1). Unknown events are tolerated.
12773    pub fn on_realtime_text(&mut self, text: &str) {
12774        if self.stopped || !self.transport_enabled || self.security_preflight {
12775            return;
12776        }
12777        match parse_control(text) {
12778            Ok(ControlMessage::Hello { requires_sync, .. }) => {
12779                if requires_sync {
12780                    // §8.1: pull before trusting the socket for continuity.
12781                    self.set_sync_needed(true, true);
12782                }
12783            }
12784            Ok(ControlMessage::Presence {
12785                scope_key,
12786                kind,
12787                actor_id,
12788                client_id,
12789                doc,
12790                error,
12791                ..
12792            }) => {
12793                self.apply_presence(scope_key, kind, actor_id, client_id, doc, error);
12794            }
12795            Ok(ControlMessage::Wake { .. }) => {
12796                // §8.3: any wake-up means "run a pull soon", never data.
12797                self.set_sync_needed(true, true);
12798            }
12799            _ => {}
12800        }
12801    }
12802
12803    /// Inbound binary delta: a complete SSP2 response (§8.2), applied like
12804    /// a pull response per section; an unapplied delta is a wake-up.
12805    /// Mailbox hosts send durable acknowledgements on their I/O executor.
12806    pub fn on_realtime_binary_queued(&mut self, bytes: &[u8]) -> Vec<String> {
12807        let mut transport = crate::round::DownloadResults::default();
12808        self.on_realtime_binary(&mut transport, bytes);
12809        transport.controls
12810    }
12811
12812    /// §8.2: a delta applies exactly like a live section and sets the replica
12813    /// cursor to its `throughSeq`; one at or below the replica cursor is a
12814    /// duplicate. A delta the client does not apply (no live unit, a status
12815    /// other than `active`, a decode or apply failure) is a drop and raises
12816    /// sync-needed.
12817    pub fn on_realtime_binary(&mut self, transport: &mut dyn Transport, bytes: &[u8]) {
12818        if self.stopped || !self.transport_enabled || self.security_preflight {
12819            return;
12820        }
12821        #[cfg(feature = "bench-internals")]
12822        let decode_phase = self.benchmark_phases.start(Phase::ResponseDecode);
12823        let message = match decode_message(bytes) {
12824            Ok(m) if m.msg_kind == MsgKind::Response => m,
12825            _ => {
12826                self.set_sync_needed(true, true);
12827                return;
12828            }
12829        };
12830        #[cfg(feature = "bench-internals")]
12831        drop(decode_phase);
12832        // §8.2: a delta's header means what a pull response's header means. A
12833        // schema floor (§1.6) is recorded, stops syncing and applies nothing.
12834        if let Some(Frame::RespHeader {
12835            required_schema_version: Some(required),
12836            latest_schema_version,
12837            ..
12838        }) = message.frames.first()
12839        {
12840            let floor = SchemaFloor {
12841                required_schema_version: Some(*required),
12842                latest_schema_version: *latest_schema_version,
12843            };
12844            if self.set_schema_floor(Some(floor)).is_err() {
12845                self.set_sync_needed(true, true);
12846            }
12847            return;
12848        }
12849        let through = message.frames.iter().find_map(|frame| match frame {
12850            Frame::LiveEnd { through_seq, .. } => Some(*through_seq),
12851            _ => None,
12852        });
12853        let epoch = self.get_meta(LOG_EPOCH_KEY).ok().flatten();
12854        let mut dropped = !matches!(message.frames.first(), Some(Frame::RespHeader {
12855                reset_required: false,
12856                log_epoch,
12857                ..
12858            }) if Some(log_epoch) == epoch.as_ref());
12859        let mut applied: Option<i64> = None;
12860        dropped |= self.begin_coalesced_apply().is_err();
12861        let mut receivers: Option<Vec<UnitRecord>> = None;
12862        for frame in message.frames {
12863            if dropped {
12864                break;
12865            }
12866            match frame {
12867                Frame::LiveStart { status, .. } => {
12868                    let Ok(cursor) = self.stored(replica_cursor(&self.conn)) else {
12869                        dropped = true;
12870                        break;
12871                    };
12872                    if status != LiveStatus::Active || cursor < 0 {
12873                        dropped = true;
12874                    } else if !through.is_some_and(|through| through <= cursor) {
12875                        // A catching-up unit receives no delta (§8.2).
12876                        receivers = self.stored(load_units(&self.conn)).ok().map(|units| {
12877                            units
12878                                .into_iter()
12879                                .filter(|unit| {
12880                                    unit.status == UnitStatus::Active
12881                                        && unit.echo.is_some()
12882                                        && unit.cursor.is_none()
12883                                })
12884                                .collect()
12885                        });
12886                        dropped = receivers.is_none();
12887                    }
12888                }
12889                Frame::Commit {
12890                    commit_seq,
12891                    tables,
12892                    changes,
12893                    ..
12894                } => {
12895                    let Some(receivers) = receivers.clone() else {
12896                        continue;
12897                    };
12898                    dropped = self
12899                        .apply_live_commit(
12900                            &receivers,
12901                            &tables,
12902                            changes,
12903                            commit_seq,
12904                            AppliedServerCommitSource::Delta,
12905                        )
12906                        .is_err();
12907                }
12908                Frame::LiveEnd { through_seq, .. } if receivers.is_some() => {
12909                    dropped = self
12910                        .set_meta(REPLICA_CURSOR_KEY, &through_seq.to_string())
12911                        .is_err();
12912                    applied = Some(through_seq);
12913                }
12914                Frame::Error { .. } => dropped = true,
12915                _ => {}
12916            }
12917        }
12918        // §1.4 rule 4: the ack follows the COMMIT. A failed COMMIT restored
12919        // the replica cursor, so the delta counts as a drop.
12920        if self.finish_coalesced_apply().is_err() || dropped {
12921            applied = None;
12922            dropped = true;
12923        }
12924        if let Some(cursor) = applied {
12925            // §8.2 ack point: the delta's throughSeq.
12926            let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
12927            let _ = transport.realtime_send(&ack);
12928        }
12929        if dropped {
12930            self.set_sync_needed(true, true);
12931        }
12932    }
12933
12934    /// §8.2 ack point after a pull round on a live connection: the replica
12935    /// cursor while it is at least `0`.
12936    fn ack_after_pull(&mut self, transport: &mut dyn Transport) {
12937        if !self.realtime_connected() {
12938            return;
12939        }
12940        if let Ok(cursor @ 0..) = self.stored(replica_cursor(&self.conn)) {
12941            let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
12942            let _ = transport.realtime_send(&ack);
12943        }
12944    }
12945}
12946
12947#[cfg(test)]
12948mod previous_version_wiring_tests;
12949
12950#[cfg(test)]
12951mod identity_tests;
12952
12953#[cfg(test)]
12954mod insert_intent_tests;
12955
12956#[cfg(test)]
12957mod robustness_tests;
12958
12959#[cfg(test)]
12960mod range_image_tests;