Skip to main content

syncular_client/
client.rs

1//! The Syncular v2 Rust client core (SPEC.md client-behavior contract):
2//! rusqlite local storage, §3.2/§3.3 effective-scope persistence + purge,
3//! §4 pull/cursor/bootstrap (§4.7 resume, §5.6 segment application), §6
4//! push with outbox order, §7 optimistic apply / rollback / replay-on-top,
5//! §2.3 clientCommitId idempotency, §8 realtime client rules, §10 errors.
6//!
7//! Built from `SPEC.md` and the committed `ssp2` codec alone — no
8//! reference to the v1 Rust tree or the v2 TypeScript client.
9
10use crate::{ProgressObserver, ProgressPhase, ProgressState};
11use ssp2::encode_message;
12use std::borrow::Cow;
13#[cfg(test)]
14use std::cell::Cell;
15use std::cell::RefCell;
16use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
17use std::sync::Arc;
18
19use rusqlite::types::{ToSqlOutput, Value as SqlValue, ValueRef};
20use rusqlite::{Connection, OpenFlags, OptionalExtension};
21use serde::{Deserialize, Serialize};
22use serde_json::{json, Map, Value};
23use sha2::{Digest, Sha256};
24use ssp2::decode::WIRE_VERSION;
25use ssp2::model::{
26    FencedRecord, Frame, LiveStatus, MediaType, Message, MsgKind, Op, OpResult, PushStatus,
27    SubStatus,
28};
29use ssp2::primitives::RawJson;
30use ssp2::registration::canonical_scope_json;
31use ssp2::segment::{decode_rows_segment, Column, ColumnType, ColumnValue, Row, RowsSegment};
32use ssp2::{decode_message, encode_presence_publish, parse_control, ControlMessage, PresenceKind};
33
34use crate::api::{
35    AppliedServerCommit, AppliedServerCommitSource, ClientChangeBatch, ClientDiagnosticsHost,
36    ClientDiagnosticsLease, ClientDiagnosticsReplica, ClientDiagnosticsSchema,
37    ClientDiagnosticsSnapshot, ClientDiagnosticsStorage, ClientError, ClientLimits, CommandEffects,
38    CommitDelivery, CommitOperation, CommitOperationOutcome, CommitOutcome, CommitOutcomeQuery,
39    CommitOutcomeResolution, CommitOutcomeStatus, ConflictRecord, CoverageSnapshot,
40    CurrentBaseProof, CurrentBaseProofReason, DiagnosticLastChange, DiagnosticLastRound,
41    DiagnosticQueryFailure, DiagnosticRoundCounters, DiagnosticUnits, EchoRecord, FetchedBlob,
42    LeaseState, LocalDataPurgeInput, LocalDataPurgeResult, LocalDataPurgeTarget,
43    LocalDataRebootstrapInput, LocalDataRebootstrapResult, Mutation, PendingCommit, PresencePeer,
44    PreviousVersionStatus, QueryOwner, QueryReadFailure, QueryRow, QuerySnapshot, QueryValue,
45    RejectionDetails, RejectionRecord, ResolveCommitOutcomeInput, RetainedCommitRow,
46    RetainedUniqueConflict, RowState, SchemaFloor, SnapshotRead, SyncIntent, SyncOutcome,
47    SyncReport, SyncStatusSnapshot, TableChange, UnitCatchup, UnitRecord, UnitRestriction,
48    UnitScopes, UnitStatus, WindowBase, WindowChange, WindowCoverage, WindowState, WindowUnitRef,
49    CLIENT_DIAGNOSTICS_VERSION, MAX_DIAGNOSTIC_QUERY_FAILURES,
50};
51use crate::api::{RealtimePolicy, RealtimeState, REALTIME_UNAVAILABLE_CODE};
52#[cfg(feature = "bench-internals")]
53use crate::bench::{Phase, Recorder};
54use crate::previous_version::{
55    capture_previous_version_from_replica, first_incompatibility,
56    reconcile_previous_version_at_boot, set_local_schema_descriptor,
57    sweep_previous_version_container, PendingCommitAudit, PreviousVersionContextConfig,
58    PreviousVersionLifecycle, PreviousVersionReadSpec, PreviousVersionSnapshot,
59};
60use crate::schema::{parse_schema_json, ClientSchema, FtsIndexSchema, TableSchema};
61use crate::transport::{BlobDownload, SegmentRequest, Transport, TransportError};
62use crate::values::{
63    bytes_to_hex, column_value_to_json, decode_row_bytes, encode_sparse_row_json, full_row_values,
64    is_canonical_bytes_envelope, json_to_column_value, json_to_scope_map, normalize_values_casing,
65    render_row_id_json, scope_map_to_json, SealedRow,
66};
67
68/// §4.2 default: rows and SQLite images, both applied in committed chunks.
69/// Bit 3 (signed URLs, §5.4) is
70/// added per transport capability at request-build time.
71const DEFAULT_ACCEPT: u8 = 0b0111;
72const ACCEPT_INLINE_ROWS: u8 = 1 << 0;
73const ACCEPT_EXTERNAL_ROWS: u8 = 1 << 1;
74const ACCEPT_SQLITE: u8 = 1 << 2;
75const ACCEPT_SIGNED_URLS: u8 = 1 << 3;
76const MAX_DIAGNOSTIC_DOMAINS: usize = 256;
77
78/// §10.1 `recommendedAction` carried by a server HTTP error body.
79fn recommended_action_of(details: &Option<Value>) -> Option<String> {
80    details
81        .as_ref()?
82        .get("recommendedAction")?
83        .as_str()
84        .map(str::to_owned)
85}
86
87/// §7.4.1 persisted local schema-version marker (`_syncular_meta` key).
88pub(crate) const LOCAL_SCHEMA_VERSION_KEY: &str = "localSchemaVersion";
89const LOCAL_REVISION_KEY: &str = "localRevision";
90/// §4.5 replica cursor; absent means `-1`.
91const REPLICA_CURSOR_KEY: &str = "replicaCursor";
92/// Hex of the last pull `LIVE_END.echoDigest` (§4.2); absent means empty.
93const ECHO_DIGEST_KEY: &str = "echoDigest";
94/// Hex of the registration digest of the list the server stores (§4.1).
95const REGISTERED_DIGEST_KEY: &str = "registrationDigest";
96/// §7.4.3: table name → generation, persisted with the declaration evidence.
97const TABLE_GENERATIONS_KEY: &str = "localTableGenerations";
98/// §7.1: the identity of the outbox sequence.
99const OUTBOX_ID_KEY: &str = "outboxId";
100const CLIENT_ID_KEY: &str = "clientId";
101const LEASE_STATE_KEY: &str = "leaseState";
102const SCHEMA_FLOOR_KEY: &str = "schemaFloor";
103pub(crate) const LOG_EPOCH_KEY: &str = "logEpoch";
104/// Persisted fail-closed quarantine marker: present while a security preflight
105/// is pending and `activateSecurity` has yet to run. Storing it in the replica
106/// keeps the gate with the data it protects, so a rebuilt host handle reopening
107/// the same file returns in preflight even though its in-memory state is fresh.
108const SECURITY_PREFLIGHT_PENDING_KEY: &str = "securityPreflightPending";
109const LOCAL_REBOOTSTRAP_RECEIPT_VERSION: u8 = 2;
110const MAX_JS_SAFE_INTEGER: u64 = 9_007_199_254_740_991;
111/// §7.4.4 client-local code: a pending outbox commit cannot re-encode under
112/// the new schema after a bump. Never a wire code (§10.3).
113const OUTBOX_INCOMPATIBLE_CODE: &str = "sync.outbox_incompatible";
114/// §5.11 client-local code: an encode at the push seam resolved no usable key
115/// id or named an unknown key. Never a wire code (§10.3).
116const ENCRYPT_FAILED_CODE: &str = "client.encrypt_failed";
117const INVALID_REQUEST_CODE: &str = "sync.invalid_request:";
118/// §7.4.4: static text for a persisted commit the current codec refuses.
119const LEGACY_VALUE_REFUSAL_MESSAGE: &str =
120    "the persisted commit carries values the current codec refuses";
121/// §7.4.4: bounded reason on every intrinsic value drop.
122const INVALID_STORED_VALUES_REASON: &str = "invalid_stored_values";
123/// §7.2.2: an outcome with a pending decision accepts no other resolution.
124const REPLACEMENT_PENDING: &str =
125    "client.replacement_pending: the outcome has a pending replacement";
126/// §7.2.2 client-local code: a pending replacement was not written.
127const REPLACEMENT_REFUSED_CODE: &str = "sync.replacement_refused";
128
129#[derive(Debug, Serialize, Deserialize)]
130#[serde(rename_all = "camelCase", deny_unknown_fields)]
131struct PersistedLocalDataRebootstrapReceipt {
132    version: u8,
133    retained_commits: u64,
134    reset_subscriptions: u64,
135}
136
137fn invalid_local_rebootstrap_receipt() -> String {
138    "sync.local_corrupt: persisted local rebootstrap receipt is invalid".to_owned()
139}
140
141fn encode_local_rebootstrap_receipt(
142    retained_commits: usize,
143    reset_subscriptions: usize,
144) -> Result<String, String> {
145    let retained_commits =
146        u64::try_from(retained_commits).map_err(|_| invalid_local_rebootstrap_receipt())?;
147    let reset_subscriptions =
148        u64::try_from(reset_subscriptions).map_err(|_| invalid_local_rebootstrap_receipt())?;
149    if retained_commits > MAX_JS_SAFE_INTEGER || reset_subscriptions > MAX_JS_SAFE_INTEGER {
150        return Err(invalid_local_rebootstrap_receipt());
151    }
152    serde_json::to_string(&PersistedLocalDataRebootstrapReceipt {
153        version: LOCAL_REBOOTSTRAP_RECEIPT_VERSION,
154        retained_commits,
155        reset_subscriptions,
156    })
157    .map_err(|_| invalid_local_rebootstrap_receipt())
158}
159
160fn decode_local_rebootstrap_receipt(value: &str) -> Result<(usize, usize), String> {
161    // Pre-0.15.36 markers proved application but did not retain the receipt.
162    if value == "v1" {
163        return Ok((0, 0));
164    }
165    let receipt: PersistedLocalDataRebootstrapReceipt =
166        serde_json::from_str(value).map_err(|_| invalid_local_rebootstrap_receipt())?;
167    if receipt.version != LOCAL_REBOOTSTRAP_RECEIPT_VERSION
168        || receipt.retained_commits > MAX_JS_SAFE_INTEGER
169        || receipt.reset_subscriptions > MAX_JS_SAFE_INTEGER
170    {
171        return Err(invalid_local_rebootstrap_receipt());
172    }
173    Ok((
174        usize::try_from(receipt.retained_commits)
175            .map_err(|_| invalid_local_rebootstrap_receipt())?,
176        usize::try_from(receipt.reset_subscriptions)
177            .map_err(|_| invalid_local_rebootstrap_receipt())?,
178    ))
179}
180
181#[cfg(test)]
182mod observation_tests;
183
184/// Internal state for hosts that perform network I/O outside the client owner.
185#[doc(hidden)]
186#[derive(Clone)]
187pub struct CurrentBasePreparation {
188    client_commit_id: String,
189    sequence: i64,
190    /// §7.2.2: the pending replacement a drive evaluates; `None` for
191    /// `proveCurrentBase`.
192    replacement: Option<String>,
193    units: Vec<UnitRecord>,
194    transport_epoch: u64,
195    registration_epoch: u64,
196    security_epoch: u64,
197}
198
199#[cfg(test)]
200mod units_tests;
201
202#[cfg(test)]
203mod replace_outcome_tests;
204
205#[derive(Debug, Clone)]
206struct OutboxOp {
207    upsert: bool,
208    table: String,
209    row_id: String,
210    base_version: Option<i64>,
211    /// Schema-agnostic local form (§0): driver JSON values, encoded with
212    /// the current codec at send time. A key's presence IS the §6.1 sparse
213    /// presence set — both the overlay and the wire payload derive from it.
214    values: Option<Map<String, Value>>,
215    /// §7.1 insert intent: `mutate` recorded this upsert while the row was
216    /// absent from the visible replica. Replay and send-time encoding present
217    /// every current-schema column; a column `values` lacks is NULL.
218    insert: bool,
219}
220
221impl OutboxOp {
222    /// The values this upsert presents under the current `table` (§7.1). An
223    /// insert-intent operation presents every column, a missing one as NULL;
224    /// `None` when a missing column is non-nullable, which leaves the commit
225    /// unencodable (§7.4.4), or when the operation carries no values.
226    fn presented_values(&self, table: &TableSchema) -> Option<Cow<'_, Map<String, Value>>> {
227        let values = self.values.as_ref()?;
228        if !self.insert || table.columns.iter().all(|c| values.contains_key(&c.name)) {
229            return Some(Cow::Borrowed(values));
230        }
231        full_row_values(table, values.clone()).ok().map(Cow::Owned)
232    }
233}
234
235impl From<CommitOperation> for OutboxOp {
236    fn from(operation: CommitOperation) -> Self {
237        Self {
238            upsert: operation.op == "upsert",
239            table: operation.table,
240            row_id: operation.row_id,
241            base_version: operation.base_version,
242            values: operation.values,
243            insert: operation.insert,
244        }
245    }
246}
247
248/// §7.5 static authoring refusal; the table and column travel in details.
249fn invalid_authoring(message: &str, details: Value) -> ClientError {
250    ClientError {
251        code: "sync.invalid_request".to_owned(),
252        message: message.to_owned(),
253        details: Some(details),
254        retryable: false,
255    }
256}
257
258/// §7.1 chained base: `later` repeats `earlier`'s positive base on the same
259/// row and presents only columns `earlier` presents.
260fn chains_on(earlier: &OutboxOp, later: &OutboxOp) -> bool {
261    // Only an upsert carries values, and it always carries its primary key.
262    earlier.row_id == later.row_id
263        && earlier.table == later.table
264        && later.base_version.is_some_and(|base| base > 0)
265        && earlier.base_version == later.base_version
266        && later.values.as_ref().is_some_and(|values| {
267            values.keys().all(|column| {
268                earlier
269                    .values
270                    .as_ref()
271                    .is_some_and(|earlier| earlier.contains_key(column))
272            })
273        })
274}
275
276impl From<&OutboxOp> for CommitOperation {
277    fn from(operation: &OutboxOp) -> Self {
278        Self {
279            table: operation.table.clone(),
280            row_id: operation.row_id.clone(),
281            op: if operation.upsert { "upsert" } else { "delete" }.to_owned(),
282            base_version: operation.base_version,
283            values: operation.values.clone(),
284            insert: operation.insert,
285        }
286    }
287}
288
289#[derive(Debug, Clone)]
290struct OutboxCommit {
291    client_commit_id: String,
292    ops: Vec<OutboxOp>,
293}
294
295/// One commit's `(table, value keys, insert intent)` per upsert operation — the classification
296/// shape the §7.4.4 send-time drop and the RFC 0005 D6 audit share, so the two
297/// can never disagree about which commits cannot re-encode.
298fn commit_audit_operations(commit: &OutboxCommit) -> Vec<(&str, Vec<&str>, bool)> {
299    commit
300        .ops
301        .iter()
302        .filter(|operation| operation.upsert)
303        .map(|operation| {
304            (
305                operation.table.as_str(),
306                operation
307                    .values
308                    .as_ref()
309                    .map(|values| values.keys().map(String::as_str).collect())
310                    .unwrap_or_default(),
311                operation.insert,
312            )
313        })
314        .collect()
315}
316
317type FailedCommit = (OutboxCommit, Map<String, Value>);
318
319#[derive(Debug, Clone)]
320struct CompiledLocalDataPurgeTarget {
321    table: String,
322    selectors: Vec<(String, Vec<String>)>,
323}
324
325struct StoredCommitOutcomeRow {
326    sequence: i64,
327    client_commit_id: String,
328    status: String,
329    recorded_at_ms: i64,
330    results_json: String,
331    operations_json: Option<String>,
332    resolution: String,
333    resolved_at_ms: Option<i64>,
334    replacement_client_commit_id: Option<String>,
335}
336
337impl StoredCommitOutcomeRow {
338    fn from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result<Self> {
339        Ok(Self {
340            sequence: row.get(0)?,
341            client_commit_id: row.get(1)?,
342            status: row.get(2)?,
343            recorded_at_ms: row.get(3)?,
344            results_json: row.get(4)?,
345            operations_json: row.get(5)?,
346            resolution: row.get(6)?,
347            resolved_at_ms: row.get(7)?,
348            replacement_client_commit_id: row.get(8)?,
349        })
350    }
351}
352
353/// Section outcome distinguishing the §5.6 unit-local fail-closed
354/// path from a round-aborting failure (§1.4 rule 5).
355/// §4.4 a unit section's `SUB_START`.
356struct SectionStart {
357    id: String,
358    status: SubStatus,
359    reason_code: String,
360    bootstrap: bool,
361    fenced: Vec<FencedRecord>,
362}
363
364enum SectionError {
365    FailClosed,
366    /// §5.5: the server refused a segment with `sync.forbidden`.
367    Refused,
368    /// §5.3 client memory: the round deferred this image's body to a later
369    /// pull of the same resume token.
370    Deferred,
371    Abort(String, String),
372    Transport(TransportError),
373}
374
375#[derive(Debug)]
376pub(crate) struct RequestMeta {
377    pushed_ids: Vec<String>,
378    /// Hex of the registration digest this pull carried; the server stores
379    /// that list once the round completes (§4.1).
380    registration_digest: String,
381    pub(crate) accept: u8,
382    /// §6.1 splitBatch: outbox commits held back from THIS request because
383    /// the running operation count reached the push cap — the next round
384    /// pushes them (`sync_needed` stays set while any remain).
385    deferred_commits: usize,
386    /// §5.6 applied-segment record per unit: segment id → rows applied at
387    /// the unit's current cursor and token.
388    pub(crate) applied: BTreeMap<String, BTreeMap<String, i64>>,
389    /// §5.3 client memory: (unit, segment id) of the first image whose body
390    /// the round did not fetch because the batch already holds two images.
391    /// That unit's section stops there without persisting `SUB_END`.
392    pub(crate) deferred: BTreeSet<(String, String)>,
393}
394
395#[derive(Default)]
396struct ChangeAccumulator {
397    /// None means table-wide; Some is the exact scoped domain.
398    tables: BTreeMap<String, Option<BTreeSet<String>>>,
399    windows: BTreeMap<String, BTreeSet<String>>,
400    status: bool,
401    conflicts: bool,
402    rejections: bool,
403    outcomes: bool,
404}
405
406impl ChangeAccumulator {
407    fn table(&mut self, table: &str) {
408        self.tables.insert(table.to_owned(), None);
409    }
410
411    fn scope(&mut self, table: &str, key: String) {
412        match self.tables.get_mut(table) {
413            Some(None) => {}
414            Some(Some(keys)) => {
415                keys.insert(key);
416            }
417            None => {
418                self.tables
419                    .insert(table.to_owned(), Some(BTreeSet::from([key])));
420            }
421        }
422    }
423
424    fn window(&mut self, base_key: &str, value: &str) {
425        self.windows
426            .entry(base_key.to_owned())
427            .or_default()
428            .insert(value.to_owned());
429    }
430
431    fn touched(&self) -> bool {
432        !self.tables.is_empty()
433            || !self.windows.is_empty()
434            || self.status
435            || self.conflicts
436            || self.rejections
437            || self.outcomes
438    }
439}
440
441/// §6.1: the server caps total operations per request (reference default
442/// 500) and rejects the whole batch with `sync.too_many_operations`; the
443/// client "splits and retries". Splitting happens at build time: commits are
444/// included IN ORDER until the operation budget is spent, the rest wait for
445/// the next round.
446const PUSH_OPS_PER_REQUEST: usize = 500;
447const MAX_LOCAL_PURGE_TARGETS: usize = 64;
448const MAX_LOCAL_PURGE_SELECTORS: usize = 8;
449const MAX_LOCAL_PURGE_VALUES: usize = 128;
450const MAX_LOCAL_PURGE_VALUE_LENGTH: usize = 256;
451pub const SECURITY_PREFLIGHT_REQUIRED_CODE: &str = "client.security_preflight_required";
452
453/// Dirty visible tables; `None` means a whole-replica recovery/reset.
454struct OverlayDirty(RefCell<Option<BTreeSet<String>>>);
455
456impl OverlayDirty {
457    fn clean() -> Self {
458        Self(RefCell::new(Some(BTreeSet::new())))
459    }
460    fn get(&self) -> bool {
461        self.0
462            .borrow()
463            .as_ref()
464            .is_none_or(|tables| !tables.is_empty())
465    }
466    fn set(&self, dirty: bool) {
467        *self.0.borrow_mut() = if dirty { None } else { Some(BTreeSet::new()) };
468    }
469    fn table(&self, table: &str) {
470        if let Some(tables) = self.0.borrow_mut().as_mut() {
471            tables.insert(table.to_owned());
472        }
473    }
474    fn snapshot(&self) -> Option<BTreeSet<String>> {
475        self.0.borrow().clone()
476    }
477    fn restore(&self, tables: Option<BTreeSet<String>>) {
478        *self.0.borrow_mut() = tables;
479    }
480}
481
482/// A copy-on-write in-memory mirror: a [`Rollback`] shares it, and the first
483/// write after a capture copies it once.
484#[derive(Debug, Clone, Default)]
485struct Shared<T>(Arc<T>);
486
487impl<T> std::ops::Deref for Shared<T> {
488    type Target = T;
489    fn deref(&self) -> &T {
490        &self.0
491    }
492}
493
494impl<T: Clone> std::ops::DerefMut for Shared<T> {
495    fn deref_mut(&mut self) -> &mut T {
496        Arc::make_mut(&mut self.0)
497    }
498}
499
500impl<T> From<T> for Shared<T> {
501    fn from(value: T) -> Self {
502        Self(Arc::new(value))
503    }
504}
505
506/// The in-memory mirrors of durable state, captured before a local
507/// transaction and restored when it rolls back. Queues only grow inside a
508/// transaction, so their lengths suffice.
509#[derive(Debug, Clone)]
510struct Rollback {
511    outbox: Shared<Vec<Arc<OutboxCommit>>>,
512    failed_commits: Shared<Vec<FailedCommit>>,
513    conflicts: Shared<Vec<ConflictRecord>>,
514    rejections: Shared<Vec<RejectionRecord>>,
515    schema_floor: Option<SchemaFloor>,
516    lease_state: Option<LeaseState>,
517    stopped: bool,
518    upgrading: bool,
519    upgrade_kept_tables: bool,
520    sync_needed: bool,
521    active_round: Option<(uuid::Uuid, i64, u64)>,
522    overlay_dirty: Option<BTreeSet<String>>,
523    change_queue: usize,
524    sync_intent_queue: usize,
525    last_change: Option<DiagnosticLastChange>,
526}
527
528pub struct SyncClient {
529    #[cfg(feature = "bench-internals")]
530    benchmark_phases: Recorder,
531    conn: Connection,
532    schema: ClientSchema,
533    client_id: String,
534    limits: ClientLimits,
535    outbox: Shared<Vec<Arc<OutboxCommit>>>,
536    failed_commits: Shared<Vec<FailedCommit>>,
537    retain_failed_commits: bool,
538    conflicts: Shared<Vec<ConflictRecord>>,
539    rejections: Shared<Vec<RejectionRecord>>,
540    schema_floor: Option<SchemaFloor>,
541    /// §7.3.5: the opaque auth-lease state (from LEASE frames + lease errors).
542    lease_state: Option<LeaseState>,
543    /// §1.6: the schema-floor response stops syncing until an upgrade.
544    stopped: bool,
545    /// §7.4.5: true while a schema-bump reset + first re-bootstrap is in flight.
546    upgrading: bool,
547    /// §7.4.5: the running reset kept at least one table.
548    upgrade_kept_tables: bool,
549    /// §8.4 coalesced sync-needed signal.
550    sync_needed: bool,
551    /// §8.8: how sync rounds treat the realtime binding, and its exact
552    /// availability state with the evidence behind a lost or refused one.
553    realtime_policy: RealtimePolicy,
554    realtime_state: RealtimeState,
555    realtime_reason_code: Option<String>,
556    realtime_retry_delay_ms: Option<u64>,
557    /// §8.6 presence: scopeKey → (`actorId clientId` peer key → peer).
558    presence: HashMap<String, HashMap<String, PresencePeer>>,
559    /// Client clock (epoch ms) for the §5.4 `urlExpiresAtMs` check and the
560    /// §7.8 drive deadline; the host may pin it and move the pinned value
561    /// (conformance runs on a virtual clock).
562    now_ms: Option<std::sync::Arc<std::sync::atomic::AtomicI64>>,
563    /// §5.11 client-side encryption keys (`keyId → key bytes`). Empty ⇒ E2EE
564    /// off. The encrypt/decrypt seam (`values.rs`) is compiled only under the
565    /// `e2ee` feature; without it, a schema with encrypted columns fails loud.
566    encryption: crate::values::EncryptionConfig,
567    /// Fail-closed host bootstrap gate. While set, command hosts permit only
568    /// status/lifecycle inspection and an exact authorized local purge.
569    security_preflight: bool,
570    security_epoch: u64,
571    transport_enabled: bool,
572    transport_epoch: u64,
573    /// Raised by every registration change; a current-base proof spanning
574    /// one reports missing coverage.
575    registration_epoch: u64,
576    /// RFC 0005 D2/D8: the previous-version capture config. `None` (default)
577    /// means the feature is off; the descriptor write and the container orphan
578    /// sweep still happen.
579    previous_version: Option<PreviousVersionContextConfig>,
580    /// Per-table primary-key upsert SQL, built once per (full table name) —
581    /// the row write path runs per row during bootstrap (§5.6), so the SQL
582    /// string (and, via `prepare_cached`, its compiled statement) is reused
583    /// instead of being rebuilt and re-prepared per row. Cleared on a §7.4.3
584    /// schema reset (the column lists may have changed).
585    insert_sql: RefCell<HashMap<String, String>>,
586    /// §7.1: reconcile only tables whose base/outbox changed. Whole-replica
587    /// reset and restart explicitly invalidate every table.
588    overlay_dirty: OverlayDirty,
589    // First SQLite failure of the current operation, before savepoint cleanup.
590    storage_failure: RefCell<Option<QueryReadFailure>>,
591    /// Test-only structural performance signal: response processing must not
592    /// turn a batch of acknowledgements into one full overlay rebuild each.
593    #[cfg(test)]
594    overlay_rebuild_count: Cell<usize>,
595    #[cfg(test)]
596    acknowledged_replay_count: Cell<usize>,
597    /// Test-only structural performance signal: outcome retention is enforced
598    /// once per response rather than once per acknowledgement.
599    #[cfg(test)]
600    outcome_prune_count: Cell<usize>,
601    /// Test-only pinned duration of every §5.3 image chunk, in ms.
602    #[cfg(test)]
603    image_chunk_ms: Cell<Option<f64>>,
604    /// Exact observer-transaction output drained by command/FFI hosts.
605    progress: ProgressObserver,
606    /// §7.6: random per core instance, never persisted or sent.
607    epoch: String,
608    change_queue: VecDeque<ClientChangeBatch>,
609    sync_intent_queue: VecDeque<SyncIntent>,
610    /// §8.7 `scopes` events received since the last drain.
611    scopes_changed: usize,
612    /// Explicit exponential retry policy for transient transport failures.
613    retry_delay_ms: u64,
614    /// §7.7: the background retry the current round scheduled, if any.
615    round_retry_delay_ms: Option<u64>,
616    /// §5.3: a table's next image chunk size, carried from one range image
617    /// of the table to the next.
618    image_chunk_rows: HashMap<String, u32>,
619    /// A round whose response never completed (§1.5): the client keeps its
620    /// code and reports it on every following request until a round receives a
621    /// complete, decodable response. The server already recorded the round, so
622    /// nothing else can tell it the round failed on this side.
623    round_failure: Option<String>,
624    /// The in-flight round: its id, `started_at_ms` and progress attempt.
625    active_round: Option<(uuid::Uuid, i64, u64)>,
626    last_round: Option<DiagnosticLastRound>,
627    last_change: Option<DiagnosticLastChange>,
628    /// §7.6: latest failed owned snapshot read per owner id, oldest first.
629    query_failures: Vec<DiagnosticQueryFailure>,
630    /// SPEC §1.4 rule 4: the open coalesced apply transaction of a response.
631    coalesced: Option<CoalescedApply>,
632    /// A coalesced apply transaction of the current response failed; every
633    /// later observation of that response fails until it ends.
634    coalesced_failure: Option<String>,
635}
636
637/// SPEC §1.4 rule 4: the frames and blocks of one response share one local
638/// write transaction of at most `COALESCED_APPLY_ROWS` rows. The in-memory
639/// state captured at its `BEGIN` is restored when its `COMMIT` fails.
640struct CoalescedApply {
641    rows: usize,
642    rollback: Rollback,
643}
644
645const COALESCED_APPLY_ROWS: usize = 1024;
646
647pub(crate) fn quote_ident(name: &str) -> String {
648    format!("\"{}\"", name.replace('"', "\"\""))
649}
650/// §7.4.1 `_syncular_meta` helpers. Free functions so RFC 0005's
651/// previous-version module reads and writes the SAME records as the client
652/// instead of growing a parallel meta layer.
653pub(crate) fn meta_get(conn: &Connection, key: &str) -> rusqlite::Result<Option<String>> {
654    conn.query_row(
655        "SELECT value FROM _syncular_meta WHERE key = ?1",
656        rusqlite::params![key],
657        |row| row.get::<_, String>(0),
658    )
659    .optional()
660}
661
662pub(crate) fn meta_set(conn: &Connection, key: &str, value: &str) -> rusqlite::Result<()> {
663    conn.execute(
664        "INSERT OR REPLACE INTO _syncular_meta (key, value) VALUES (?1, ?2)",
665        rusqlite::params![key, value],
666    )
667    .map(|_| ())
668}
669
670pub(crate) fn meta_delete(conn: &Connection, key: &str) -> rusqlite::Result<()> {
671    conn.execute(
672        "DELETE FROM _syncular_meta WHERE key = ?1",
673        rusqlite::params![key],
674    )
675    .map(|_| ())
676}
677
678/// Seek by the existing key while retaining the exact legacy text match.
679fn row_id_predicate(table: &TableSchema) -> String {
680    let key = quote_ident(&table.primary_key);
681    let text_match = format!("CAST({key} AS TEXT) = ?1");
682    match table.columns[table.pk_index].ty {
683        ColumnType::String | ColumnType::Json => {
684            format!("{key} = ?1 AND {text_match}")
685        }
686        // Native columns have no declared SQLite affinity. Match the stored
687        // integer bind type explicitly. Unary + removes expression affinity so
688        // SQLite can seek the typeless key; the text check preserves exact IDs.
689        ColumnType::Integer | ColumnType::Boolean => {
690            format!("{key} = +CAST(?1 AS INTEGER) AND {text_match}")
691        }
692        // REAL-to-text conversion can round distinct values to the same text.
693        // Narrowing those matches by numeric equality would change behavior.
694        _ => text_match,
695    }
696}
697
698/// §3.3/§4.8/§5.6 row ownership: rows whose local scope columns match
699/// `effective` and that no entry of `owners` matches. `owners` holds the
700/// pair scopes of every owner of the table (§3.3 row ownership); a scoped delete never removes a row one of them still covers.
701/// Scope values bind by the column's declared type, as `row_id_predicate`
702/// does. `Ok(None)` matches nothing (an empty map or value list); `Err(())`
703/// is a missing local scope-column mapping (the fail-closed case).
704fn local_scope_predicate(
705    table: &TableSchema,
706    effective: &[(String, Vec<String>)],
707    owners: &[Vec<(String, Vec<String>)>],
708) -> Result<Option<(String, Vec<SqlValue>)>, ()> {
709    if effective.is_empty() || effective.iter().any(|(_, values)| values.is_empty()) {
710        return Ok(None);
711    }
712    let column = |variable: &str| {
713        let name = table.scope_column(variable).ok_or(())?;
714        let column = table
715            .columns
716            .iter()
717            .find(|column| column.name == name)
718            .ok_or(())?;
719        Ok((quote_ident(name), column.ty == ColumnType::Integer))
720    };
721    let mut clauses = Vec::new();
722    let mut params = Vec::new();
723    for (variable, values) in effective {
724        let (key, integer) = column(variable)?;
725        let holes = vec!["?"; values.len()].join(", ");
726        clauses.push(if integer {
727            params.extend(values.iter().cloned().map(SqlValue::Text));
728            let seek = vec!["+CAST(? AS INTEGER)"; values.len()].join(", ");
729            format!("{key} IN ({seek}) AND CAST({key} AS TEXT) IN ({holes})")
730        } else {
731            format!("{key} IN ({holes})")
732        });
733        params.extend(values.iter().cloned().map(SqlValue::Text));
734    }
735    // Owners bind as one JSON parameter per variable set, so the statement
736    // keeps its size at any owner count; owners of one variable bind the
737    // union of their values. An owner holding other values of a variable
738    // `effective` names owns none of its rows and drops out. Values compare
739    // as text, the exact semantics of the effective clauses.
740    let mut groups: BTreeMap<Vec<&str>, Vec<Vec<&[String]>>> = BTreeMap::new();
741    for owner in owners {
742        if owner.is_empty()
743            || owner.iter().any(|(variable, values)| {
744                values.is_empty()
745                    || effective.iter().any(|(name, wanted)| {
746                        name == variable && wanted.iter().all(|value| !values.contains(value))
747                    })
748            })
749        {
750            continue;
751        }
752        let mut owner: Vec<_> = owner.iter().collect();
753        owner.sort_by(|a, b| a.0.cmp(&b.0));
754        groups
755            .entry(
756                owner
757                    .iter()
758                    .map(|(variable, _)| variable.as_str())
759                    .collect(),
760            )
761            .or_default()
762            .push(owner.iter().map(|(_, values)| values.as_slice()).collect());
763    }
764    let within = |variable: &str, list: &str| {
765        let (key, integer) = column(variable)?;
766        let key = if integer {
767            format!("CAST({key} AS TEXT)")
768        } else {
769            key
770        };
771        Ok(format!("{key} IN (SELECT value FROM json_each({list}))"))
772    };
773    for (variables, tuples) in groups {
774        if let [variable] = variables[..] {
775            let values: BTreeSet<&String> = tuples.iter().flatten().copied().flatten().collect();
776            params.push(SqlValue::Text(
777                Value::from_iter(values.into_iter().cloned()).to_string(),
778            ));
779            clauses.push(format!("NOT {}", within(variable, "?")?));
780        } else {
781            params.push(SqlValue::Text(
782                serde_json::to_string(&tuples).expect("owner tuples serialize"),
783            ));
784            let matches = variables
785                .iter()
786                .enumerate()
787                .map(|(index, variable)| within(variable, &format!("o.value,'$[{index}]'")))
788                .collect::<Result<Vec<_>, ()>>()?;
789            clauses.push(format!(
790                "NOT EXISTS (SELECT 1 FROM json_each(?) o WHERE {})",
791                matches.join(" AND ")
792            ));
793        }
794    }
795    Ok(Some((clauses.join(" AND "), params)))
796}
797
798/// [`local_scope_predicate`] over decoded row values (retained rows).
799fn owns_local_row(
800    table: &TableSchema,
801    effective: &[(String, Vec<String>)],
802    owners: &[Vec<(String, Vec<String>)>],
803    values: &Map<String, Value>,
804) -> bool {
805    let matches = |scopes: &[(String, Vec<String>)]| {
806        !scopes.is_empty()
807            && scopes.iter().all(|(variable, allowed)| {
808                table
809                    .scope_column(variable)
810                    .and_then(|column| values.get(column))
811                    .filter(|value| !value.is_null())
812                    .is_some_and(|value| {
813                        allowed.contains(
814                            &value
815                                .as_str()
816                                .map_or_else(|| value.to_string(), str::to_owned),
817                        )
818                    })
819            })
820    };
821    matches(effective) && !owners.iter().any(|owner| matches(owner))
822}
823
824fn is_local_operation_code_like(value: &str) -> bool {
825    let bytes = value.as_bytes();
826    bytes.first().is_some_and(u8::is_ascii_alphanumeric)
827        && bytes
828            .iter()
829            .all(|byte| byte.is_ascii_alphanumeric() || matches!(*byte, b'.' | b'_' | b':' | b'-'))
830}
831
832fn base_table(name: &str) -> String {
833    quote_ident(&format!("_syncular_base_{name}"))
834}
835
836/// §4.8 deferred eviction record: (unit id, table, effective scope map).
837type PendingEvict = (String, String, Vec<(String, Vec<String>)>);
838
839/// §4.8: a stable key for a window base: variable + canonical fixed scopes.
840fn window_base_key(base: &WindowBase) -> String {
841    format!(
842        "{}\0{}",
843        base.variable,
844        ssp2::registration::canonical_unit_json(&scope_pairs(&base.fixed_scopes))
845    )
846}
847
848/// The unit of one window value: the fixed scopes plus the value.
849fn window_unit_scopes(base: &WindowBase, value: &str) -> UnitScopes {
850    let mut scopes = base.fixed_scopes.clone();
851    scopes.insert(base.variable.clone(), value.to_owned());
852    scopes
853}
854
855/// A unit's scope map as the codec and the digests take it.
856fn scope_pairs(scopes: &UnitScopes) -> Vec<(String, String)> {
857    let mut pairs: Vec<(String, String)> = scopes
858        .iter()
859        .map(|(variable, value)| (variable.clone(), value.clone()))
860        .collect();
861    pairs.sort_by(|a, b| a.0.encode_utf16().cmp(b.0.encode_utf16()));
862    pairs
863}
864
865/// §4.1: the unit covers every table declaring all of its variables.
866fn covers(table: &TableSchema, scopes: &UnitScopes) -> bool {
867    scopes
868        .keys()
869        .all(|variable| table.scope_column(variable).is_some())
870}
871
872/// §4.4: the effective scopes of the (unit, table) pair under `echo`: the
873/// scope map as one-element lists plus the fenced lists. `None` when the
874/// fence revoked the pair.
875fn pair_scopes(
876    scopes: &UnitScopes,
877    table: &str,
878    echo: Option<&[EchoRecord]>,
879) -> Option<Vec<(String, Vec<String>)>> {
880    let fenced = echo.and_then(|echo| echo.iter().find(|record| record.table == table));
881    if fenced.is_some_and(|record| record.status == "revoked") {
882        return None;
883    }
884    let mut effective: BTreeMap<String, Vec<String>> = fenced
885        .map(|record| record.scopes.clone())
886        .unwrap_or_default();
887    for (variable, value) in scopes {
888        effective.insert(variable.clone(), vec![value.clone()]);
889    }
890    Some(effective.into_iter().collect())
891}
892
893/// §3.3 row ownership: the pair scopes of every owner of `table` except
894/// `except_id`. An owner is active with a persisted echo and live or
895/// post-bootstrap, or bootstrapping with a segment of the table in its
896/// current bootstrap, or holding a restriction that excludes the table.
897fn unit_owners(
898    units: &[UnitRecord],
899    table: &TableSchema,
900    except_id: &str,
901) -> Vec<Vec<(String, Vec<String>)>> {
902    units
903        .iter()
904        .filter(|unit| {
905            unit.id != except_id
906                && unit.status == UnitStatus::Active
907                && unit.echo.is_some()
908                && (!unit.bootstrapping()
909                    || unit.seeded.contains(&table.name)
910                    || unit
911                        .restriction
912                        .as_ref()
913                        .is_some_and(|restriction| !restriction.tables.contains(&table.name)))
914                && covers(table, &unit.scopes)
915        })
916        .filter_map(|unit| pair_scopes(&unit.scopes, &table.name, unit.echo.as_deref()))
917        .collect()
918}
919
920/// §5.5 / §5.6: the unit's scope map, each value a one-element list.
921fn unit_scope_lists(scopes: &UnitScopes) -> Vec<(String, Vec<String>)> {
922    scopes
923        .iter()
924        .map(|(variable, value)| (variable.clone(), vec![value.clone()]))
925        .collect()
926}
927
928/// §7.1 acknowledged intent of the rows `?1` names (`[[table, id], ...]`).
929/// It joins through the (tbl, id) index: a scan per acknowledgement would
930/// make draining a batch quadratic.
931const ACKED_ROWS_BY_ROW: &str = "SELECT a.op_json,a.intent_json FROM json_each(?1) k JOIN _syncular_acked_rows a ON a.tbl=json_extract(k.value,'$[0]') AND a.id=json_extract(k.value,'$[1]') WHERE ?2 IS NULL OR a.tbl IN(SELECT value FROM json_each(?2)) ORDER BY a.commit_seq,a.idx";
932
933const UNIT_COLUMNS: &str =
934    "id, scopes, explicit, status, reason_code, cursor, bootstrap_state, seeded, echo, restriction";
935
936/// A unit row; a field that does not parse is a conversion failure, which
937/// both read paths classify as `sync.local_corrupt`.
938fn unit_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result<UnitRecord> {
939    fn json<T: serde::de::DeserializeOwned>(index: usize, raw: &str) -> rusqlite::Result<T> {
940        serde_json::from_str(raw).map_err(|error| {
941            rusqlite::Error::FromSqlConversionFailure(
942                index,
943                rusqlite::types::Type::Text,
944                Box::new(error),
945            )
946        })
947    }
948    Ok(UnitRecord {
949        id: row.get(0)?,
950        scopes: json(1, &row.get::<_, String>(1)?)?,
951        explicit: row.get::<_, i64>(2)? == 1,
952        status: json(
953            3,
954            &serde_json::to_string(&row.get::<_, String>(3)?).expect("text"),
955        )?,
956        reason_code: row.get(4)?,
957        cursor: row.get(5)?,
958        bootstrap_state: row.get(6)?,
959        seeded: json(7, &row.get::<_, String>(7)?)?,
960        echo: row
961            .get::<_, Option<String>>(8)?
962            .map(|echo| json::<Vec<EchoRecord>>(8, &echo))
963            .transpose()?
964            .map(|echo| {
965                if echo
966                    .iter()
967                    .all(|record| record.status == "active" || record.status == "revoked")
968                {
969                    Ok(echo)
970                } else {
971                    Err(rusqlite::Error::FromSqlConversionFailure(
972                        8,
973                        rusqlite::types::Type::Text,
974                        "echo status is neither active nor revoked".into(),
975                    ))
976                }
977            })
978            .transpose()?,
979        restriction: row
980            .get::<_, Option<String>>(9)?
981            .map(|raw| json::<UnitRestriction>(9, &raw))
982            .transpose()?,
983    })
984}
985
986/// Every unit in registration order (§4.1).
987fn load_units(conn: &Connection) -> rusqlite::Result<Vec<UnitRecord>> {
988    conn.prepare_cached(&format!(
989        "SELECT {UNIT_COLUMNS} FROM _syncular_units ORDER BY rowid"
990    ))?
991    .query_map([], unit_from_row)?
992    .collect()
993}
994
995fn get_unit(conn: &Connection, id: &str) -> rusqlite::Result<Option<UnitRecord>> {
996    conn.query_row(
997        &format!("SELECT {UNIT_COLUMNS} FROM _syncular_units WHERE id = ?1"),
998        [id],
999        unit_from_row,
1000    )
1001    .optional()
1002}
1003
1004/// Insert or update in place: an update keeps the registration order. The
1005/// §5.6 applied-segment record ends when the cursor or the token changes.
1006fn save_unit(conn: &Connection, unit: &UnitRecord) -> rusqlite::Result<()> {
1007    conn.prepare_cached(&format!(
1008        "INSERT INTO _syncular_units({UNIT_COLUMNS}) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9,?10)
1009         ON CONFLICT(id) DO UPDATE SET explicit=excluded.explicit, status=excluded.status,
1010           reason_code=excluded.reason_code, cursor=excluded.cursor,
1011           bootstrap_state=excluded.bootstrap_state, seeded=excluded.seeded, echo=excluded.echo,
1012           restriction=excluded.restriction,
1013           applied=CASE WHEN cursor IS excluded.cursor
1014             AND bootstrap_state IS excluded.bootstrap_state THEN applied END"
1015    ))?
1016    .execute(rusqlite::params![
1017        unit.id,
1018        serde_json::to_string(&unit.scopes).expect("scope map serializes"),
1019        i64::from(unit.explicit),
1020        unit.status.name(),
1021        unit.reason_code,
1022        unit.cursor,
1023        unit.bootstrap_state,
1024        serde_json::to_string(&unit.seeded).expect("table list serializes"),
1025        unit.echo
1026            .as_ref()
1027            .map(|echo| serde_json::to_string(echo).expect("echo serializes")),
1028        unit.restriction
1029            .as_ref()
1030            .map(|restriction| serde_json::to_string(restriction).expect("restriction serializes")),
1031    ])
1032    .map(|_| ())
1033}
1034
1035/// §5.6 applied-segment record of unit `id`: segment id → rows applied at
1036/// its current cursor and token.
1037fn applied_rows(conn: &Connection, id: &str) -> rusqlite::Result<BTreeMap<String, i64>> {
1038    conn.query_row(
1039        "SELECT applied FROM _syncular_units WHERE id = ?1",
1040        [id],
1041        |row| row.get::<_, Option<String>>(0),
1042    )
1043    .optional()?
1044    .flatten()
1045    .map_or(Ok(BTreeMap::new()), |raw| {
1046        serde_json::from_str(&raw).map_err(|error| {
1047            rusqlite::Error::FromSqlConversionFailure(
1048                0,
1049                rusqlite::types::Type::Text,
1050                Box::new(error),
1051            )
1052        })
1053    })
1054}
1055
1056/// §5.6: `rows` rows of `segment_id` applied at unit `id`'s position. Runs
1057/// in the transaction that commits them.
1058fn record_applied_rows(
1059    conn: &Connection,
1060    id: &str,
1061    segment_id: &str,
1062    rows: i64,
1063) -> rusqlite::Result<()> {
1064    conn.prepare_cached(
1065        "UPDATE _syncular_units SET applied=json_patch(coalesce(applied,'{}'),?1) WHERE id=?2",
1066    )?
1067    .execute(rusqlite::params![
1068        serde_json::json!({ segment_id: rows }).to_string(),
1069        id
1070    ])
1071    .map(|_| ())
1072}
1073
1074/// §4.1 references: every window registry entry `(base, value)` holding a unit.
1075fn window_refs(conn: &Connection, unit_id: &str) -> rusqlite::Result<Vec<(String, String)>> {
1076    conn.prepare_cached("SELECT base, unit FROM _syncular_windows WHERE sub_id = ?1")?
1077        .query_map([unit_id], |row| Ok((row.get(0)?, row.get(1)?)))?
1078        .collect()
1079}
1080
1081/// §7.4.3 reset, §2.1 log reset and local rebootstrap: keep the unit
1082/// registrations and discard the synced state of every unit outside `keep`,
1083/// so the next round sends the list and bootstraps those units. A schema bump
1084/// passes `restrictions`: every reset unit keeps its echo and takes its
1085/// restriction from the map, if any (§7.4.3); the other resets discard both.
1086/// The replica cursor survives only while a kept unit is live. The caller
1087/// owns the transaction.
1088fn reset_units(
1089    conn: &Connection,
1090    keep: &BTreeSet<String>,
1091    restrictions: Option<&BTreeMap<String, UnitRestriction>>,
1092) -> rusqlite::Result<()> {
1093    conn.execute(
1094        "UPDATE _syncular_units SET status='active', reason_code=NULL, cursor=-1,
1095           bootstrap_state=NULL, seeded='[]', applied=NULL,
1096           echo=CASE WHEN ?2 IS NULL THEN NULL ELSE echo END,
1097           restriction=(SELECT value FROM json_each(?2) WHERE key=_syncular_units.id)
1098         WHERE id NOT IN (SELECT value FROM json_each(?1))",
1099        rusqlite::params![
1100            serde_json::to_string(keep).expect("unit ids serialize"),
1101            restrictions.map(|map| serde_json::to_string(map).expect("restrictions serialize")),
1102        ],
1103    )?;
1104    conn.execute_batch(
1105        "DELETE FROM _syncular_meta WHERE key IN ('echoDigest','registrationDigest');
1106         DELETE FROM _syncular_meta WHERE key='replicaCursor'
1107           AND NOT EXISTS (SELECT 1 FROM _syncular_units WHERE cursor IS NULL);",
1108    )
1109}
1110
1111/// §4.5: the replica cursor, `-1` while no unit is live.
1112fn replica_cursor(conn: &Connection) -> rusqlite::Result<i64> {
1113    meta_get(conn, REPLICA_CURSOR_KEY)?.map_or(Ok(-1), |raw| {
1114        raw.parse().map_err(|error| {
1115            rusqlite::Error::FromSqlConversionFailure(
1116                0,
1117                rusqlite::types::Type::Text,
1118                Box::new(error),
1119            )
1120        })
1121    })
1122}
1123
1124fn visible_table(name: &str) -> String {
1125    quote_ident(name)
1126}
1127
1128const FTS_SOURCE_ID_COLUMN: &str = "_syncular_source_id";
1129
1130/// §7.4.3: is a `sqlite_master` table name a synced table (visible or base),
1131/// i.e. NOT one of the durable bookkeeping tables the reset preserves?
1132fn is_synced_table_name(name: &str) -> bool {
1133    if name.starts_with("sqlite_") {
1134        return false;
1135    }
1136    if name.starts_with("_syncular_base_") {
1137        return true; // the base half of a synced table pair
1138    }
1139    // Bookkeeping: outbox, units, meta, blob cache/uploads.
1140    !name.starts_with("_syncular_")
1141}
1142
1143/// `"sha256:" + hex` of the bytes — the content address (§5.9.1).
1144fn blob_id_for(bytes: &[u8]) -> String {
1145    let digest = Sha256::digest(bytes);
1146    format!("sha256:{}", bytes_to_hex(&digest))
1147}
1148
1149/// One [`SyncClient::write_row`] bind parameter, borrowing the row-codec
1150/// value it wraps — strings/JSON/bytes bind as borrowed TEXT/BLOB (no copy
1151/// per row on the §5.6 bootstrap path), scalars bind owned.
1152enum RowParam<'a> {
1153    Cell(&'a Option<ColumnValue>),
1154    Version(i64),
1155    Text(Option<&'a str>),
1156}
1157
1158impl rusqlite::ToSql for RowParam<'_> {
1159    fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
1160        Ok(match self {
1161            RowParam::Version(v) => ToSqlOutput::Owned(SqlValue::Integer(*v)),
1162            RowParam::Text(text) => match text {
1163                Some(text) => ToSqlOutput::Borrowed(ValueRef::Text(text.as_bytes())),
1164                None => ToSqlOutput::Owned(SqlValue::Null),
1165            },
1166            RowParam::Cell(cell) => match cell {
1167                None => ToSqlOutput::Owned(SqlValue::Null),
1168                Some(ColumnValue::String(s)) => ToSqlOutput::Borrowed(ValueRef::Text(s.as_bytes())),
1169                Some(ColumnValue::Integer(i)) => ToSqlOutput::Owned(SqlValue::Integer(*i)),
1170                Some(ColumnValue::Float(f)) => ToSqlOutput::Owned(SqlValue::Real(*f)),
1171                Some(ColumnValue::Boolean(b)) => {
1172                    ToSqlOutput::Owned(SqlValue::Integer(i64::from(*b)))
1173                }
1174                Some(ColumnValue::Json(raw)) | Some(ColumnValue::BlobRef(raw)) => {
1175                    ToSqlOutput::Borrowed(ValueRef::Text(raw.0.as_bytes()))
1176                }
1177                // §5.10: crdt bytes store as BLOB, like bytes.
1178                Some(ColumnValue::Bytes(b)) | Some(ColumnValue::Crdt(b)) => {
1179                    ToSqlOutput::Borrowed(ValueRef::Blob(b))
1180                }
1181            },
1182        })
1183    }
1184}
1185
1186// Retain a typed primary key across the current import chunk without copying row payloads.
1187fn owned_sql_value(param: impl rusqlite::ToSql) -> Result<SqlValue, String> {
1188    match param.to_sql().map_err(|error| error.to_string())? {
1189        ToSqlOutput::Owned(value) => Ok(value),
1190        ToSqlOutput::Borrowed(value) => {
1191            SqlValue::try_from(value).map_err(|error| error.to_string())
1192        }
1193        _ => Err("sync.invalid_request: unsupported SQLite key parameter".into()),
1194    }
1195}
1196
1197/// §5.3 image cell → bind parameter, strict per the declared column type
1198/// (`boolean` from INTEGER 0/1, `json` from its raw TEXT, NULL only when
1199/// nullable). Mismatches are image-producer violations. Returns the cell
1200/// borrowed when the stored representation already matches what the row
1201/// codec would write, or the normalized scalar (boolean → 0/1, float from
1202/// INTEGER → REAL) otherwise — the local write is byte-identical to the
1203/// old convert-then-insert path without allocating per cell.
1204fn image_cell_param<'a>(column: &Column, value: ValueRef<'a>) -> Result<ToSqlOutput<'a>, String> {
1205    use ssp2::segment::ColumnType;
1206    let mismatch = || {
1207        Err(format!(
1208            "image column {:?} holds a value of the wrong type",
1209            column.name
1210        ))
1211    };
1212    match value {
1213        ValueRef::Null => {
1214            if !column.nullable {
1215                return Err(format!(
1216                    "image column {:?} is NULL but not nullable",
1217                    column.name
1218                ));
1219            }
1220            Ok(ToSqlOutput::Owned(SqlValue::Null))
1221        }
1222        ValueRef::Integer(i) => match column.ty {
1223            ColumnType::Integer => Ok(ToSqlOutput::Borrowed(value)),
1224            ColumnType::Boolean => Ok(ToSqlOutput::Owned(SqlValue::Integer(i64::from(i != 0)))),
1225            ColumnType::Float => Ok(ToSqlOutput::Owned(SqlValue::Real(i as f64))),
1226            _ => mismatch(),
1227        },
1228        ValueRef::Real(_) => match column.ty {
1229            ColumnType::Float => Ok(ToSqlOutput::Borrowed(value)),
1230            _ => mismatch(),
1231        },
1232        ValueRef::Text(t) => {
1233            std::str::from_utf8(t)
1234                .map_err(|_| format!("image column {:?} is not UTF-8", column.name))?;
1235            match column.ty {
1236                ColumnType::String | ColumnType::Json | ColumnType::BlobRef => {
1237                    Ok(ToSqlOutput::Borrowed(value))
1238                }
1239                _ => mismatch(),
1240            }
1241        }
1242        ValueRef::Blob(_) => match column.ty {
1243            // §5.10: a crdt column stores its opaque bytes as BLOB, like bytes.
1244            ColumnType::Bytes | ColumnType::Crdt => Ok(ToSqlOutput::Borrowed(value)),
1245            _ => mismatch(),
1246        },
1247    }
1248}
1249
1250pub(crate) fn sql_ref_to_json(column: &Column, value: rusqlite::types::ValueRef<'_>) -> Value {
1251    use rusqlite::types::ValueRef;
1252    match value {
1253        ValueRef::Null => Value::Null,
1254        ValueRef::Integer(i) => match column.ty {
1255            ssp2::segment::ColumnType::Boolean => Value::Bool(i != 0),
1256            ssp2::segment::ColumnType::Float => {
1257                serde_json::Number::from_f64(i as f64).map_or(Value::Null, Value::Number)
1258            }
1259            _ => Value::from(i),
1260        },
1261        ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
1262        ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
1263        ValueRef::Blob(b) => {
1264            let mut map = Map::new();
1265            map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
1266            Value::Object(map)
1267        }
1268    }
1269}
1270
1271/// Bind a driver JSON value form as a rusqlite parameter for [`SyncClient::query`].
1272/// Objects are accepted in lossless `{"$bytes": hex}` and
1273/// `{"$bigint": decimal}` envelope forms.
1274fn json_param_to_sql(value: &Value) -> Result<SqlValue, String> {
1275    Ok(match value {
1276        Value::Null => SqlValue::Null,
1277        Value::Bool(b) => SqlValue::Integer(i64::from(*b)),
1278        Value::Number(n) => {
1279            if let Some(i) = n.as_i64() {
1280                SqlValue::Integer(i)
1281            } else if let Some(f) = n.as_f64() {
1282                SqlValue::Real(f)
1283            } else {
1284                return Err(format!("query param number {n} is out of range"));
1285            }
1286        }
1287        Value::String(s) => SqlValue::Text(s.clone()),
1288        Value::Object(_) => {
1289            if let Some(hex) = value.get("$bytes").and_then(Value::as_str) {
1290                SqlValue::Blob(crate::values::hex_to_bytes(hex)?)
1291            } else if let Some(decimal) = value.get("$bigint").and_then(Value::as_str) {
1292                SqlValue::Integer(decimal.parse::<i64>().map_err(|_| {
1293                    format!("query bigint param {decimal:?} is outside SQLite's i64 range")
1294                })?)
1295            } else {
1296                return Err(
1297                    "query object param must be a {$bytes: hex} or {$bigint: decimal} value"
1298                        .to_owned(),
1299                );
1300            }
1301        }
1302        Value::Array(_) => return Err("query array params are not supported".to_owned()),
1303    })
1304}
1305
1306/// Map a rusqlite value with no schema column to consult (arbitrary query
1307/// output): integers/reals/text pass through by stored affinity, blobs ride
1308/// as `{"$bytes": hex}`. Distinct from [`sql_ref_to_json`], which uses the
1309/// schema column type to recover booleans/floats/json.
1310fn sql_ref_to_json_dynamic(value: rusqlite::types::ValueRef<'_>) -> Value {
1311    use rusqlite::types::ValueRef;
1312    match value {
1313        ValueRef::Null => Value::Null,
1314        ValueRef::Integer(i) => {
1315            // JSON/Tauri IPC cannot represent every SQLite i64 exactly. Keep
1316            // ordinary UI-sized integers ergonomic and envelope only values
1317            // beyond JavaScript's safe range.
1318            const MAX_SAFE_INTEGER: i64 = 9_007_199_254_740_991;
1319            if (-MAX_SAFE_INTEGER..=MAX_SAFE_INTEGER).contains(&i) {
1320                Value::from(i)
1321            } else {
1322                let mut map = Map::new();
1323                map.insert("$bigint".to_owned(), Value::from(i.to_string()));
1324                Value::Object(map)
1325            }
1326        }
1327        ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
1328        ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
1329        ValueRef::Blob(b) => {
1330            let mut map = Map::new();
1331            map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
1332            Value::Object(map)
1333        }
1334    }
1335}
1336
1337/// §7.5: SQLite failures classify by result code into [`QueryReadFailure`].
1338fn query_connection(
1339    conn: &Connection,
1340    sql: &str,
1341    params: &[QueryValue],
1342) -> Result<Vec<QueryRow>, QueryReadFailure> {
1343    let lowered_sql = crate::query_guard::lower_public_query_sql(sql);
1344    let mut stmt = crate::query_guard::prepare_read_only(conn, &lowered_sql)?;
1345    let bound: Vec<SqlValue> = params
1346        .iter()
1347        .map(json_param_to_sql)
1348        .collect::<Result<_, _>>()?;
1349    // §2.4: an unaliased physical version column stays engine-internal, so a
1350    // `SELECT *` row feeds straight back into mutation values. An explicit
1351    // alias (`_sync_version AS v`) returns it as ordinary query data. The
1352    // §5.11 sealed columns follow the same rule.
1353    let columns: Vec<(usize, String)> = stmt
1354        .column_names()
1355        .enumerate()
1356        .filter(|(_, name)| {
1357            !matches!(
1358                *name,
1359                "_syncular_version" | "_sync_sealed_key" | "_sync_sealed"
1360            )
1361        })
1362        .map(|(i, name)| (i, name.to_owned()))
1363        .collect();
1364    let bound_refs: Vec<&dyn rusqlite::ToSql> =
1365        bound.iter().map(|v| v as &dyn rusqlite::ToSql).collect();
1366    let mut sql_rows = stmt.query(bound_refs.as_slice())?;
1367    let mut out = Vec::new();
1368    while let Some(row) = sql_rows.next()? {
1369        let mut record = Map::new();
1370        for (i, name) in &columns {
1371            let value = row.get_ref(*i)?;
1372            record.insert(name.clone(), sql_ref_to_json_dynamic(value));
1373        }
1374        out.push(record);
1375    }
1376    Ok(out)
1377}
1378
1379fn persisted_window_state(
1380    conn: &Connection,
1381    base: &WindowBase,
1382) -> Result<WindowState, QueryReadFailure> {
1383    let mut stmt = conn.prepare(
1384        "SELECT windows.unit, units.status, units.cursor, units.bootstrap_state
1385           FROM _syncular_windows AS windows
1386           LEFT JOIN _syncular_units AS units ON units.id = windows.sub_id
1387          WHERE windows.base = ?1
1388          ORDER BY windows.unit ASC",
1389    )?;
1390    let rows = stmt.query_map(rusqlite::params![window_base_key(base)], |row| {
1391        Ok((
1392            row.get::<_, String>(0)?,
1393            row.get::<_, Option<String>>(1)?,
1394            row.get::<_, Option<i64>>(2)?,
1395            row.get::<_, Option<String>>(3)?,
1396        ))
1397    })?;
1398    let mut units = Vec::new();
1399    let mut pending = Vec::new();
1400    for row in rows {
1401        let (unit, status, cursor, bootstrap_state) = row?;
1402        // §4.8: complete iff registered, active and not bootstrapping.
1403        if status.as_deref() != Some("active")
1404            || cursor.is_some_and(|cursor| cursor < 0 || bootstrap_state.is_some())
1405        {
1406            pending.push(unit.clone());
1407        }
1408        units.push(unit);
1409    }
1410    Ok(WindowState { units, pending })
1411}
1412
1413/// One read-only statement in a §7.5 snapshot read.
1414#[derive(Debug, Clone, Copy)]
1415pub struct SnapshotStatement<'a> {
1416    pub sql: &'a str,
1417    pub params: &'a [Value],
1418}
1419
1420/// A §7.5 atomic read request: read-only statements, window coverage, unit
1421/// catch-up states, and commit deliveries, all resolved from one SQLite read
1422/// transaction and one revision.
1423#[derive(Debug, Clone, Default)]
1424pub struct SnapshotReadRequest<'a> {
1425    pub statements: Vec<SnapshotStatement<'a>>,
1426    pub coverage: &'a [WindowCoverage],
1427    pub units: Vec<String>,
1428    pub commit_ids: Vec<String>,
1429}
1430
1431#[cfg(test)]
1432thread_local! {
1433    /// Test-only rendezvous that runs between the statements of one snapshot
1434    /// read. A test uses it to commit on another connection mid-read and prove
1435    /// the read transaction's snapshot and revision stay stable.
1436    static SNAPSHOT_READ_HOOK: std::cell::RefCell<Option<Box<dyn FnOnce()>>> =
1437        std::cell::RefCell::new(None);
1438}
1439
1440#[cfg(test)]
1441fn run_snapshot_read_hook() {
1442    SNAPSHOT_READ_HOOK.with(|slot| {
1443        if let Some(hook) = slot.borrow_mut().take() {
1444            hook();
1445        }
1446    });
1447}
1448
1449/// §7.5: resolve a request over one SQLite read transaction. The revision,
1450/// every statement's rows, coverage, catch-up states, and deliveries are
1451/// captured under one `SAVEPOINT`; any statement's failure rolls the whole
1452/// read back and releases the connection.
1453fn snapshot_read_connection(
1454    conn: &Connection,
1455    request: &SnapshotReadRequest<'_>,
1456) -> Result<SnapshotRead, QueryReadFailure> {
1457    conn.execute_batch("SAVEPOINT syncular_snapshot_read")?;
1458    let corrupt = |message: &str| QueryReadFailure {
1459        code: Some("sync.local_corrupt"),
1460        sqlite_code: None,
1461        sqlite_message: None,
1462        rollback_failure: None,
1463        message: message.to_owned(),
1464    };
1465    let stored_failure = |error: rusqlite::Error, message: &str| match error {
1466        rusqlite::Error::Utf8Error(..)
1467        | rusqlite::Error::InvalidColumnType(..)
1468        | rusqlite::Error::FromSqlConversionFailure(..)
1469        | rusqlite::Error::IntegralValueOutOfRange(..) => corrupt(message),
1470        _ => QueryReadFailure::from(error),
1471    };
1472    let result: Result<SnapshotRead, QueryReadFailure> = (|| {
1473        // The durable revision as a canonical decimal `u64`; a missing marker
1474        // is the legacy zero, and any other non-canonical value is corrupt.
1475        let revision: Option<String> = conn
1476            .query_row(
1477                "SELECT value FROM _syncular_meta WHERE key = ?1",
1478                rusqlite::params![LOCAL_REVISION_KEY],
1479                |row| row.get(0),
1480            )
1481            .optional()
1482            .map_err(|error| stored_failure(error, "persisted local revision is invalid"))?;
1483        let revision = match revision {
1484            None => "0".to_owned(),
1485            Some(value)
1486                if value == "0"
1487                    || (matches!(value.as_bytes().first(), Some(b'1'..=b'9'))
1488                        && value.bytes().all(|b| b.is_ascii_digit())
1489                        && value.parse::<u64>().is_ok()) =>
1490            {
1491                value
1492            }
1493            Some(_) => return Err(corrupt("persisted local revision is invalid")),
1494        };
1495        let mut queries = Vec::with_capacity(request.statements.len());
1496        for statement in &request.statements {
1497            queries.push(query_connection(conn, statement.sql, statement.params)?);
1498            #[cfg(test)]
1499            run_snapshot_read_hook();
1500        }
1501        let mut pending = Vec::new();
1502        let mut missing = Vec::new();
1503        for requested in request.coverage {
1504            let base_key = window_base_key(&requested.base);
1505            let state = persisted_window_state(conn, &requested.base)?;
1506            for unit in BTreeSet::from_iter(requested.units.iter().cloned()) {
1507                let reference = WindowUnitRef {
1508                    base_key: base_key.clone(),
1509                    unit: unit.clone(),
1510                };
1511                if !state.units.iter().any(|held| held == &unit) {
1512                    missing.push(reference);
1513                } else if state.pending.iter().any(|held| held == &unit) {
1514                    pending.push(reference);
1515                }
1516            }
1517        }
1518        let mut units = Vec::with_capacity(request.units.len());
1519        for id in &request.units {
1520            let exists: bool = conn.query_row(
1521                "SELECT EXISTS(SELECT 1 FROM _syncular_units WHERE id = ?1)",
1522                [id],
1523                |row| row.get(0),
1524            )?;
1525            if !exists {
1526                units.push(UnitCatchup::Unknown { id: id.clone() });
1527                continue;
1528            }
1529            // Every persisted field is required to be well typed; a corrupt
1530            // record fails typed instead of inventing a state.
1531            let unit = get_unit(conn, id)
1532                .map_err(|error| stored_failure(error, "persisted unit state is invalid"))?
1533                .ok_or_else(|| corrupt("persisted unit state is invalid"))?;
1534            let cursor = match unit.cursor {
1535                Some(cursor) => cursor,
1536                None => replica_cursor(conn)
1537                    .map_err(|error| stored_failure(error, "persisted unit state is invalid"))?,
1538            };
1539            if cursor.unsigned_abs() > MAX_JS_SAFE_INTEGER {
1540                return Err(corrupt("persisted unit state is invalid"));
1541            }
1542            let active = unit.status == UnitStatus::Active;
1543            let pending = unit.bootstrapping();
1544            units.push(UnitCatchup::Known {
1545                id: id.clone(),
1546                status: unit.status,
1547                live: unit.cursor.is_none(),
1548                cursor,
1549                has_resume_token: unit.bootstrap_state.is_some(),
1550                bootstrap_complete: active && !pending,
1551                known_pending_pages: active && pending,
1552                reason_code: unit.reason_code,
1553            });
1554        }
1555        let mut deliveries = Vec::with_capacity(request.commit_ids.len());
1556        for id in &request.commit_ids {
1557            let pending: bool = conn.query_row(
1558                "SELECT EXISTS(SELECT 1 FROM _syncular_outbox WHERE commit_id = ?1)",
1559                rusqlite::params![id],
1560                |row| row.get(0),
1561            )?;
1562            if pending {
1563                deliveries.push(CommitDelivery::Pending {
1564                    client_commit_id: id.clone(),
1565                });
1566                continue;
1567            }
1568            let row = conn
1569                .query_row(
1570                    "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
1571                            resolution, resolved_at_ms, replacement_client_commit_id
1572                       FROM _syncular_commit_outcomes WHERE client_commit_id = ?1",
1573                    rusqlite::params![id],
1574                    StoredCommitOutcomeRow::from_row,
1575                )
1576                .optional()
1577                .map_err(|error| stored_failure(error, "persisted commit outcome is invalid"))?;
1578            deliveries.push(match row {
1579                Some(row) => CommitDelivery::Known {
1580                    client_commit_id: id.clone(),
1581                    outcome: SyncClient::outcome_from_row(row)
1582                        .map_err(|_| corrupt("persisted commit outcome is invalid"))?,
1583                },
1584                None => CommitDelivery::Unknown {
1585                    client_commit_id: id.clone(),
1586                },
1587            });
1588        }
1589        Ok(SnapshotRead {
1590            revision,
1591            queries,
1592            coverage: CoverageSnapshot {
1593                complete: pending.is_empty() && missing.is_empty(),
1594                pending,
1595                missing,
1596            },
1597            units,
1598            deliveries,
1599        })
1600    })();
1601    match result {
1602        Ok(read) => {
1603            conn.execute_batch("RELEASE syncular_snapshot_read")?;
1604            Ok(read)
1605        }
1606        Err(mut error) => {
1607            if let Err(rollback) = conn
1608                .execute_batch("ROLLBACK TO syncular_snapshot_read; RELEASE syncular_snapshot_read")
1609            {
1610                error.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
1611            }
1612            Err(error)
1613        }
1614    }
1615}
1616
1617fn snapshot_connection(
1618    conn: &Connection,
1619    sql: &str,
1620    params: &[Value],
1621    coverage: &[WindowCoverage],
1622) -> Result<QuerySnapshot, QueryReadFailure> {
1623    let read = snapshot_read_connection(
1624        conn,
1625        &SnapshotReadRequest {
1626            statements: vec![SnapshotStatement { sql, params }],
1627            coverage,
1628            units: Vec::new(),
1629            commit_ids: Vec::new(),
1630        },
1631    )?;
1632    Ok(QuerySnapshot {
1633        revision: read.revision,
1634        rows: read
1635            .queries
1636            .into_iter()
1637            .next()
1638            .expect("single-query snapshot returns one result"),
1639        coverage: read.coverage,
1640    })
1641}
1642
1643/// A long-lived read-only SQLite sidecar for latency-critical native views.
1644/// Network rounds stay serialized on the mutable core owner, while atomic
1645/// query snapshots use this independent connection and therefore never queue
1646/// behind HTTP/WebSocket latency.
1647pub struct FileQuerySnapshotReader {
1648    path: String,
1649    conn: Option<Connection>,
1650}
1651
1652impl FileQuerySnapshotReader {
1653    #[must_use]
1654    pub fn new(path: impl Into<String>) -> Self {
1655        Self {
1656            path: path.into(),
1657            conn: None,
1658        }
1659    }
1660
1661    fn connection(&mut self) -> Result<&Connection, QueryReadFailure> {
1662        if self.conn.is_none() {
1663            let conn = Connection::open_with_flags(
1664                &self.path,
1665                OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
1666            )
1667            .map_err(QueryReadFailure::from)?;
1668            conn.busy_timeout(std::time::Duration::from_millis(250))
1669                .map_err(QueryReadFailure::from)?;
1670            self.conn = Some(conn);
1671        }
1672        self.conn
1673            .as_ref()
1674            .ok_or_else(|| QueryReadFailure::from("read sidecar connection missing".to_owned()))
1675    }
1676
1677    /// One atomic read over the shared primitive: every statement, coverage,
1678    /// catch-up, and delivery under one `SAVEPOINT` and one revision.
1679    pub fn snapshot_read(
1680        &mut self,
1681        request: &SnapshotReadRequest<'_>,
1682    ) -> Result<SnapshotRead, QueryReadFailure> {
1683        snapshot_read_connection(self.connection()?, request)
1684    }
1685
1686    /// The current durable local revision. A host can skip a read when it is
1687    /// unchanged; a corrupt or unreadable marker fails typed.
1688    pub fn local_revision(&mut self) -> Result<String, QueryReadFailure> {
1689        Ok(self
1690            .snapshot_read(&SnapshotReadRequest::default())?
1691            .revision)
1692    }
1693
1694    /// One unit's persisted catch-up state (see the §7.5 contract).
1695    pub fn unit_catchup(&mut self, id: &str) -> Result<UnitCatchup, QueryReadFailure> {
1696        let read = self.snapshot_read(&SnapshotReadRequest {
1697            units: vec![id.to_owned()],
1698            ..Default::default()
1699        })?;
1700        Ok(read
1701            .units
1702            .into_iter()
1703            .next()
1704            .expect("one requested catch-up"))
1705    }
1706
1707    /// One client commit's delivery status: pending, its retained outcome, or
1708    /// an explicit unknown (never held, or pruned by retention).
1709    pub fn commit_delivery(&mut self, id: &str) -> Result<CommitDelivery, QueryReadFailure> {
1710        let read = self.snapshot_read(&SnapshotReadRequest {
1711            commit_ids: vec![id.to_owned()],
1712            ..Default::default()
1713        })?;
1714        Ok(read
1715            .deliveries
1716            .into_iter()
1717            .next()
1718            .expect("one requested delivery"))
1719    }
1720
1721    /// Hosts report an owned failure, and the first success after it, to the
1722    /// owning core through [`SyncClient::record_query_read`] (§7.6).
1723    pub fn query_snapshot(
1724        &mut self,
1725        sql: &str,
1726        params: &[Value],
1727        coverage: &[WindowCoverage],
1728    ) -> Result<QuerySnapshot, QueryReadFailure> {
1729        snapshot_connection(self.connection()?, sql, params, coverage)
1730    }
1731}
1732
1733fn validate_authority_reads(
1734    schema: &ClientSchema,
1735    declarations: &[crate::api::AuthorityReadDeclaration],
1736) -> Result<(), String> {
1737    let mut seen = BTreeSet::new();
1738    for read in declarations {
1739        let table = schema
1740            .table(&read.table)
1741            .ok_or("client.authority_read_forbidden: authority table is not declared")?;
1742        let plain = |name: &str| {
1743            table.columns.iter().enumerate().any(|(i, c)| {
1744                c.name == name
1745                    && name != "authority_version"
1746                    && !name.starts_with("_sync")
1747                    && !table.encrypted_columns.iter().any(|e| e.index == i)
1748                    && matches!(
1749                        c.ty,
1750                        ssp2::segment::ColumnType::String
1751                            | ssp2::segment::ColumnType::Integer
1752                            | ssp2::segment::ColumnType::Float
1753                            | ssp2::segment::ColumnType::Boolean
1754                            | ssp2::segment::ColumnType::Json
1755                    )
1756            })
1757        };
1758        if !seen.insert(&read.table)
1759            || read.columns.is_empty()
1760            || read.columns.iter().collect::<BTreeSet<_>>().len() != read.columns.len()
1761            || !read.columns.contains(&table.primary_key)
1762            || !read.columns.iter().all(|c| plain(c))
1763            || read.scopes.is_empty()
1764            || !read.scopes.iter().all(|(v, values)| {
1765                table
1766                    .scope_column(v)
1767                    .is_some_and(|c| read.columns.iter().any(|column| column == c) && plain(c))
1768                    && !values.is_empty()
1769                    && values.iter().all(|v| !v.is_empty() && v != "*")
1770            })
1771        {
1772            return Err("client.authority_read_forbidden: authority reads require plain columns, a primary key and nonempty plain scope selectors".into());
1773        }
1774    }
1775    Ok(())
1776}
1777
1778// §7.4.2: every open and recreation checks before any persistent writes.
1779/// A marker read that hit a recognized SQLite storage failure keeps that
1780/// classification (BUSY/LOCKED stay retryable); only an unclassified read
1781/// failure is the invalid-marker `sync.local_corrupt`.
1782fn schema_marker_read_failure(error: rusqlite::Error) -> String {
1783    let failure = QueryReadFailure::from(error);
1784    if failure.code.is_some() {
1785        failure.to_string()
1786    } else {
1787        "sync.local_corrupt: persisted local schema marker is unreadable or invalid".to_owned()
1788    }
1789}
1790
1791/// §7.1: reject an invalid configured client limit before any storage is
1792/// opened or written. Zero would wedge the push queue and an out-of-range
1793/// value would differ per target; the TS core enforces the same bounds.
1794fn validate_client_limits(limits: &crate::api::ClientLimits) -> Result<(), String> {
1795    if limits.outcome_retention_max_entries == Some(0) {
1796        return Err("sync.invalid_request: outcomeRetentionMaxEntries must be positive".to_owned());
1797    }
1798    if [
1799        limits.max_push_commits_per_request,
1800        limits.max_push_operations_per_request,
1801        limits.max_push_request_bytes,
1802    ]
1803    .into_iter()
1804    .flatten()
1805    .any(|value| value == 0 || value > u32::MAX as usize)
1806    {
1807        return Err(
1808            "sync.invalid_request: push limit must be an integer in 1..=4294967295".to_owned(),
1809        );
1810    }
1811    Ok(())
1812}
1813
1814fn read_local_schema_version(conn: &Connection, requested: i32) -> Result<Option<i32>, String> {
1815    let corrupt =
1816        || "sync.local_corrupt: persisted local schema marker is unreadable or invalid".to_owned();
1817    let table_type: Option<String> = conn
1818        .query_row(
1819            "SELECT type FROM sqlite_master WHERE name = '_syncular_meta'",
1820            [],
1821            |row| row.get(0),
1822        )
1823        .optional()
1824        .map_err(schema_marker_read_failure)?;
1825    let Some(table_type) = table_type else {
1826        return Ok(None);
1827    };
1828    if table_type != "table" {
1829        return Err(corrupt());
1830    }
1831    // §7.4.2: a duplicated marker cannot be resolved. `query_row` would take
1832    // whichever row SQLite returns first, and an older one would drive a
1833    // destructive reset over a newer replica.
1834    let mut statement = conn
1835        .prepare("SELECT value FROM _syncular_meta WHERE key = ?1")
1836        .map_err(schema_marker_read_failure)?;
1837    let mut rows = statement
1838        .query([LOCAL_SCHEMA_VERSION_KEY])
1839        .map_err(schema_marker_read_failure)?;
1840    let value: Option<String> = rows
1841        .next()
1842        .map_err(schema_marker_read_failure)?
1843        .map(|row| row.get(0))
1844        .transpose()
1845        .map_err(schema_marker_read_failure)?;
1846    if rows.next().map_err(schema_marker_read_failure)?.is_some() {
1847        return Err(corrupt());
1848    }
1849    let Some(value) = value else {
1850        let has_descriptor = conn
1851            .query_row(
1852                "SELECT 1 FROM _syncular_meta WHERE key = ?1 LIMIT 1",
1853                [crate::previous_version::LOCAL_SCHEMA_DESCRIPTOR_KEY],
1854                |_| Ok(()),
1855            )
1856            .optional()
1857            .map_err(schema_marker_read_failure)?
1858            .is_some();
1859        if has_descriptor {
1860            return Err(corrupt());
1861        }
1862        return Ok(None);
1863    };
1864    if !matches!(value.as_bytes().first(), Some(b'1'..=b'9'))
1865        || !value.bytes().all(|b| b.is_ascii_digit())
1866    {
1867        return Err(corrupt());
1868    }
1869    let version = value.parse::<i32>().map_err(|_| corrupt())?;
1870    if version > requested {
1871        return Err(
1872            "client.schema_downgrade: persisted local schema is newer than the requested schema"
1873                .to_owned(),
1874        );
1875    }
1876    Ok(Some(version))
1877}
1878
1879/// Open a replica file. On a browser worker the file must recover a hot
1880/// journal before any statement reads it: `wasm_vfs` replaces the installed
1881/// OPFS VFS's reserved lock callback before the connection opens its file. A
1882/// new browser replica uses 64 KiB pages (§5.3), because every OPFS call costs
1883/// the same whatever its size; a database with tables keeps its page size. Its
1884/// page cache holds 64 MiB, because SQLite's 2,000 KiB default is 31 such pages
1885/// and every cache miss is an OPFS call.
1886fn open_replica(path: &str) -> Result<Connection, String> {
1887    let browser = cfg!(all(target_arch = "wasm32", target_os = "unknown"))
1888        && !path.is_empty()
1889        && path != ":memory:";
1890    #[cfg(all(target_arch = "wasm32", target_os = "unknown"))]
1891    if browser {
1892        crate::wasm_vfs::disable_foreign_lock_check()?;
1893    }
1894    let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
1895    if browser {
1896        conn.execute_batch("PRAGMA page_size = 65536; PRAGMA cache_size = -65536")
1897            .map_err(|e| format!("open db {path:?}: {e}"))?;
1898    }
1899    Ok(conn)
1900}
1901
1902impl SyncClient {
1903    pub fn new_with_identity(
1904        client_id: Option<String>,
1905        schema_json: &Value,
1906        limits: ClientLimits,
1907    ) -> Result<Self, String> {
1908        let resolved = client_id.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
1909        let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
1910        Self::with_connection(resolved, schema_json, limits, conn)
1911    }
1912
1913    pub fn new(
1914        client_id: String,
1915        schema_json: &Value,
1916        limits: ClientLimits,
1917    ) -> Result<Self, String> {
1918        let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
1919        Self::with_connection(client_id, schema_json, limits, conn)
1920    }
1921
1922    /// Build a client backed by an on-disk SQLite database at `path` — the
1923    /// seam a native host (Tauri plugin, FFI file-DB variant) uses to persist
1924    /// across process restarts. `create_tables` runs `IF NOT EXISTS`, so
1925    /// re-opening the same file reuses the persisted rows. Keeps rusqlite out
1926    /// of the command router's dependency set (the router only holds a path).
1927    pub fn open_path(
1928        client_id: String,
1929        schema_json: &Value,
1930        limits: ClientLimits,
1931        path: &str,
1932    ) -> Result<Self, String> {
1933        validate_client_limits(&limits)?;
1934        parse_schema_json(schema_json)?;
1935        let conn = open_replica(path)?;
1936        Self::with_connection(client_id, schema_json, limits, conn)
1937    }
1938
1939    /// Open a file replica and resolve its client id. Native targets require
1940    /// WAL for the independent snapshot reader; `wasm32-unknown-unknown`
1941    /// accepts the sqlite-wasm worker journal instead.
1942    pub fn open_path_with_identity(
1943        client_id: Option<String>,
1944        schema_json: &Value,
1945        limits: ClientLimits,
1946        path: &str,
1947    ) -> Result<Self, String> {
1948        Self::open_path_with_identity_journal(
1949            client_id,
1950            schema_json,
1951            limits,
1952            path,
1953            !cfg!(all(target_arch = "wasm32", target_os = "unknown")),
1954        )
1955    }
1956
1957    fn open_path_with_identity_journal(
1958        client_id: Option<String>,
1959        schema_json: &Value,
1960        limits: ClientLimits,
1961        path: &str,
1962        require_wal: bool,
1963    ) -> Result<Self, String> {
1964        validate_client_limits(&limits)?;
1965        // §7.4.2: an invalid requested version is refused before the replica is
1966        // even created, so a rejected constructor leaves no file behind.
1967        parse_schema_json(schema_json)?;
1968        let conn = open_replica(path)?;
1969        // A file-backed native client uses an independent read connection for
1970        // latency-critical snapshots. WAL is SQLite's intended reader/writer
1971        // concurrency mode: a view read never holds a rollback-journal lock
1972        // that delays the client's next commit. The busy timeout must precede
1973        // this replica's own startup transaction; WAL is applied only after
1974        // startup succeeded, because a refused open must leave the replica's
1975        // journal mode and contents untouched (§7.4.2).
1976        conn.busy_timeout(std::time::Duration::from_millis(250))
1977            .map_err(|error| format!("configure db {path:?} busy timeout: {error}"))?;
1978        // Durability is the caller's contract here, not the VFS default: a
1979        // rollback-journal commit needs FULL before it writes the database
1980        // pages, or a killed worker can leave a half-written image behind.
1981        // WAL below inherits the same setting.
1982        conn.pragma_update(None, "synchronous", "FULL")
1983            .map_err(|error| format!("configure db {path:?} synchronous: {error}"))?;
1984        let client = Self::with_connection_identity(client_id, schema_json, limits, conn)?;
1985        // A browser worker's sqlite-wasm VFS has no WAL mode, and the wasm
1986        // host has no independent snapshot reader. The WAL requirement exists
1987        // for native file replicas, where it is the reader/writer boundary.
1988        if require_wal {
1989            let journal_mode: String = client
1990                .conn
1991                .pragma_update_and_check(None, "journal_mode", "WAL", |row| row.get(0))
1992                .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
1993            if journal_mode != "wal" {
1994                return Err(
1995                    "sync.invalid_request: file-backed client requires WAL journal mode".to_owned(),
1996                );
1997            }
1998        }
1999        Ok(client)
2000    }
2001
2002    /// Delete a file replica at `path`, including its `-journal`, `-wal`, and
2003    /// `-shm` sidecars, through the VFS that owns it. Use this after
2004    /// `client.storage_corrupt`: an OPFS SAH pool stores those files under
2005    /// opaque names, so deleting the pool directory behind SQLite bypasses the
2006    /// pool's slot accounting.
2007    ///
2008    /// Close the client and every [`FileQuerySnapshotReader`] on `path` first;
2009    /// a VFS cannot delete a file another handle still owns. A file that does
2010    /// not exist counts as deleted, so a cleanly closed replica without
2011    /// sidecars resets. The next open
2012    /// creates a fresh replica (new client id) and re-bootstraps from the
2013    /// server. A memory replica has nothing to delete and fails with
2014    /// `sync.invalid_request`.
2015    pub fn reset_replica(path: &str) -> Result<(), String> {
2016        if path.is_empty() || path == ":memory:" {
2017            return Err("sync.invalid_request: a memory replica has nothing to reset".to_owned());
2018        }
2019        // SAFETY: a null VFS name asks SQLite for its process-wide default VFS,
2020        // the same one `Connection::open(path)` above selects. `xDelete` is
2021        // that VFS's own function pointer, and `c_name` outlives the call.
2022        let vfs = unsafe { rusqlite::ffi::sqlite3_vfs_find(std::ptr::null()) };
2023        if vfs.is_null() {
2024            return Err("client.storage_io: no SQLite VFS is registered".to_owned());
2025        }
2026        let delete = unsafe { (*vfs).xDelete }
2027            .ok_or_else(|| "client.storage_io: the SQLite VFS cannot delete files".to_owned())?;
2028        for suffix in ["-wal", "-shm", "-journal", ""] {
2029            let file_name = format!("{path}{suffix}");
2030            let c_name = std::ffi::CString::new(file_name.as_str()).map_err(|_| {
2031                format!("sync.invalid_request: replica path {file_name:?} contains a NUL byte")
2032            })?;
2033            // SAFETY: `vfs` and its `xDelete` pointer come from the live
2034            // SQLite library; `c_name` is a valid NUL-terminated path.
2035            let code = unsafe { delete(vfs, c_name.as_ptr(), 1) };
2036            // A cleanly closed WAL replica has no sidecars; a missing file is
2037            // already deleted.
2038            if code != rusqlite::ffi::SQLITE_OK && code != rusqlite::ffi::SQLITE_IOERR_DELETE_NOENT
2039            {
2040                return Err(format!(
2041                    "client.storage_io: deleting replica file {file_name:?} failed with SQLite code {code}"
2042                ));
2043            }
2044        }
2045        Ok(())
2046    }
2047
2048    #[must_use]
2049    pub fn client_id(&self) -> &str {
2050        &self.client_id
2051    }
2052
2053    /// Rotate the caller's client id (§1.5) and return the new one. Use it
2054    /// after a `sync.invalid_client_id` refusal (`recommendedAction:
2055    /// resetClientId`): the next round carries the new id. A live realtime
2056    /// socket is closed so the next connection registers with the new id.
2057    /// The replica and its outbox stay in place; wipe them yourself when the
2058    /// stored data belongs to the account that signed out.
2059    pub fn reset_client_id(&mut self, transport: &mut dyn Transport) -> Result<String, String> {
2060        self.disconnect_realtime(transport);
2061        let client_id = uuid::Uuid::new_v4().to_string();
2062        self.set_meta(CLIENT_ID_KEY, &client_id)?;
2063        self.client_id.clone_from(&client_id);
2064        self.round_failure = None;
2065        Ok(client_id)
2066    }
2067
2068    /// Build a client over a caller-supplied rusqlite connection — the seam a
2069    /// native host (Tauri plugin, FFI file-DB variant) uses to back the core
2070    /// with an on-disk database (`Connection::open(path)`) rather than the
2071    /// default `:memory:`. The connection MUST be fresh (no pre-existing
2072    /// syncular tables); `create_tables` runs `IF NOT EXISTS`, so re-opening
2073    /// the same file across process restarts reuses the persisted rows.
2074    pub fn with_connection(
2075        client_id: String,
2076        schema_json: &Value,
2077        limits: ClientLimits,
2078        conn: Connection,
2079    ) -> Result<Self, String> {
2080        Self::with_connection_identity(Some(client_id), schema_json, limits, conn)
2081    }
2082
2083    /// [`Self::with_connection`] for `open_path_with_identity`: no requested id
2084    /// means a durable replica's persisted `clientId` wins over a fresh one.
2085    fn with_connection_identity(
2086        client_id: Option<String>,
2087        schema_json: &Value,
2088        limits: ClientLimits,
2089        conn: Connection,
2090    ) -> Result<Self, String> {
2091        validate_client_limits(&limits)?;
2092        let schema = parse_schema_json(schema_json)?;
2093        validate_authority_reads(&schema, &limits.authority_reads)?;
2094        // RFC 0005 D8: resolve the previous-version config before the opening
2095        // reset runs — the capture happens inside it. Bad bounds fail loud.
2096        let previous_version = match limits.previous_version_context {
2097            Some(config) => {
2098                config.validate()?;
2099                Some(config)
2100            }
2101            None => None,
2102        };
2103        let mut client = SyncClient {
2104            #[cfg(feature = "bench-internals")]
2105            benchmark_phases: Recorder::default(),
2106            conn,
2107            schema,
2108            client_id: client_id
2109                .clone()
2110                .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()),
2111            limits,
2112            outbox: Shared::default(),
2113            failed_commits: Shared::default(),
2114            retain_failed_commits: false,
2115            conflicts: Shared::default(),
2116            rejections: Shared::default(),
2117            schema_floor: None,
2118            lease_state: None,
2119            stopped: false,
2120            upgrading: false,
2121            upgrade_kept_tables: false,
2122            sync_needed: false,
2123            realtime_policy: RealtimePolicy::Optional,
2124            realtime_state: RealtimeState::Disconnected,
2125            realtime_reason_code: None,
2126            realtime_retry_delay_ms: None,
2127            presence: HashMap::new(),
2128            now_ms: None,
2129            encryption: crate::values::EncryptionConfig::default(),
2130            security_preflight: false,
2131            security_epoch: 0,
2132            transport_enabled: true,
2133            transport_epoch: 0,
2134            registration_epoch: 0,
2135            previous_version,
2136            insert_sql: RefCell::new(HashMap::new()),
2137            overlay_dirty: OverlayDirty::clean(),
2138            storage_failure: RefCell::new(None),
2139            #[cfg(test)]
2140            overlay_rebuild_count: Cell::new(0),
2141            #[cfg(test)]
2142            acknowledged_replay_count: Cell::new(0),
2143            #[cfg(test)]
2144            outcome_prune_count: Cell::new(0),
2145            #[cfg(test)]
2146            image_chunk_ms: Cell::new(None),
2147            progress: ProgressObserver::default(),
2148            epoch: uuid::Uuid::new_v4().to_string(),
2149            change_queue: VecDeque::new(),
2150            sync_intent_queue: VecDeque::new(),
2151            scopes_changed: 0,
2152            retry_delay_ms: 250,
2153            round_retry_delay_ms: None,
2154            image_chunk_rows: HashMap::new(),
2155            round_failure: None,
2156            active_round: None,
2157            last_round: None,
2158            last_change: None,
2159            query_failures: Vec::new(),
2160            coalesced: None,
2161            coalesced_failure: None,
2162        };
2163        // The row write path leans on the prepared-statement cache (two
2164        // insert statements per synced table, plus the bookkeeping
2165        // statements); size it so a multi-table schema never thrashes.
2166        client
2167            .conn
2168            .set_prepared_statement_cache_capacity(64.max(client.schema.tables.len() * 4));
2169        // §7.4.2: one write transaction (SQLite `BEGIN IMMEDIATE`, a reserved
2170        // writer) covers the marker read, the identity check, and every
2171        // bookkeeping/schema write, so a concurrent open that upgraded the
2172        // replica cannot interleave between the guard and the writes, and a
2173        // refusal leaves the replica unchanged.
2174        client
2175            .conn
2176            .execute_batch("BEGIN IMMEDIATE")
2177            .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
2178        let startup = (|| -> Result<(), String> {
2179            // The marker guard runs inside the write transaction, so the version
2180            // it validates is the version every write below commits under.
2181            let marker = read_local_schema_version(&client.conn, client.schema.version)?;
2182            // New app indexes may reference columns that only exist after a
2183            // version-bump reset.
2184            client.create_bookkeeping_tables()?;
2185            match client.get_meta(CLIENT_ID_KEY)? {
2186                Some(existing) => match client_id {
2187                    Some(requested) if existing != requested => {
2188                        return Err(format!(
2189                            "client.identity_mismatch: this database belongs to {existing:?}; refusing to rebind it to {requested:?}"
2190                        ))
2191                    }
2192                    _ => client.client_id = existing,
2193                },
2194                None => client.set_meta(CLIENT_ID_KEY, &client.client_id)?,
2195            }
2196            client.restore_persisted_state()?;
2197            client.failed_commits = client.load_failed_commits()?.into();
2198            match marker {
2199                None => {
2200                    client.create_synced_tables()?;
2201                    client.persist_schema_version()?;
2202                    client.save_unit_scope_schema()?;
2203                }
2204                Some(version) if version == client.schema.version => {
2205                    client.create_synced_tables()?;
2206                    // RFC 0005 D1: the same-version open backfills the descriptor
2207                    // for a database first opened by an unaware binary, so the
2208                    // NEXT bump can capture without needing a schema bump first.
2209                    set_local_schema_descriptor(&client.conn, &client.schema)
2210                        .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
2211                    client.save_unit_scope_schema()?;
2212                }
2213                Some(_) => client.run_schema_reset()?,
2214            }
2215            // RFC 0005 D9: the reset sweep cannot see a container whose capture
2216            // committed in its OWN file while the replica's savepoint never
2217            // released — a crash at that seam leaves a container beside a matching
2218            // OLD marker, and the next open at that version runs no reset. Discard
2219            // it before anything can read it. No-op when no container exists.
2220            client.reconcile_previous_version_at_boot()?;
2221            client.clear_satisfied_persisted_schema_floor()?;
2222            client.prune_unknown_units(false)?;
2223            // §7.1/§7.4.4: recover intrinsically codec-invalid persisted values at
2224            // the full startup/reset reconciliation boundary, inside this open
2225            // transaction and after the schema and identity guards, so a commit
2226            // an earlier version accepted is dropped with durable rejection
2227            // evidence before any replay. A table the current schema removed
2228            // keeps the §7.4.4 send-time classification, and an unresolved
2229            // encryption key stays with the send seam because hosts configure
2230            // keys after open.
2231            client.drop_unencodable_outbox(false, None)?;
2232            if marker == Some(client.schema.version)
2233                && (!client.outbox.is_empty()
2234                    || !client.failed_commits.is_empty()
2235                    || client.has_acknowledged_rows()?)
2236            {
2237                // Reconstruct the visible optimistic overlay from the durable
2238                // base plus outbox instead of trusting a process-interrupted
2239                // mirror.
2240                client.overlay_dirty.set(true);
2241                client.rebuild_overlay()?;
2242            }
2243            // Every persisted active unit needs one catch-up pull on
2244            // open: realtime only covers changes after connection, while an
2245            // idempotent setWindow correctly creates no fresh command effect.
2246            // Pending outbox work has the same restart requirement. The core
2247            // owns this intent so native hosts never poll or require an
2248            // application-issued sync().
2249            client.enqueue_startup_sync_if_needed()?;
2250            Ok(())
2251        })();
2252        match startup {
2253            Ok(()) => match client.conn.execute_batch("COMMIT") {
2254                // §4.8: an eviction a crash interrupted finishes before any
2255                // read, with or without a transport; a replica reopened in
2256                // preflight finishes it at activation.
2257                Ok(()) => {
2258                    if !client.security_preflight {
2259                        client.drain_pending_evictions()?;
2260                    }
2261                    Ok(client)
2262                }
2263                Err(error) => {
2264                    let failure = Self::sqlite_failure(&client.storage_failure, error);
2265                    if let Err(rollback) = client.conn.execute_batch("ROLLBACK") {
2266                        // Keep a failed rollback's typed storage classification.
2267                        let _ = Self::sqlite_failure(&client.storage_failure, rollback);
2268                    }
2269                    Err(failure)
2270                }
2271            },
2272            Err(error) => {
2273                if let Err(rollback) = client.conn.execute_batch("ROLLBACK") {
2274                    // Keep a failed rollback's typed storage classification.
2275                    let _ = Self::sqlite_failure(&client.storage_failure, rollback);
2276                }
2277                Err(error)
2278            }
2279        }
2280    }
2281
2282    /// Pin the client clock (epoch ms) — the §5.4 expiry check runs
2283    /// against this instead of system time (conformance virtual clock).
2284    pub fn set_now_ms(&mut self, now_ms: i64) {
2285        self.now_ms = Some(std::sync::Arc::new(now_ms.into()));
2286    }
2287
2288    /// The pinned clock, shared: a host that stores into it moves the client
2289    /// clock while a call runs (a virtual clock that advances per request).
2290    pub fn pinned_clock(&self) -> Option<std::sync::Arc<std::sync::atomic::AtomicI64>> {
2291        self.now_ms.clone()
2292    }
2293
2294    /// §5.11: install the client-side encryption keys (`keyId → key bytes`).
2295    /// The command router parses these from the `create` command's
2296    /// `encryption` config (keys as `{$bytes: hex}`).
2297    pub fn set_encryption(&mut self, encryption: crate::values::EncryptionConfig) {
2298        self.encryption = encryption;
2299    }
2300
2301    #[must_use]
2302    pub fn security_lifecycle(&self) -> &'static str {
2303        if self.security_preflight {
2304            "preflight"
2305        } else {
2306            "active"
2307        }
2308    }
2309
2310    #[must_use]
2311    pub fn security_preflight(&self) -> bool {
2312        self.security_preflight
2313    }
2314
2315    /// Host-owned, non-persisted network gate. Local operations remain available.
2316    pub fn set_transport_enabled(&mut self, transport: &mut dyn Transport, enabled: bool) {
2317        if self.transport_enabled != enabled {
2318            self.transport_enabled = enabled;
2319            self.transport_epoch = self.transport_epoch.wrapping_add(1);
2320            self.sync_intent_queue.clear();
2321            self.sync_intent_queue
2322                .push_back(if enabled && !self.security_preflight {
2323                    SyncIntent::Interactive
2324                } else {
2325                    SyncIntent::None
2326                });
2327        }
2328        // A captured reply keeps its normal apply/revocation semantics.
2329        if !enabled && self.active_round.is_none() {
2330            self.disconnect_realtime(transport);
2331        }
2332    }
2333
2334    pub fn transport_enabled(&self) -> bool {
2335        self.transport_enabled
2336    }
2337
2338    /// Quarantine this replica: enter the fail-closed gate, release all
2339    /// core-owned key material, and record the gate in the database so
2340    /// reopening re-enters preflight until `activate_security` clears it.
2341    ///
2342    /// Use [`SyncClient::seal_security_on_teardown`] for the shutdown barrier,
2343    /// which must NOT leave that durable mark.
2344    pub fn begin_security_preflight(&mut self) -> Result<(), String> {
2345        let entering = !self.security_preflight;
2346        self.seal_security_on_teardown();
2347        // Persist the quarantine so it survives handle teardown and restart:
2348        // reopening this replica re-enters preflight until activation clears it.
2349        self.set_meta(SECURITY_PREFLIGHT_PENDING_KEY, "1")?;
2350        if entering {
2351            self.emit_status_batch()?;
2352        }
2353        Ok(())
2354    }
2355
2356    /// One change batch carrying the status snapshot (§5.11 lifecycle).
2357    fn emit_status_batch(&mut self) -> Result<(), String> {
2358        self.begin_observation("syncular_status")?;
2359        let batch = ChangeAccumulator {
2360            status: true,
2361            ..ChangeAccumulator::default()
2362        };
2363        self.finish_observation("syncular_status", batch)
2364            .inspect_err(|_| self.rollback_observation("syncular_status"))
2365    }
2366
2367    /// Release core-owned key material as a host tears the client down,
2368    /// WITHOUT recording a quarantine.
2369    ///
2370    /// Shutting an activated client down is not a quarantine event. Persisting
2371    /// the gate here marks every cleanly closed replica as pending, and the
2372    /// next plain `create` is then refused permanently: the reopen path
2373    /// restores the flag from the marker and the create guard rejects it. The
2374    /// in-memory flag still closes the gate for anything still holding this
2375    /// instance, which is all a teardown barrier needs.
2376    pub fn seal_security_on_teardown(&mut self) {
2377        self.cancel_sync_round();
2378        self.security_preflight = true;
2379        self.security_epoch = self.security_epoch.wrapping_add(1);
2380        self.encryption = crate::values::EncryptionConfig::default();
2381        self.sync_intent_queue.clear();
2382    }
2383
2384    /// Install the post-authentication keyring and release the host loop.
2385    pub fn activate_security(
2386        &mut self,
2387        encryption: crate::values::EncryptionConfig,
2388    ) -> Result<(), String> {
2389        if !self.security_preflight {
2390            return Err(
2391                "sync.invalid_request: activateSecurity requires security preflight".to_owned(),
2392            );
2393        }
2394        // §4.8: an interrupted eviction finishes before protected reads open.
2395        self.drain_pending_evictions()?;
2396        self.enqueue_startup_sync_if_needed()?;
2397        self.delete_meta(SECURITY_PREFLIGHT_PENDING_KEY)?;
2398        self.encryption = encryption;
2399        self.security_preflight = false;
2400        // §5.11: rows sealed under a key this keyring now holds open at once.
2401        self.reveal_sealed_rows()
2402            .map_err(|error| format!("{}: {}", error.code, error.message))?;
2403        self.emit_status_batch()
2404    }
2405
2406    fn clock_now_ms(&self) -> i64 {
2407        self.now_ms
2408            .as_ref()
2409            .map_or_else(crate::wall_clock_ms, |clock| {
2410                clock.load(std::sync::atomic::Ordering::Relaxed)
2411            })
2412    }
2413
2414    fn create_bookkeeping_tables(&self) -> Result<(), String> {
2415        let exists = |name: &str| {
2416            self.conn
2417                .query_row(
2418                    "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?1)",
2419                    [name],
2420                    |row| row.get::<_, bool>(0),
2421                )
2422                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2423        };
2424        let outbox_existed = exists("_syncular_outbox")?;
2425        // Durable client bookkeeping (outbox + units + meta).
2426        self.conn
2427            .execute_batch(
2428                "CREATE TABLE IF NOT EXISTS _syncular_outbox (
2429                   seq INTEGER PRIMARY KEY AUTOINCREMENT,
2430                   commit_id TEXT NOT NULL UNIQUE, ops_json TEXT NOT NULL,
2431                   created_at_ms INTEGER NOT NULL DEFAULT 0);
2432                 CREATE TABLE IF NOT EXISTS _syncular_row_deliveries (
2433                   tbl TEXT NOT NULL, id TEXT NOT NULL, commit_seq INTEGER NOT NULL, PRIMARY KEY(tbl,id));
2434                 CREATE TABLE IF NOT EXISTS _syncular_acked_rows (
2435                   commit_id TEXT NOT NULL, idx INTEGER NOT NULL, tbl TEXT NOT NULL, id TEXT NOT NULL,
2436                   commit_seq INTEGER NOT NULL, op_json TEXT NOT NULL, intent_json TEXT,
2437                   PRIMARY KEY(commit_id, idx));
2438                 CREATE INDEX IF NOT EXISTS _syncular_acked_rows_row
2439                   ON _syncular_acked_rows(tbl, id);
2440                 CREATE TABLE IF NOT EXISTS _syncular_failed_commits (
2441                   seq INTEGER PRIMARY KEY AUTOINCREMENT, client_commit_id TEXT NOT NULL UNIQUE,
2442                   operations_json TEXT NOT NULL, initial_json TEXT NOT NULL);
2443                 CREATE TABLE IF NOT EXISTS _syncular_commit_outcomes (
2444                   seq INTEGER PRIMARY KEY AUTOINCREMENT,
2445                   client_commit_id TEXT NOT NULL UNIQUE,
2446                   status TEXT NOT NULL CHECK(status IN ('applied', 'cached', 'conflict', 'rejected')),
2447                   recorded_at_ms INTEGER NOT NULL,
2448                   results_json TEXT NOT NULL,
2449                   operations_json TEXT,
2450                   resolution TEXT NOT NULL DEFAULT 'active'
2451                     CHECK(resolution IN ('active', 'resolved_keep_server', 'superseded', 'dismissed')),
2452                   resolved_at_ms INTEGER,
2453                   replacement_client_commit_id TEXT);
2454                 CREATE INDEX IF NOT EXISTS _syncular_commit_outcomes_resolution_seq
2455                   ON _syncular_commit_outcomes(resolution, seq);
2456                 CREATE TABLE IF NOT EXISTS _syncular_units (
2457                   id TEXT PRIMARY KEY, scopes TEXT NOT NULL,
2458                   explicit INTEGER NOT NULL DEFAULT 0,
2459                   status TEXT NOT NULL DEFAULT 'active', reason_code TEXT,
2460                   cursor INTEGER DEFAULT -1, bootstrap_state TEXT,
2461                   seeded TEXT NOT NULL DEFAULT '[]', echo TEXT, applied TEXT, restriction TEXT);
2462                 CREATE TABLE IF NOT EXISTS _syncular_meta (
2463                   key TEXT PRIMARY KEY, value TEXT NOT NULL);
2464                 CREATE TABLE IF NOT EXISTS _syncular_windows (
2465                   base TEXT NOT NULL, unit TEXT NOT NULL, sub_id TEXT NOT NULL,
2466                   PRIMARY KEY (base, unit));
2467                 CREATE INDEX IF NOT EXISTS _syncular_windows_unit ON _syncular_windows(sub_id);
2468                 CREATE TABLE IF NOT EXISTS _syncular_pending_evictions (
2469                   unit_id TEXT NOT NULL, tbl TEXT NOT NULL,
2470                   effective_scopes TEXT NOT NULL, PRIMARY KEY (unit_id, tbl));
2471                 CREATE TABLE IF NOT EXISTS _syncular_blob_commit_refs(
2472                   commit_id TEXT NOT NULL, blob_id TEXT NOT NULL,
2473                   PRIMARY KEY(commit_id, blob_id));
2474                 CREATE INDEX IF NOT EXISTS _syncular_blob_commit_refs_body
2475                   ON _syncular_blob_commit_refs(blob_id);",
2476            )
2477            .map_err(|e| e.to_string())?;
2478        // The §5.6 applied-segment record and the §7.4.3 restriction, added
2479        // after the units table shipped.
2480        for column in ["applied", "restriction"] {
2481            if !self
2482                .conn
2483                .prepare("SELECT 1 FROM pragma_table_info('_syncular_units') WHERE name = ?1")
2484                .and_then(|mut stmt| stmt.exists([column]))
2485                .map_err(|e| e.to_string())?
2486            {
2487                self.conn
2488                    .execute_batch(&format!(
2489                        "ALTER TABLE _syncular_units ADD COLUMN {column} TEXT"
2490                    ))
2491                    .map_err(|e| e.to_string())?;
2492            }
2493        }
2494        // An outbox created before `pendingCommits` carried creation times.
2495        let _ = self.conn.execute_batch(
2496            "ALTER TABLE _syncular_outbox ADD COLUMN created_at_ms INTEGER NOT NULL DEFAULT 0",
2497        );
2498        // Migrate an outcome journal created before failed aggregate
2499        // envelopes were retained. Historical rows intentionally stay NULL.
2500        let _ = self
2501            .conn
2502            .execute_batch("ALTER TABLE _syncular_commit_outcomes ADD COLUMN operations_json TEXT");
2503        if self.get_meta(LOCAL_REVISION_KEY)?.is_none() {
2504            self.set_meta(LOCAL_REVISION_KEY, "0")?;
2505        }
2506        // §7.1: `outboxId` names the outbox sequence. A new outbox table starts
2507        // a new sequence, so it replaces the id in the same transaction.
2508        if !outbox_existed || self.get_meta(OUTBOX_ID_KEY)?.is_none() {
2509            self.set_meta(OUTBOX_ID_KEY, &uuid::Uuid::new_v4().to_string())?;
2510        }
2511        // §9: a replica written under wire version 3 keeps its outbox, journal,
2512        // rows and pending evictions and registers its units fresh. Its
2513        // subscriptions, cursors, echoes, resume tokens and window entries go;
2514        // units left from an earlier version 4 life restart.
2515        if exists("_syncular_window_pending_evict")? {
2516            let invalid: bool = self
2517                .conn
2518                .query_row(
2519                    "SELECT EXISTS(SELECT 1 FROM _syncular_window_pending_evict
2520                       WHERE NOT json_valid(effective_scopes)
2521                          OR json_type(effective_scopes) <> 'object'
2522                          OR EXISTS(SELECT 1 FROM json_each(effective_scopes)
2523                               WHERE type <> 'array'
2524                                  OR EXISTS(SELECT 1 FROM json_each(value) AS v
2525                                       WHERE v.type <> 'text')))",
2526                    [],
2527                    |row| row.get(0),
2528                )
2529                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
2530            if invalid {
2531                return Err(
2532                    "sync.local_corrupt: a version 3 pending eviction cannot be upgraded"
2533                        .to_owned(),
2534                );
2535            }
2536            self.conn
2537                .execute_batch(
2538                    "INSERT OR REPLACE INTO _syncular_pending_evictions
2539                       SELECT sub_id, tbl, effective_scopes FROM _syncular_window_pending_evict;
2540                     DROP TABLE _syncular_window_pending_evict;
2541                     DROP TABLE IF EXISTS _syncular_subscriptions;
2542                     DELETE FROM _syncular_windows;",
2543                )
2544                .and_then(|()| reset_units(&self.conn, &BTreeSet::new(), None))
2545                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
2546        }
2547        // §5.9.7 blob cache. Mutable upload state stays off the body row so
2548        // completing a large upload does not rewrite its BLOB record.
2549        if self.schema_has_blobs() {
2550            if exists("_syncular_blobs")? {
2551                let mut statement = self
2552                    .conn
2553                    .prepare("PRAGMA table_info(_syncular_blobs)")
2554                    .map_err(|e| e.to_string())?;
2555                let mut columns = statement
2556                    .query_map([], |row| row.get::<_, String>(1))
2557                    .map_err(|e| e.to_string())?
2558                    .collect::<Result<Vec<_>, _>>()
2559                    .map_err(|e| e.to_string())?;
2560                columns.sort();
2561                let mut expected = vec![
2562                    "blob_id",
2563                    "byte_length",
2564                    "bytes",
2565                    "created_at_ms",
2566                    "media_type",
2567                ];
2568                expected.sort_unstable();
2569                if columns != expected {
2570                    return Err(
2571                        "sync.schema_mismatch: Local blob schema is incompatible".to_owned()
2572                    );
2573                }
2574            } else {
2575                self.conn
2576                    .execute_batch(
2577                        "CREATE TABLE _syncular_blobs(
2578                           blob_id TEXT PRIMARY KEY,
2579                           bytes BLOB NOT NULL,
2580                           byte_length INTEGER NOT NULL,
2581                           media_type TEXT,
2582                           created_at_ms INTEGER NOT NULL);",
2583                    )
2584                    .map_err(|e| e.to_string())?;
2585            }
2586            self.conn
2587                .execute_batch(
2588                    "CREATE TABLE IF NOT EXISTS _syncular_blob_uploads(
2589                       blob_id TEXT PRIMARY KEY,
2590                       media_type TEXT,
2591                       created_at_ms INTEGER NOT NULL);",
2592                )
2593                .map_err(|e| e.to_string())?;
2594        }
2595        Ok(())
2596    }
2597
2598    /// True iff any synced table declares a `blob_ref` column (§5.9).
2599    fn schema_has_blobs(&self) -> bool {
2600        self.schema
2601            .tables
2602            .iter()
2603            .any(|t| t.columns.iter().any(|c| c.ty == ColumnType::BlobRef))
2604    }
2605
2606    // -- meta (§7.4.1 marker, bookkeeping) ------------------------------------
2607
2608    fn get_meta(&self, key: &str) -> Result<Option<String>, String> {
2609        meta_get(&self.conn, key)
2610            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2611    }
2612
2613    // The caller holds the startup/reset transaction. Either both schema
2614    // records persist or the enclosing operation fails and rolls back.
2615    fn persist_schema_version(&self) -> Result<(), String> {
2616        self.conn
2617            .execute(
2618                "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
2619                rusqlite::params![LOCAL_SCHEMA_VERSION_KEY, self.schema.version.to_string()],
2620            )
2621            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
2622        set_local_schema_descriptor(&self.conn, &self.schema)
2623            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2624    }
2625
2626    pub(crate) fn set_meta(&self, key: &str, value: &str) -> Result<(), String> {
2627        meta_set(&self.conn, key, value)
2628            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2629    }
2630
2631    fn delete_meta(&self, key: &str) -> Result<(), String> {
2632        meta_delete(&self.conn, key)
2633            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2634    }
2635
2636    /// §7.1: the durable outbox in FIFO order, in schema-agnostic local form.
2637    fn load_outbox(&self) -> Result<Vec<Arc<OutboxCommit>>, String> {
2638        let mut stmt = self
2639            .conn
2640            .prepare("SELECT commit_id, ops_json FROM _syncular_outbox ORDER BY seq ASC")
2641            .map_err(|error| error.to_string())?;
2642        let rows = stmt
2643            .query_map([], |row| {
2644                Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
2645            })
2646            .map_err(|error| error.to_string())?;
2647        let mut commits = Vec::new();
2648        for row in rows {
2649            let (client_commit_id, raw) = row.map_err(|error| error.to_string())?;
2650            let entries: Vec<Value> = serde_json::from_str(&raw).map_err(|error| {
2651                format!("invalid persisted outbox {client_commit_id:?}: {error}")
2652            })?;
2653            let mut ops = Vec::with_capacity(entries.len());
2654            for entry in entries {
2655                let op = entry.get("op").and_then(Value::as_str).unwrap_or("delete");
2656                ops.push(OutboxOp {
2657                    upsert: op == "upsert",
2658                    table: entry
2659                        .get("table")
2660                        .and_then(Value::as_str)
2661                        .ok_or_else(|| "persisted outbox operation missing table".to_owned())?
2662                        .to_owned(),
2663                    row_id: entry
2664                        .get("rowId")
2665                        .and_then(Value::as_str)
2666                        .ok_or_else(|| "persisted outbox operation missing rowId".to_owned())?
2667                        .to_owned(),
2668                    base_version: entry.get("baseVersion").and_then(Value::as_i64),
2669                    // A legacy entry's `changedFields` key is ignored:
2670                    // the values map is the presence set (§6.1).
2671                    values: entry.get("values").and_then(Value::as_object).cloned(),
2672                    insert: entry.get("insert") == Some(&Value::Bool(true)),
2673                });
2674            }
2675            commits.push(Arc::new(OutboxCommit {
2676                client_commit_id,
2677                ops,
2678            }));
2679        }
2680        Ok(commits)
2681    }
2682
2683    fn restore_persisted_state(&mut self) -> Result<(), String> {
2684        self.outbox = self.load_outbox()?.into();
2685
2686        self.prune_commit_outcomes()?;
2687        let active = self.commit_outcomes(CommitOutcomeQuery {
2688            active_only: true,
2689            ..CommitOutcomeQuery::default()
2690        })?;
2691        self.conflicts = active
2692            .iter()
2693            .flat_map(|outcome| outcome.results.iter())
2694            .filter_map(|result| match result {
2695                CommitOperationOutcome::Conflict { conflict } => Some(conflict.clone()),
2696                _ => None,
2697            })
2698            .collect::<Vec<_>>()
2699            .into();
2700        self.rejections = active
2701            .iter()
2702            .flat_map(|outcome| outcome.results.iter())
2703            .filter_map(|result| match result {
2704                CommitOperationOutcome::Error { rejection } => Some(rejection.clone()),
2705                _ => None,
2706            })
2707            .collect::<Vec<_>>()
2708            .into();
2709
2710        self.lease_state = self
2711            .get_meta(LEASE_STATE_KEY)?
2712            .map(|raw| serde_json::from_str(&raw))
2713            .transpose()
2714            .map_err(|error| format!("invalid persisted lease state: {error}"))?;
2715        self.schema_floor = self
2716            .get_meta(SCHEMA_FLOOR_KEY)?
2717            .map(|raw| serde_json::from_str(&raw))
2718            .transpose()
2719            .map_err(|error| format!("invalid persisted schema floor: {error}"))?;
2720        self.stopped = self.schema_floor.is_some();
2721        // A replica left in unactivated preflight reopens gated: the quarantine
2722        // decision persists with the data across handle teardown and restart.
2723        if self.get_meta(SECURITY_PREFLIGHT_PENDING_KEY)?.as_deref() == Some("1") {
2724            self.security_preflight = true;
2725        }
2726        Ok(())
2727    }
2728
2729    fn save_unit_scope_schema(&self) -> Result<(), String> {
2730        let declarations: BTreeMap<_, BTreeMap<_, _>> = self
2731            .schema
2732            .tables
2733            .iter()
2734            .map(|table| {
2735                (
2736                    table.name.clone(),
2737                    table
2738                        .scope_variables
2739                        .iter()
2740                        .map(|scope| {
2741                            (
2742                                scope.variable.clone(),
2743                                [scope.prefix.clone(), scope.column.clone()],
2744                            )
2745                        })
2746                        .collect(),
2747                )
2748            })
2749            .collect();
2750        self.set_meta(
2751            "subscriptionScopeSchema",
2752            &serde_json::to_string(&declarations).expect("scope schema serialization"),
2753        )?;
2754        let generations: BTreeMap<_, _> = self
2755            .schema
2756            .tables
2757            .iter()
2758            .filter_map(|table| Some((&table.name, table.generation.as_ref()?)))
2759            .collect();
2760        self.set_meta(
2761            TABLE_GENERATIONS_KEY,
2762            &serde_json::to_string(&generations).expect("generations serialize"),
2763        )
2764    }
2765
2766    /// Drop units that cover no table of the running schema, with their window
2767    /// registry rows and pending evictions; never translate scopes. At a bump
2768    /// (§7.4.3) a unit also needs every variable unchanged in prefix and column
2769    /// on every table it covered before.
2770    fn prune_unknown_units(&mut self, bump: bool) -> Result<(), String> {
2771        let previous: Option<BTreeMap<String, BTreeMap<String, [String; 2]>>> = if bump {
2772            self.get_meta("subscriptionScopeSchema")?
2773                .map(|raw| serde_json::from_str(&raw))
2774                .transpose()
2775                .map_err(|_| "sync.local_corrupt: invalid persisted scope schema".to_owned())?
2776        } else {
2777            None
2778        };
2779        // A table keeps the variables' meaning when each has the same prefix
2780        // and column as before. An old binary stored no declaration evidence;
2781        // matching names cannot prove unchanged meaning, so the app registers
2782        // again.
2783        let unchanged = |name: &str, variables: &[&String]| {
2784            let (Some(old), Some(table)) = (
2785                previous.as_ref().and_then(|previous| previous.get(name)),
2786                self.schema.table(name),
2787            ) else {
2788                return false;
2789            };
2790            variables.iter().all(|variable| {
2791                table
2792                    .scope_variables
2793                    .iter()
2794                    .find(|scope| &scope.variable == *variable)
2795                    .is_some_and(|scope| {
2796                        old.get(*variable) == Some(&[scope.prefix.clone(), scope.column.clone()])
2797                    })
2798            })
2799        };
2800        let compatible = |scopes: &UnitScopes| {
2801            if !self.schema.tables.iter().any(|table| covers(table, scopes)) {
2802                return false;
2803            }
2804            if !bump {
2805                return true;
2806            }
2807            let Some(previous) = &previous else {
2808                return false;
2809            };
2810            previous.iter().all(|(name, old)| {
2811                !scopes.keys().all(|variable| old.contains_key(variable))
2812                    || unchanged(name, &scopes.keys().collect::<Vec<_>>())
2813            })
2814        };
2815        let mut stale: BTreeSet<String> = self
2816            .stored(load_units(&self.conn))?
2817            .into_iter()
2818            .filter(|unit| !compatible(&unit.scopes))
2819            .map(|unit| unit.id)
2820            .collect();
2821        // A shrunk window's deferred eviction outlives its unit row and keeps
2822        // its per-table scopes. A table the schema no longer has cannot be
2823        // evicted, and at a bump a table whose scope declarations changed must
2824        // not be evicted by the outdated rule.
2825        for (id, table, effective) in self.load_pending_evictions()? {
2826            if self.schema.table(&table).is_none()
2827                || (bump
2828                    && !unchanged(
2829                        &table,
2830                        &effective
2831                            .iter()
2832                            .map(|(variable, _)| variable)
2833                            .collect::<Vec<_>>(),
2834                    ))
2835            {
2836                stale.insert(id);
2837            }
2838        }
2839        for id in &stale {
2840            for sql in [
2841                "DELETE FROM _syncular_windows WHERE sub_id = ?1",
2842                "DELETE FROM _syncular_pending_evictions WHERE unit_id = ?1",
2843                "DELETE FROM _syncular_units WHERE id = ?1",
2844            ] {
2845                self.conn
2846                    .execute(sql, rusqlite::params![id])
2847                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
2848            }
2849        }
2850        Ok(())
2851    }
2852
2853    pub fn local_revision(&self) -> Result<u64, String> {
2854        Ok(self
2855            .get_meta(LOCAL_REVISION_KEY)?
2856            .and_then(|value| value.parse().ok())
2857            .unwrap_or(0))
2858    }
2859
2860    /// §7.5: resolve a snapshot read on the mutable owner connection through
2861    /// the same primitive the read-only sidecar uses, so the command surface
2862    /// and the file sidecar return one revision, rows, coverage, catch-up, and
2863    /// delivery shape. The structured [`QueryReadFailure`] is preserved; an
2864    /// `owner` records its failure (and its next success) in diagnostics.
2865    pub fn snapshot_read(
2866        &mut self,
2867        request: &SnapshotReadRequest<'_>,
2868        owner: Option<QueryOwner<'_>>,
2869    ) -> Result<SnapshotRead, QueryReadFailure> {
2870        let result = snapshot_read_connection(&self.conn, request).and_then(|mut read| {
2871            for delivery in &mut read.deliveries {
2872                if let CommitDelivery::Known { outcome, .. } = delivery {
2873                    *outcome = self
2874                        .attach_retained_rows(outcome.clone())
2875                        .map_err(QueryReadFailure::from)?;
2876                }
2877            }
2878            Ok(read)
2879        });
2880        if let Some(owner) = owner {
2881            self.record_query_read(owner, result.as_ref().err());
2882        }
2883        result
2884    }
2885
2886    /// §7.5: one client commit's delivery status with the owner's retained
2887    /// rows: pending, its journal outcome, or an explicit unknown.
2888    pub fn commit_delivery(&mut self, id: &str) -> Result<CommitDelivery, QueryReadFailure> {
2889        let read = self.snapshot_read(
2890            &SnapshotReadRequest {
2891                commit_ids: vec![id.to_owned()],
2892                ..Default::default()
2893            },
2894            None,
2895        )?;
2896        Ok(read
2897            .deliveries
2898            .into_iter()
2899            .next()
2900            .expect("one requested delivery"))
2901    }
2902
2903    #[must_use]
2904    pub fn status_snapshot(&self) -> SyncStatusSnapshot {
2905        SyncStatusSnapshot {
2906            current_schema_version: self.schema.version,
2907            outbox: self.outbox.len(),
2908            upgrading: self.upgrading,
2909            upgrade_kept_tables: self.upgrade_kept_tables,
2910            lease_state: self.lease_state.clone(),
2911            schema_floor: self.schema_floor.clone(),
2912            sync_needed: self.sync_needed,
2913            previous_version_context: self.previous_version_status(),
2914            security_lifecycle: self.security_lifecycle(),
2915        }
2916    }
2917
2918    /// RFC 0005 A2: container presence for a host that must act on it (Diego's
2919    /// rollback precondition), read without creating the file.
2920    fn previous_version_status(&self) -> PreviousVersionStatus {
2921        let Some(replica_path) = self.previous_version_replica_path() else {
2922            return PreviousVersionStatus {
2923                present: false,
2924                created_at_ms: None,
2925            };
2926        };
2927        match crate::previous_version::read_previous_version_record(&replica_path) {
2928            Ok(Some(record)) => PreviousVersionStatus {
2929                present: true,
2930                created_at_ms: Some(record.created_at_ms),
2931            },
2932            _ => PreviousVersionStatus {
2933                present: std::path::Path::new(
2934                    &crate::previous_version::previous_version_container_path(&replica_path),
2935                )
2936                .exists(),
2937                created_at_ms: None,
2938            },
2939        }
2940    }
2941
2942    /// RFC 0005 D7: read captured rows from the previous local schema. `state`
2943    /// is always `previousVersion`; the container never makes
2944    /// `query_snapshot()` coverage complete. Lease stop/expiry, scope
2945    /// revocation, coverage completion and the TTL DISCARD the container and
2946    /// close the read with their own reason.
2947    pub fn previous_version_snapshot(
2948        &self,
2949        spec: &PreviousVersionReadSpec,
2950    ) -> Result<PreviousVersionSnapshot, String> {
2951        let now_ms = self.clock_now_ms();
2952        crate::previous_version::previous_version_snapshot(
2953            &self.conn,
2954            self.previous_version_replica_path().as_deref(),
2955            self.previous_version.as_ref(),
2956            PreviousVersionLifecycle {
2957                lease_inactive: self.previous_version_lease_inactive(now_ms),
2958                scope_revoked: self
2959                    .stored(load_units(&self.conn))?
2960                    .iter()
2961                    .any(|unit| unit.status == UnitStatus::Revoked),
2962                coverage_complete: self.previous_version_coverage_complete()?,
2963            },
2964            now_ms,
2965            self.schema.version,
2966            spec,
2967        )
2968    }
2969
2970    /// RFC 0005 D6: the advisory pre-reset compatibility audit.
2971    pub fn previous_version_audit(
2972        &self,
2973    ) -> Result<Option<crate::previous_version::PreviousVersionAudit>, String> {
2974        crate::previous_version::stored_previous_version_audit(&self.conn)
2975            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2976    }
2977
2978    /// RFC 0005 A2/D9: the executable downgrade step. Drop the container file
2979    /// and both metadata records, and report whether anything was present.
2980    /// Idempotent: absence is a successful no-op.
2981    pub fn previous_version_discard(
2982        &self,
2983    ) -> Result<crate::previous_version::PreviousVersionDiscardOutcome, String> {
2984        self.drop_previous_version()
2985    }
2986
2987    fn drop_previous_version(
2988        &self,
2989    ) -> Result<crate::previous_version::PreviousVersionDiscardOutcome, String> {
2990        let Some(replica_path) = self.previous_version_replica_path() else {
2991            return Ok(crate::previous_version::PreviousVersionDiscardOutcome {
2992                present: false,
2993                discarded: false,
2994            });
2995        };
2996        crate::previous_version::discard_previous_version(&self.conn, &replica_path)
2997    }
2998
2999    /// RFC 0005 D7: `leaseState.errorCode` or an expired lease.
3000    fn previous_version_lease_inactive(&self, now_ms: i64) -> bool {
3001        let Some(lease) = &self.lease_state else {
3002            return false;
3003        };
3004        if lease.error_code.is_some() {
3005            return true;
3006        }
3007        lease.expires_at_ms.is_some_and(|expires| expires <= now_ms)
3008    }
3009
3010    /// RFC 0005 D7: §7.4.5 replacement coverage — no active unit is
3011    /// bootstrapping. An empty active set is not completion (mirrors the TS
3012    /// predicate exactly).
3013    fn previous_version_coverage_complete(&self) -> Result<bool, String> {
3014        let units = self.stored(load_units(&self.conn))?;
3015        let mut active = units
3016            .iter()
3017            .filter(|unit| unit.status == UnitStatus::Active)
3018            .peekable();
3019        Ok(active.peek().is_some() && !active.any(UnitRecord::bootstrapping))
3020    }
3021
3022    /// RFC 0005 D7: lifetime triggers evaluated once per sync round, after the
3023    /// apply. A container that no longer has a reason to exist is discarded so
3024    /// the rows do not outlive the window they were kept for.
3025    fn previous_version_lifetime_check(&self) {
3026        let Some(config) = self.previous_version.filter(|config| config.enabled) else {
3027            return;
3028        };
3029        let now_ms = self.clock_now_ms();
3030        if self.previous_version_lease_inactive(now_ms) {
3031            let _ = self.drop_previous_version();
3032            return;
3033        }
3034        let Some(replica_path) = self.previous_version_replica_path() else {
3035            return;
3036        };
3037        let Ok(Some(record)) = crate::previous_version::read_previous_version_record(&replica_path)
3038        else {
3039            return;
3040        };
3041        let Ok(units) = load_units(&self.conn) else {
3042            return;
3043        };
3044        if units.iter().any(|unit| unit.status == UnitStatus::Revoked) {
3045            let _ = self.drop_previous_version();
3046            return;
3047        }
3048        if now_ms - record.created_at_ms > config.max_age_ms {
3049            let _ = self.drop_previous_version();
3050            return;
3051        }
3052        if self.previous_version_coverage_complete() == Ok(true) {
3053            let _ = self.drop_previous_version();
3054        }
3055    }
3056
3057    pub fn diagnostics_snapshot(&self) -> Result<ClientDiagnosticsSnapshot, String> {
3058        let registered = self.stored(load_units(&self.conn))?;
3059        // Both meta values in one statement: the FFI host recomputes
3060        // diagnostics after every command.
3061        let (local_revision, replica_cursor): (Option<String>, Option<String>) = self.stored(
3062            self.conn
3063                .prepare_cached(
3064                    "SELECT (SELECT value FROM _syncular_meta WHERE key = ?1),
3065                            (SELECT value FROM _syncular_meta WHERE key = ?2)",
3066                )
3067                .and_then(|mut statement| {
3068                    statement.query_row([LOCAL_REVISION_KEY, REPLICA_CURSOR_KEY], |row| {
3069                        Ok((row.get(0)?, row.get(1)?))
3070                    })
3071                }),
3072        )?;
3073        let mut units = DiagnosticUnits::default();
3074        for unit in &registered {
3075            *match unit.status {
3076                UnitStatus::Revoked => &mut units.revoked,
3077                UnitStatus::Failed => &mut units.failed,
3078                UnitStatus::Active if !unit.bootstrapping() => &mut units.complete,
3079                UnitStatus::Active
3080                    if unit.reason_code.as_deref() == Some("sync.cursor_expired") =>
3081                {
3082                    &mut units.reset
3083                }
3084                UnitStatus::Active => &mut units.bootstrapping,
3085            } += 1;
3086        }
3087        units.reason_codes = registered
3088            .iter()
3089            .filter_map(|unit| unit.reason_code.as_deref().map(Self::diagnostic_code))
3090            .collect::<BTreeSet<_>>()
3091            .into_iter()
3092            .collect();
3093        let captured_at_ms = self.clock_now_ms();
3094        let lease = if let Some(error_code) = self
3095            .lease_state
3096            .as_ref()
3097            .and_then(|state| state.error_code.clone())
3098        {
3099            ClientDiagnosticsLease {
3100                state: "stopped".to_owned(),
3101                expires_at_ms: self
3102                    .lease_state
3103                    .as_ref()
3104                    .and_then(|state| state.expires_at_ms),
3105                error_code: Some(Self::diagnostic_code(&error_code)),
3106            }
3107        } else if let Some(expires_at_ms) = self
3108            .lease_state
3109            .as_ref()
3110            .and_then(|state| state.expires_at_ms)
3111        {
3112            ClientDiagnosticsLease {
3113                state: if expires_at_ms <= captured_at_ms {
3114                    "expired".to_owned()
3115                } else {
3116                    "active".to_owned()
3117                },
3118                expires_at_ms: Some(expires_at_ms),
3119                error_code: None,
3120            }
3121        } else {
3122            ClientDiagnosticsLease {
3123                state: "none".to_owned(),
3124                expires_at_ms: None,
3125                error_code: None,
3126            }
3127        };
3128        let connectivity = match self.last_round.as_ref() {
3129            _ if !self.transport_enabled && self.active_round.is_none() => "offline",
3130            Some(round) if round.status == "succeeded" => "online",
3131            Some(round)
3132                if round.status == "failed"
3133                    && matches!(
3134                        round.cause_kind,
3135                        Some(
3136                            crate::TransportCauseKind::Network | crate::TransportCauseKind::Connect
3137                        )
3138                    ) =>
3139            {
3140                "offline"
3141            }
3142            Some(round) if round.status == "failed" && round.http_status.is_some() => "online",
3143            _ => "unknown",
3144        };
3145        Ok(ClientDiagnosticsSnapshot {
3146            version: CLIENT_DIAGNOSTICS_VERSION,
3147            captured_at_ms,
3148            host: ClientDiagnosticsHost {
3149                kind: "direct".to_owned(),
3150                role: "single".to_owned(),
3151                connectivity: connectivity.to_owned(),
3152                realtime: self.realtime_state,
3153                realtime_policy: self.realtime_policy,
3154                realtime_reason_code: self.realtime_reason_code.clone(),
3155                realtime_retry_delay_ms: self.realtime_retry_delay_ms,
3156            },
3157            security_lifecycle: self.security_lifecycle().to_owned(),
3158            schema: ClientDiagnosticsSchema {
3159                current_version: self.schema.version,
3160                upgrading: self.upgrading,
3161                required_version: self
3162                    .schema_floor
3163                    .as_ref()
3164                    .and_then(|floor| floor.required_schema_version),
3165                latest_version: self
3166                    .schema_floor
3167                    .as_ref()
3168                    .and_then(|floor| floor.latest_schema_version),
3169            },
3170            replica: ClientDiagnosticsReplica {
3171                local_revision: local_revision
3172                    .and_then(|value| value.parse::<u64>().ok())
3173                    .unwrap_or(0)
3174                    .to_string(),
3175                sync_needed: self.sync_needed,
3176                pending_outbox: self.outbox.len(),
3177            },
3178            lease,
3179            active_units: registered
3180                .iter()
3181                .filter(|unit| unit.status == UnitStatus::Active)
3182                .count(),
3183            units,
3184            replica_cursor: replica_cursor.map_or(Ok(-1), |raw| {
3185                raw.parse()
3186                    .map_err(|_| "sync.local_corrupt: persisted unit state is invalid".to_owned())
3187            })?,
3188            last_round: self.last_round.clone(),
3189            last_change: self.last_change.clone(),
3190            storage: self.diagnostics_storage(),
3191            query_failures: self.query_failures.clone(),
3192        })
3193    }
3194
3195    fn diagnostics_storage(&self) -> ClientDiagnosticsStorage {
3196        let read = || -> Result<ClientDiagnosticsStorage, rusqlite::Error> {
3197            let page_count: i64 = self
3198                .conn
3199                .query_row("PRAGMA page_count", [], |row| row.get(0))?;
3200            let page_size: i64 = self
3201                .conn
3202                .query_row("PRAGMA page_size", [], |row| row.get(0))?;
3203            // One cached aggregate, since the FFI host recomputes diagnostics
3204            // after every command; every invocation still reads current storage.
3205            let (outbox_bytes, outcome_entries, outcome_bytes): (i64, i64, i64) = self
3206                .conn
3207                .prepare_cached(
3208                    "SELECT (SELECT COALESCE(SUM(LENGTH(ops_json)), 0) FROM _syncular_outbox),
3209                            COUNT(*), COALESCE(SUM(LENGTH(results_json) + COALESCE(LENGTH(operations_json), 0)), 0)
3210                       FROM _syncular_commit_outcomes",
3211                )?
3212                .query_row([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))?;
3213            let blob_bytes = if self.schema_has_blobs() {
3214                self.conn
3215                    .prepare_cached("SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs")?
3216                    .query_row([], |row| row.get(0))?
3217            } else {
3218                0
3219            };
3220            let pressure = self
3221                .limits
3222                .blob_cache_max_bytes
3223                .is_some_and(|limit| blob_bytes > limit);
3224            Ok(ClientDiagnosticsStorage {
3225                status: if pressure { "pressure" } else { "healthy" }.to_owned(),
3226                database_bytes_approx: Some(page_count.saturating_mul(page_size).max(0)),
3227                pending_outbox_bytes_approx: Some(outbox_bytes.max(0)),
3228                retained_outcome_bytes_approx: Some(outcome_bytes.max(0)),
3229                retained_outcome_entries: Some(outcome_entries.max(0)),
3230                blob_cache_bytes_approx: Some(blob_bytes.max(0)),
3231                pressure_reason_code: pressure.then(|| "client.blob_cache_over_limit".to_owned()),
3232            })
3233        };
3234        read().unwrap_or_else(|_| ClientDiagnosticsStorage {
3235            status: "unreadable".to_owned(),
3236            database_bytes_approx: None,
3237            pending_outbox_bytes_approx: None,
3238            retained_outcome_bytes_approx: None,
3239            retained_outcome_entries: None,
3240            blob_cache_bytes_approx: None,
3241            pressure_reason_code: None,
3242        })
3243    }
3244
3245    pub fn drain_change_batches(&mut self) -> Vec<ClientChangeBatch> {
3246        self.change_queue.drain(..).collect()
3247    }
3248
3249    pub fn drain_sync_intents(&mut self) -> Vec<SyncIntent> {
3250        let intents = self.sync_intent_queue.drain(..).collect();
3251        if self.transport_enabled {
3252            intents
3253        } else {
3254            vec![SyncIntent::None]
3255        }
3256    }
3257
3258    /// §8.7 host announcement: the number of `scopes` events received since
3259    /// the last drain. The event changes no sync state; the host surfaces each
3260    /// one to the app, whose discovery registers the newly readable units.
3261    pub fn drain_scopes_changed(&mut self) -> usize {
3262        std::mem::take(&mut self.scopes_changed)
3263    }
3264
3265    fn schedule_background_retry(&mut self) -> u64 {
3266        let delay_ms = self.retry_delay_ms;
3267        self.sync_intent_queue
3268            .push_back(SyncIntent::Background { delay_ms });
3269        self.retry_delay_ms = (self.retry_delay_ms * 2).min(30_000);
3270        self.round_retry_delay_ms = Some(delay_ms);
3271        delay_ms
3272    }
3273
3274    fn reset_background_retry(&mut self) {
3275        self.retry_delay_ms = 250;
3276    }
3277
3278    fn retryable_transport_code(code: &str) -> bool {
3279        code == "transport.failed"
3280            || code == "transport.timeout"
3281            || code == "transport.unavailable"
3282            || code == "sync.transport_failed"
3283    }
3284
3285    /// §1.8: the transport reported a refused or unreachable endpoint. The
3286    /// native transport says `connect` (DNS failure, connection refused,
3287    /// reset, or aborted); a browser fetch transport says `network`. Repeating
3288    /// the request on a backoff repairs neither, so no retry is scheduled.
3289    fn unreachable_transport(details: &Option<serde_json::Value>) -> bool {
3290        matches!(
3291            details
3292                .as_ref()
3293                .and_then(|details| details.get("causeKind"))
3294                .and_then(serde_json::Value::as_str),
3295            Some("connect" | "network")
3296        )
3297    }
3298
3299    /// §1.5: a segment fetch that fails inside an otherwise complete round is
3300    /// invisible to the server (the round response arrived whole), so its code
3301    /// rides the following requests like a socket-round failure. A code the
3302    /// frame grammar rejects is not reportable and the previous code stays.
3303    fn segment_fetch_failed(&mut self, error: TransportError) -> SectionError {
3304        if ssp2::model::valid_round_failure_code(&error.code) {
3305            self.round_failure = Some(error.code.clone());
3306        }
3307        SectionError::Transport(error)
3308    }
3309
3310    /// A failed round worth a background retry: a transport failure, or a
3311    /// §10.2 code the catalog marks retryable (clients may hardcode the
3312    /// catalog metadata, §10.1). Matches the TypeScript core, which retries
3313    /// on the server's `retryable` flag.
3314    fn retryable_failure_code(code: &str) -> bool {
3315        Self::retryable_transport_code(code)
3316            || matches!(
3317                code,
3318                "sync.auth_required"
3319                    | "sync.auth_lease_required"
3320                    | "sync.auth_lease_revoked"
3321                    | "sync.internal_error"
3322                    | "sync.idempotency_cache_miss"
3323                    | "sync.schema_not_ready"
3324                    | "sync.segment_build_timeout"
3325                    | "sync.segment_expired"
3326                    | "sync.rate_limited"
3327                    | "sync.websocket_connection_limit"
3328                    | "sync.idle_timeout"
3329            )
3330    }
3331
3332    fn diagnostic_code(code: &str) -> String {
3333        let valid = !code.is_empty()
3334            && code.len() <= 96
3335            && code.contains('.')
3336            && code
3337                .bytes()
3338                .next()
3339                .is_some_and(|byte| byte.is_ascii_lowercase())
3340            && code.bytes().all(|byte| {
3341                byte.is_ascii_lowercase()
3342                    || byte.is_ascii_digit()
3343                    || matches!(byte, b'.' | b'_' | b'-')
3344            });
3345        if valid {
3346            code.to_owned()
3347        } else {
3348            "client.unknown_failure".to_owned()
3349        }
3350    }
3351
3352    fn set_sync_needed(&mut self, value: bool, interactive: bool) {
3353        if self.sync_needed != value {
3354            if self.begin_observation("syncular_status").is_ok() {
3355                self.sync_needed = value;
3356                let batch = ChangeAccumulator {
3357                    status: true,
3358                    ..ChangeAccumulator::default()
3359                };
3360                if self.finish_observation("syncular_status", batch).is_err() {
3361                    self.rollback_observation("syncular_status");
3362                }
3363            } else {
3364                self.sync_needed = value;
3365            }
3366        }
3367        if value && interactive {
3368            self.sync_intent_queue.push_back(SyncIntent::Interactive);
3369        }
3370    }
3371
3372    /// A unit-store result: a row that does not parse is `sync.local_corrupt`;
3373    /// every other SQLite error keeps its storage classification.
3374    fn stored<T>(&self, result: rusqlite::Result<T>) -> Result<T, String> {
3375        result.map_err(|error| match error {
3376            rusqlite::Error::FromSqlConversionFailure(..) => {
3377                "sync.local_corrupt: persisted unit state is invalid".to_owned()
3378            }
3379            error => Self::sqlite_failure(&self.storage_failure, error),
3380        })
3381    }
3382
3383    fn sqlite_failure(
3384        storage_failure: &RefCell<Option<QueryReadFailure>>,
3385        error: rusqlite::Error,
3386    ) -> String {
3387        let message = error.to_string();
3388        let failure = QueryReadFailure::from(error);
3389        let code = failure.code;
3390        if code.is_some() && storage_failure.borrow().is_none() {
3391            *storage_failure.borrow_mut() = Some(failure);
3392        }
3393        match code {
3394            Some(code) => format!("{code}: {message}"),
3395            None => message,
3396        }
3397    }
3398
3399    fn begin_observation(&self, name: &str) -> Result<(), String> {
3400        if let Some(failure) = &self.coalesced_failure {
3401            return Err(failure.clone());
3402        }
3403        // SQLite rolls a whole transaction back on some I/O and capacity
3404        // errors; later frames must not commit on their own after that.
3405        if self.coalesced.is_some() && self.conn.is_autocommit() {
3406            return Err("the coalesced apply transaction was rolled back".to_owned());
3407        }
3408        self.conn
3409            .execute_batch(&format!("SAVEPOINT {name}"))
3410            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
3411    }
3412
3413    /// SPEC §1.4 rule 4: open a coalesced apply transaction unless a caller
3414    /// already owns one.
3415    fn begin_coalesced_apply(&mut self) -> Result<(), String> {
3416        if self.coalesced.is_some() || !self.conn.is_autocommit() {
3417            return Ok(());
3418        }
3419        self.conn
3420            .execute_batch("BEGIN")
3421            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3422        self.coalesced = Some(CoalescedApply {
3423            rows: 0,
3424            rollback: self.capture(),
3425        });
3426        Ok(())
3427    }
3428
3429    fn capture(&self) -> Rollback {
3430        Rollback {
3431            outbox: self.outbox.clone(),
3432            failed_commits: self.failed_commits.clone(),
3433            conflicts: self.conflicts.clone(),
3434            rejections: self.rejections.clone(),
3435            schema_floor: self.schema_floor.clone(),
3436            lease_state: self.lease_state.clone(),
3437            stopped: self.stopped,
3438            upgrading: self.upgrading,
3439            upgrade_kept_tables: self.upgrade_kept_tables,
3440            sync_needed: self.sync_needed,
3441            active_round: self.active_round,
3442            overlay_dirty: self.overlay_dirty.snapshot(),
3443            change_queue: self.change_queue.len(),
3444            sync_intent_queue: self.sync_intent_queue.len(),
3445            last_change: self.last_change.clone(),
3446        }
3447    }
3448
3449    fn restore(&mut self, rollback: Rollback) {
3450        self.outbox = rollback.outbox;
3451        self.failed_commits = rollback.failed_commits;
3452        self.conflicts = rollback.conflicts;
3453        self.rejections = rollback.rejections;
3454        self.schema_floor = rollback.schema_floor;
3455        self.lease_state = rollback.lease_state;
3456        self.stopped = rollback.stopped;
3457        self.upgrading = rollback.upgrading;
3458        self.upgrade_kept_tables = rollback.upgrade_kept_tables;
3459        self.sync_needed = rollback.sync_needed;
3460        self.active_round = rollback.active_round;
3461        self.overlay_dirty.restore(rollback.overlay_dirty);
3462        self.change_queue.truncate(rollback.change_queue);
3463        self.sync_intent_queue.truncate(rollback.sync_intent_queue);
3464        self.last_change = rollback.last_change;
3465    }
3466
3467    /// Account the rows of the next frame or block. A unit that would take the
3468    /// coalesced transaction past the bound commits the earlier units first.
3469    fn coalesce_unit(&mut self, rows: usize) -> Result<(), String> {
3470        let Some(open) = &self.coalesced else {
3471            return Ok(());
3472        };
3473        if open.rows > 0 && open.rows + rows > COALESCED_APPLY_ROWS {
3474            self.commit_coalesced_apply()?;
3475            self.begin_coalesced_apply()?;
3476        }
3477        if let Some(open) = &mut self.coalesced {
3478            open.rows += rows;
3479        }
3480        Ok(())
3481    }
3482
3483    /// COMMIT the coalesced transaction. A failed COMMIT rolls every coalesced
3484    /// frame back, restores the in-memory state of its `BEGIN`, drops the
3485    /// change batches and sync intents queued since, and fails the rest of the
3486    /// response.
3487    fn commit_coalesced_apply(&mut self) -> Result<(), String> {
3488        let Some(open) = self.coalesced.take() else {
3489            return Ok(());
3490        };
3491        let committed = if self.conn.is_autocommit() {
3492            Err("the coalesced apply transaction was rolled back".to_owned())
3493        } else {
3494            self.conn
3495                .execute_batch("COMMIT")
3496                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
3497        };
3498        let Err(error) = committed else {
3499            return Ok(());
3500        };
3501        if !self.conn.is_autocommit() {
3502            if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
3503                let _ = Self::sqlite_failure(&self.storage_failure, rollback);
3504            }
3505        }
3506        self.restore(open.rollback);
3507        self.coalesced_failure = Some(error.clone());
3508        Err(error)
3509    }
3510
3511    /// End the response's coalescing: COMMIT what is open and clear a failure.
3512    fn finish_coalesced_apply(&mut self) -> Result<(), String> {
3513        let committed = self.commit_coalesced_apply();
3514        match self.coalesced_failure.take() {
3515            Some(failure) => Err(failure),
3516            None => committed,
3517        }
3518    }
3519
3520    fn rollback_observation(&self, name: &str) {
3521        if let Err(error) = self
3522            .conn
3523            .execute_batch(&format!("ROLLBACK TO {name}; RELEASE {name}"))
3524        {
3525            if let Some(failure) = self.storage_failure.borrow_mut().as_mut() {
3526                if failure.rollback_failure.is_none() {
3527                    failure.rollback_failure = Some(Box::new(QueryReadFailure::from(error)));
3528                }
3529            }
3530        }
3531    }
3532
3533    fn finish_observation(
3534        &mut self,
3535        name: &str,
3536        mut batch: ChangeAccumulator,
3537    ) -> Result<(), String> {
3538        #[cfg(feature = "bench-internals")]
3539        let _phase = self.benchmark_phases.start(Phase::ObservationCommit);
3540        if self.failed_commits.iter().any(|(commit, _)| {
3541            commit
3542                .ops
3543                .iter()
3544                .any(|operation| batch.tables.contains_key(&operation.table))
3545        }) {
3546            batch.outcomes = true;
3547        }
3548        if !batch.touched() {
3549            self.conn
3550                .execute_batch(&format!("RELEASE {name}"))
3551                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3552            return Ok(());
3553        }
3554        let revision = self
3555            .local_revision()?
3556            .checked_add(1)
3557            .ok_or_else(|| "local revision exhausted u64".to_owned())?;
3558        self.conn
3559            .execute(
3560                "INSERT OR REPLACE INTO _syncular_meta(key, value) VALUES (?1, ?2)",
3561                rusqlite::params![LOCAL_REVISION_KEY, revision.to_string()],
3562            )
3563            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3564        let status = batch.status.then(|| self.status_snapshot());
3565        let event = ClientChangeBatch {
3566            revision: revision.to_string(),
3567            tables: batch
3568                .tables
3569                .into_iter()
3570                .map(|(table, scope_keys)| TableChange {
3571                    table,
3572                    scope_keys: scope_keys.map(|keys| keys.into_iter().collect()),
3573                })
3574                .collect(),
3575            windows: batch
3576                .windows
3577                .into_iter()
3578                .map(|(base_key, units)| WindowChange {
3579                    base_key,
3580                    units: units.into_iter().collect(),
3581                })
3582                .collect(),
3583            status,
3584            conflicts_changed: batch.conflicts,
3585            rejections_changed: batch.rejections,
3586            outcomes_changed: batch.outcomes,
3587            applied_commits: Vec::new(),
3588        };
3589        self.conn
3590            .execute_batch(&format!("RELEASE {name}"))
3591            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3592        let mut diagnostic_tables = event
3593            .tables
3594            .iter()
3595            .map(|entry| entry.table.clone())
3596            .collect::<BTreeSet<_>>()
3597            .into_iter()
3598            .collect::<Vec<_>>();
3599        // §7.6: the selected variables of the changed window bases.
3600        let mut diagnostic_windows = event
3601            .windows
3602            .iter()
3603            .map(|entry| {
3604                entry
3605                    .base_key
3606                    .split('\0')
3607                    .next()
3608                    .unwrap_or_default()
3609                    .to_owned()
3610            })
3611            .collect::<BTreeSet<_>>()
3612            .into_iter()
3613            .collect::<Vec<_>>();
3614        let domains_truncated = diagnostic_tables.len() > MAX_DIAGNOSTIC_DOMAINS
3615            || diagnostic_windows.len() > MAX_DIAGNOSTIC_DOMAINS;
3616        diagnostic_tables.truncate(MAX_DIAGNOSTIC_DOMAINS);
3617        diagnostic_windows.truncate(MAX_DIAGNOSTIC_DOMAINS);
3618        self.last_change = Some(DiagnosticLastChange {
3619            revision: event.revision.clone(),
3620            recorded_at_ms: self.clock_now_ms(),
3621            tables: diagnostic_tables,
3622            windows: diagnostic_windows,
3623            domains_truncated,
3624            status_changed: event.status.is_some(),
3625            conflicts_changed: event.conflicts_changed,
3626            rejections_changed: event.rejections_changed,
3627            outcomes_changed: event.outcomes_changed,
3628        });
3629        self.change_queue.push_back(event);
3630        Ok(())
3631    }
3632
3633    fn record_scope_map(
3634        &self,
3635        batch: &mut ChangeAccumulator,
3636        table_name: &str,
3637        scopes: &[(String, Vec<String>)],
3638    ) {
3639        let Some(table) = self.schema.table(table_name) else {
3640            return;
3641        };
3642        // A bulk scoped delete whose map leaves a declared variable unnamed
3643        // cannot list the deleted rows' keys for it: the change is table-wide.
3644        if table.scope_variables.iter().any(|scope| {
3645            !scopes
3646                .iter()
3647                .any(|(variable, _)| *variable == scope.variable)
3648        }) {
3649            batch.table(table_name);
3650            return;
3651        }
3652        for (variable, values) in scopes {
3653            let Some(scope) = table
3654                .scope_variables
3655                .iter()
3656                .find(|scope| &scope.variable == variable)
3657            else {
3658                continue;
3659            };
3660            for value in values {
3661                batch.scope(table_name, format!("{}:{value}", scope.prefix));
3662            }
3663        }
3664    }
3665
3666    /// Record a row's current scope keys from the base or visible table.
3667    fn record_row_scopes(
3668        &self,
3669        batch: &mut ChangeAccumulator,
3670        table_name: &str,
3671        row_id: &str,
3672        base: bool,
3673    ) -> bool {
3674        let Some(table) = self.schema.table(table_name) else {
3675            return false;
3676        };
3677        if table.scope_variables.is_empty() {
3678            return false;
3679        }
3680        let columns = table
3681            .scope_variables
3682            .iter()
3683            .map(|scope| quote_ident(&scope.column))
3684            .collect::<Vec<_>>()
3685            .join(", ");
3686        let full_table = if base {
3687            base_table(table_name)
3688        } else {
3689            visible_table(table_name)
3690        };
3691        let sql = format!(
3692            "SELECT {columns} FROM {full_table} WHERE {} LIMIT 1",
3693            row_id_predicate(table)
3694        );
3695        let Ok(mut stmt) = self.conn.prepare_cached(&sql) else {
3696            return false;
3697        };
3698        let values = stmt.query_row(rusqlite::params![row_id], |row| {
3699            let mut values = Vec::with_capacity(table.scope_variables.len());
3700            for index in 0..table.scope_variables.len() {
3701                values.push(row.get::<_, Option<String>>(index)?);
3702            }
3703            Ok(values)
3704        });
3705        let Ok(values) = values else {
3706            return false;
3707        };
3708        let mut recorded = false;
3709        for (scope, value) in table.scope_variables.iter().zip(values) {
3710            if let Some(value) = value {
3711                batch.scope(table_name, format!("{}:{value}", scope.prefix));
3712                recorded = true;
3713            }
3714        }
3715        recorded
3716    }
3717
3718    fn record_commit_changes(
3719        &self,
3720        batch: &mut ChangeAccumulator,
3721        tables: &[String],
3722        changes: &[ssp2::model::Change],
3723    ) {
3724        #[cfg(feature = "bench-internals")]
3725        let _phase = self.benchmark_phases.start(Phase::ObservationPrepare);
3726        for change in changes {
3727            let Some(table_name) = tables.get(change.table_index as usize) else {
3728                continue;
3729            };
3730            let mut precise = self.record_row_scopes(batch, table_name, &change.row_id, true);
3731            if let Some(table) = self.schema.table(table_name) {
3732                for (variable, value) in &change.scopes {
3733                    if let Some(scope) = table
3734                        .scope_variables
3735                        .iter()
3736                        .find(|scope| &scope.variable == variable)
3737                    {
3738                        batch.scope(table_name, format!("{}:{value}", scope.prefix));
3739                        precise = true;
3740                    }
3741                }
3742            }
3743            if !precise {
3744                batch.table(table_name);
3745            }
3746        }
3747    }
3748
3749    /// Whether a base or visible row matches the scope predicate; `false`
3750    /// without a local scope-column mapping. A SQLite failure is an error.
3751    fn scoped_rows_exist(
3752        &self,
3753        table_name: &str,
3754        effective: &[(String, Vec<String>)],
3755        owners: &[Vec<(String, Vec<String>)>],
3756    ) -> Result<bool, String> {
3757        let Some(table) = self.schema.table(table_name) else {
3758            return Ok(false);
3759        };
3760        let Ok(Some((predicate, params))) = local_scope_predicate(table, effective, owners) else {
3761            return Ok(false);
3762        };
3763        for source in [base_table(table_name), visible_table(table_name)] {
3764            if self
3765                .conn
3766                .query_row(
3767                    &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {predicate})"),
3768                    rusqlite::params_from_iter(&params),
3769                    |row| row.get::<_, bool>(0),
3770                )
3771                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
3772            {
3773                return Ok(true);
3774            }
3775        }
3776        Ok(false)
3777    }
3778
3779    /// §5.6 first-page clear of `unit_id`'s pair: delete its rows except
3780    /// those another owner covers (§3.3), failing closed without a local
3781    /// scope-column mapping. Owners are read only when a row matches, inside
3782    /// the caller's transaction. Returns whether rows matched.
3783    fn clear_scope(
3784        &mut self,
3785        table: &TableSchema,
3786        effective: &[(String, Vec<String>)],
3787        unit_id: &str,
3788    ) -> Result<bool, SectionError> {
3789        let storage = |message| SectionError::Abort("storage.failed".to_owned(), message);
3790        let matched = self
3791            .scoped_rows_exist(&table.name, effective, &[])
3792            .map_err(storage)?;
3793        let owners = if matched {
3794            unit_owners(
3795                &self.stored(load_units(&self.conn)).map_err(storage)?,
3796                table,
3797                unit_id,
3798            )
3799        } else {
3800            Vec::new()
3801        };
3802        let changed = matched
3803            && (owners.is_empty()
3804                || self
3805                    .scoped_rows_exist(&table.name, effective, &owners)
3806                    .map_err(storage)?);
3807        if !self
3808            .purge_scope_rows(&table.name, effective, &owners)
3809            .map_err(storage)?
3810        {
3811            return Err(SectionError::FailClosed);
3812        }
3813        Ok(changed)
3814    }
3815
3816    /// §7.4.5: true while a schema-bump reset + first re-bootstrap runs.
3817    pub fn upgrading(&self) -> bool {
3818        self.upgrading
3819    }
3820
3821    /// §7.4.2 "app ships new code": swap to a NEW generated schema while
3822    /// keeping this client's local database (identity, outbox, tables). The
3823    /// §7.4.1 marker check then fires the wipe/re-bootstrap flow when the
3824    /// version increases. Recreation runs the same marker guard as opening
3825    /// a durable replica, holds the guard and every write it authorizes under
3826    /// one write transaction, and preserves the current client (compiled
3827    /// schema, outbox, tables) on refusal.
3828    pub fn recreate_with_schema(&mut self, schema_json: &Value) -> Result<(), String> {
3829        let new_schema = parse_schema_json(schema_json)?;
3830        validate_authority_reads(&new_schema, &self.limits.authority_reads)?;
3831        self.conn
3832            .execute_batch("BEGIN IMMEDIATE")
3833            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3834        let prior_schema = std::mem::replace(&mut self.schema, new_schema);
3835        // §7.4.3/§7.4.4: a reset inside this transaction mutates in-memory
3836        // state (dropped outbox commits, rejections, upgrade and floor flags,
3837        // overlay dirtiness) and appends change/intent batches. A COMMIT that
3838        // fails after the reset succeeded rolls the database back, so every
3839        // one of those fields is captured here and restored with it.
3840        let rollback = self.capture();
3841        let guarded = (|| -> Result<(), String> {
3842            // §7.4.2: the marker is re-read under the write lock the caller
3843            // holds, so a replica another process upgraded meanwhile refuses
3844            // this recreation instead of having its marker and tables
3845            // downgraded.
3846            let marker = read_local_schema_version(&self.conn, self.schema.version)?;
3847            self.create_bookkeeping_tables()?;
3848            match marker {
3849                Some(version) if version < self.schema.version => self.run_schema_reset()?,
3850                None => {
3851                    self.create_synced_tables()?;
3852                    self.persist_schema_version()?;
3853                }
3854                _ => {}
3855            }
3856            // §7.4.2: recreation IS the boot for the in-memory core, so reload the
3857            // durable outbox the same-version path is about to replay. Only the
3858            // outbox is reloaded: the recovery below reads it, and the caller's
3859            // rollback snapshot already covers this field.
3860            self.outbox = self.load_outbox()?.into();
3861            // §7.1/§7.4.4: recover intrinsically codec-invalid persisted values at
3862            // this guarded boundary once the schema and marker agree. A bump went
3863            // through `run_schema_reset`, which performs the same recovery after
3864            // it creates the new physical tables and before its final replay.
3865            self.drop_unencodable_outbox(false, None)?;
3866            if marker == Some(self.schema.version)
3867                && (!self.outbox.is_empty()
3868                    || !self.failed_commits.is_empty()
3869                    || self.has_acknowledged_rows()?)
3870            {
3871                self.overlay_dirty.set(true);
3872                self.rebuild_overlay()?;
3873            }
3874            self.prune_unknown_units(false)?;
3875            self.save_unit_scope_schema()?;
3876            // The conformance recreate is the in-memory equivalent of reopening
3877            // a durable client. Apply the same startup catch-up contract even
3878            // when the schema itself did not change.
3879            self.enqueue_startup_sync_if_needed()?;
3880            Ok(())
3881        })();
3882        let outcome = match guarded {
3883            Ok(()) => self
3884                .conn
3885                .execute_batch("COMMIT")
3886                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error)),
3887            Err(error) => Err(error),
3888        };
3889        if let Err(error) = outcome {
3890            if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
3891                // Keep a failed rollback's typed storage classification.
3892                let _ = Self::sqlite_failure(&self.storage_failure, rollback);
3893            }
3894            self.schema = prior_schema;
3895            self.restore(rollback);
3896            // Prepared insert statements were derived from the reverted schema.
3897            self.insert_sql.borrow_mut().clear();
3898            return Err(error);
3899        }
3900        if !self.security_preflight {
3901            self.drain_pending_evictions()?;
3902        }
3903        Ok(())
3904    }
3905
3906    fn enqueue_startup_sync_if_needed(&mut self) -> Result<(), String> {
3907        let startup_work = !self.stopped
3908            && (!self.outbox.is_empty()
3909                || self.has_acknowledged_rows()?
3910                || !self.load_pending_evictions()?.is_empty()
3911                || self
3912                    .stored(load_units(&self.conn))?
3913                    .iter()
3914                    .any(|unit| unit.status == UnitStatus::Active));
3915        if startup_work {
3916            self.sync_needed = true;
3917            self.sync_intent_queue.push_back(SyncIntent::Interactive);
3918        }
3919        Ok(())
3920    }
3921
3922    /// A native client persists schema-floor stops across process restarts.
3923    /// Once the running app already satisfies that floor, the persisted stop
3924    /// is only stale evidence from an older server round (for example, the
3925    /// server was restarted after catching up to an app that was ahead).
3926    /// Clear it and let the normal startup pull re-negotiate. If the server is
3927    /// still incompatible it will return the floor again in that first round.
3928    fn clear_satisfied_persisted_schema_floor(&mut self) -> Result<(), String> {
3929        let satisfied = self
3930            .schema_floor
3931            .as_ref()
3932            .and_then(|floor| floor.required_schema_version)
3933            .is_some_and(|required| self.schema.version >= required);
3934        if !satisfied {
3935            return Ok(());
3936        }
3937        self.delete_meta(SCHEMA_FLOOR_KEY)?;
3938        self.schema_floor = None;
3939        self.stopped = false;
3940        Ok(())
3941    }
3942
3943    /// RFC 0005 D1: discard an orphan or stale container at boot. The reset
3944    /// path already swept it; this covers a same-version or fresh-install open,
3945    /// which runs no reset and therefore no sweep — plus the D7 TTL, which only
3946    /// an aware binary can apply.
3947    fn reconcile_previous_version_at_boot(&self) -> Result<(), String> {
3948        let Some(replica_path) = self.previous_version_replica_path() else {
3949            return Ok(());
3950        };
3951        let max_age_ms = self
3952            .previous_version
3953            .filter(|config| config.enabled)
3954            .map(|config| config.max_age_ms);
3955        reconcile_previous_version_at_boot(
3956            &self.conn,
3957            &replica_path,
3958            self.schema.version,
3959            max_age_ms,
3960            self.clock_now_ms(),
3961        )
3962    }
3963
3964    /// RFC 0005 D3: the sibling container path, or `None` when the replica has
3965    /// no file — an in-memory replica returns an empty path, and a `file:` URI
3966    /// has no plain sibling path — so the feature resolves as off there.
3967    fn previous_version_replica_path(&self) -> Option<String> {
3968        let path = self.conn.path()?;
3969        if path.is_empty() || path.starts_with("file:") {
3970            return None;
3971        }
3972        Some(path.to_owned())
3973    }
3974
3975    /// §7.4.3 reset: whole-database local reset EXCEPT the outbox, clientId,
3976    /// and leaseState. Drops/recreates every synced table from the new
3977    /// schema, resets unit sync state (keeping registrations), clears
3978    /// the schema-floor stop state, rewrites the marker, drops outbox commits
3979    /// that cannot re-encode (§7.4.4), and replays the survivors on top.
3980    fn run_schema_reset(&mut self) -> Result<(), String> {
3981        self.begin_observation("syncular_schema_reset")?;
3982        let rollback = self.capture();
3983        let mut batch = ChangeAccumulator::default();
3984        let result = self
3985            .run_schema_reset_observed(&mut batch, true)
3986            .and_then(|()| self.finish_observation("syncular_schema_reset", batch));
3987        if let Err(error) = result {
3988            self.rollback_observation("syncular_schema_reset");
3989            self.restore(rollback);
3990            return Err(error);
3991        }
3992        // After the commit: a rolled-back reset keeps the in-flight round.
3993        self.cancel_sync_round();
3994        Ok(())
3995    }
3996
3997    fn run_log_epoch_reset(&mut self, log_epoch: &str) -> Result<Vec<String>, String> {
3998        if self.get_meta(LOG_EPOCH_KEY)?.is_none() {
3999            self.begin_observation("syncular_epoch_acquisition")?;
4000            let rollback = self.capture();
4001            if let Err(error) = self.conn.execute(
4002                "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
4003                rusqlite::params![LOG_EPOCH_KEY, log_epoch],
4004            ) {
4005                self.rollback_observation("syncular_epoch_acquisition");
4006                return Err(Self::sqlite_failure(&self.storage_failure, error));
4007            }
4008            self.sync_needed = true;
4009            let batch = ChangeAccumulator {
4010                status: true,
4011                ..ChangeAccumulator::default()
4012            };
4013            if let Err(error) = self.finish_observation("syncular_epoch_acquisition", batch) {
4014                self.rollback_observation("syncular_epoch_acquisition");
4015                self.restore(rollback);
4016                return Err(error);
4017            }
4018            self.sync_intent_queue.push_back(SyncIntent::Interactive);
4019            return Ok(Vec::new());
4020        }
4021        let resets = self
4022            .stored(load_units(&self.conn))?
4023            .into_iter()
4024            .map(|unit| unit.id)
4025            .collect();
4026        self.begin_observation("syncular_log_epoch_reset")?;
4027        let rollback = self.capture();
4028        let mut batch = ChangeAccumulator::default();
4029        let result = self
4030            .run_schema_reset_observed(&mut batch, false)
4031            .and_then(|()| {
4032                self.conn
4033                    .execute(
4034                        "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
4035                        rusqlite::params![LOG_EPOCH_KEY, log_epoch],
4036                    )
4037                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4038                self.sync_needed = true;
4039                batch.status = true;
4040                self.finish_observation("syncular_log_epoch_reset", batch)
4041            });
4042        if let Err(error) = result {
4043            self.rollback_observation("syncular_log_epoch_reset");
4044            self.restore(rollback);
4045            return Err(error);
4046        }
4047        self.cancel_sync_round();
4048        self.sync_intent_queue.push_back(SyncIntent::Interactive);
4049        Ok(resets)
4050    }
4051
4052    fn run_schema_reset_observed(
4053        &mut self,
4054        batch: &mut ChangeAccumulator,
4055        capture: bool,
4056    ) -> Result<(), String> {
4057        // §7.4.2: the reset is the only path that wipes local tables for a
4058        // version bump, and it rewrites the marker last. Re-read the marker
4059        // under this reset's transaction: a replica another process upgraded
4060        // meanwhile refuses here instead of being reset to this client's
4061        // older schema.
4062        read_local_schema_version(&self.conn, self.schema.version)?;
4063        // §7.4.3: a schema bump keeps every table whose generation equals the
4064        // persisted one; the log-epoch reset keeps nothing. The previous
4065        // declaration evidence decides which units cover only kept tables.
4066        let mut kept = BTreeSet::new();
4067        let mut previous_declarations: Option<BTreeMap<String, BTreeMap<String, Value>>> = None;
4068        if capture {
4069            let persisted: BTreeMap<String, String> = self
4070                .get_meta(TABLE_GENERATIONS_KEY)?
4071                .map(|raw| serde_json::from_str(&raw))
4072                .transpose()
4073                .map_err(|_| "sync.local_corrupt: invalid persisted table generations".to_owned())?
4074                .unwrap_or_default();
4075            kept = self
4076                .schema
4077                .tables
4078                .iter()
4079                .filter(|table| {
4080                    table.generation.is_some()
4081                        && table.generation.as_ref() == persisted.get(&table.name)
4082                })
4083                .map(|table| table.name.clone())
4084                .collect();
4085            previous_declarations = self
4086                .get_meta("subscriptionScopeSchema")?
4087                .map(|raw| serde_json::from_str(&raw))
4088                .transpose()
4089                .map_err(|_| "sync.local_corrupt: invalid persisted scope schema".to_owned())?;
4090        }
4091        // Kept tables keep both halves, their FTS projections with the FTS5
4092        // shadow tables and id mapping, and their row-level bookkeeping.
4093        let mut kept_objects = BTreeSet::new();
4094        for table in self
4095            .schema
4096            .tables
4097            .iter()
4098            .filter(|table| kept.contains(&table.name))
4099        {
4100            kept_objects.insert(table.name.clone());
4101            kept_objects.insert(format!("_syncular_base_{}", table.name));
4102            for index in &table.fts_indexes {
4103                kept_objects.insert(index.name.clone());
4104                for shadow in ["data", "idx", "content", "docsize", "config"] {
4105                    kept_objects.insert(format!("{}_{shadow}", index.name));
4106                }
4107            }
4108        }
4109        let kept_json = serde_json::to_string(&kept).expect("table names serialize");
4110        for sql in [
4111            "DELETE FROM _syncular_acked_rows WHERE tbl NOT IN (SELECT value FROM json_each(?1))",
4112            "DELETE FROM _syncular_row_deliveries WHERE tbl NOT IN (SELECT value FROM json_each(?1))",
4113        ] {
4114            self.conn
4115                .execute(sql, [&kept_json])
4116                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4117        }
4118        self.upgrading = true;
4119        self.upgrade_kept_tables = !kept.is_empty();
4120        batch.status = true;
4121        // RFC 0005 D6: classify the pending outbox against the NEW compiled
4122        // schema BEFORE the wipe, and record the advisory audit. This pass
4123        // drops nothing, so the audit sees the original pending intent; the
4124        // §7.4.4 send-time drop and the intrinsic value recovery later in this
4125        // reset are the paths that remove a commit. The classification is
4126        // shared with the send-time drop so the two cannot disagree about what
4127        // is incompatible.
4128        if self.previous_version.is_some_and(|config| config.enabled) {
4129            let previous_version = self
4130                .get_meta(LOCAL_SCHEMA_VERSION_KEY)?
4131                .and_then(|value| value.parse::<i32>().ok())
4132                .unwrap_or(self.schema.version);
4133            let pending = self
4134                .outbox
4135                .iter()
4136                .map(|commit| PendingCommitAudit {
4137                    commit_id: &commit.client_commit_id,
4138                    operations: commit_audit_operations(commit),
4139                })
4140                .collect::<Vec<_>>();
4141            crate::previous_version::write_previous_version_audit(
4142                &self.conn,
4143                &crate::previous_version::build_previous_version_audit(
4144                    &self.schema,
4145                    &pending,
4146                    previous_version,
4147                    self.schema.version,
4148                    self.clock_now_ms(),
4149                ),
4150            )?;
4151        }
4152        // RFC 0005 D5: the container lives in its own FILE, so the reset can
4153        // never reach it. Sweep first (unconditional — also on the log-epoch
4154        // reset, which captures nothing), then capture BEFORE the wipe drops
4155        // any row. A refused capture records its reason and leaves no file.
4156        if let Some(replica_path) = self.previous_version_replica_path() {
4157            sweep_previous_version_container(&self.conn, &replica_path)?;
4158            if capture {
4159                if let Some(config) = self.previous_version.filter(|config| config.enabled) {
4160                    let previous_version = self
4161                        .get_meta(LOCAL_SCHEMA_VERSION_KEY)?
4162                        .and_then(|value| value.parse::<i32>().ok())
4163                        .unwrap_or(self.schema.version);
4164                    capture_previous_version_from_replica(
4165                        &self.conn,
4166                        &replica_path,
4167                        previous_version,
4168                        self.schema.version,
4169                        &config,
4170                        self.clock_now_ms(),
4171                    )?;
4172                }
4173            }
4174        }
4175        for table in self
4176            .schema
4177            .tables
4178            .iter()
4179            .filter(|table| !kept.contains(&table.name))
4180        {
4181            batch.table(&table.name);
4182        }
4183        for (base_key, value) in self.load_window_registry()? {
4184            batch.window(&base_key, &value);
4185        }
4186        if capture {
4187            self.prune_unknown_units(true)?;
4188        }
4189        // The per-table insert SQL is derived from the OLD column lists.
4190        self.insert_sql.borrow_mut().clear();
4191        self.overlay_dirty.set(true);
4192        // Drop every synced table (base + visible) that currently exists —
4193        // discovered from sqlite_master so a bump that adds/removes tables is
4194        // handled. Bookkeeping tables (`_syncular_outbox/_units/_meta`
4195        // and the blob cache) are preserved; base tables are `_syncular_base_*`
4196        // so they are matched explicitly, not by the bookkeeping filter.
4197        // Drop virtual tables first so SQLite removes every FTS shadow table
4198        // atomically instead of the generic discovery tearing it apart.
4199        let virtual_tables: Vec<String> = {
4200            let mut stmt = self
4201                .conn
4202                .prepare(
4203                    "SELECT name FROM sqlite_master WHERE type = 'table' AND sql LIKE 'CREATE VIRTUAL TABLE%'",
4204                )
4205                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4206            let rows = stmt
4207                .query_map([], |row| row.get::<_, String>(0))
4208                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4209            rows.filter_map(Result::ok)
4210                .filter(|name| is_synced_table_name(name) && !kept_objects.contains(name))
4211                .collect()
4212        };
4213        for name in virtual_tables {
4214            self.conn
4215                .execute_batch(&format!(
4216                    "DROP TABLE IF EXISTS {}",
4217                    quote_ident(&format!("_syncular_fts_{name}"))
4218                ))
4219                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4220            self.conn
4221                .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
4222                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4223        }
4224        let existing: Vec<String> = {
4225            let mut stmt = self
4226                .conn
4227                .prepare("SELECT name FROM sqlite_master WHERE type = 'table'")
4228                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4229            let rows = stmt
4230                .query_map([], |row| row.get::<_, String>(0))
4231                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4232            rows.filter_map(Result::ok)
4233                .filter(|name| is_synced_table_name(name) && !kept_objects.contains(name))
4234                .collect()
4235        };
4236        for name in &existing {
4237            if let Some(table) = name.strip_prefix("_syncular_base_") {
4238                batch.table(table);
4239            } else if !name.starts_with("_syncular_") {
4240                batch.table(name);
4241            }
4242        }
4243        for name in existing {
4244            self.conn
4245                .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
4246                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4247        }
4248        // Recreate the synced tables from the NEW schema.
4249        self.create_synced_tables()?;
4250        // Reset the sync state of every unit that covers a table the reset
4251        // did not keep, under the previous declarations or the new schema,
4252        // keeping the registration.
4253        let keep_units = match &previous_declarations {
4254            Some(previous) if !kept.is_empty() => self
4255                .stored(load_units(&self.conn))?
4256                .into_iter()
4257                .filter(|unit| {
4258                    previous.iter().all(|(name, old)| {
4259                        kept.contains(name) || !unit.scopes.keys().all(|v| old.contains_key(v))
4260                    }) && self
4261                        .schema
4262                        .tables
4263                        .iter()
4264                        .all(|table| kept.contains(&table.name) || !covers(table, &unit.scopes))
4265                })
4266                .map(|unit| unit.id)
4267                .collect(),
4268            _ => BTreeSet::new(),
4269        };
4270        // §7.4.3 restriction: an active unit that is not kept and whose rows
4271        // of a kept table are complete through a cursor C (its restriction's,
4272        // its post-bootstrap P, or the replica cursor it is live under)
4273        // bootstraps only its other covered tables and joins at C.
4274        let restrictions = if capture {
4275            let replica = self.stored(replica_cursor(&self.conn))?;
4276            let mut restrictions = BTreeMap::new();
4277            for unit in self.stored(load_units(&self.conn))? {
4278                if keep_units.contains(&unit.id) || unit.status != UnitStatus::Active {
4279                    continue;
4280                }
4281                let cursor = match (&unit.restriction, unit.cursor) {
4282                    (Some(restriction), _) => restriction.cursor,
4283                    (None, Some(cursor)) if cursor >= 0 && unit.bootstrap_state.is_none() => cursor,
4284                    (None, None) if replica >= 0 => replica,
4285                    _ => continue,
4286                };
4287                let covered: Vec<&String> = self
4288                    .schema
4289                    .tables
4290                    .iter()
4291                    .filter(|table| covers(table, &unit.scopes))
4292                    .map(|table| &table.name)
4293                    .collect();
4294                let mut tables: Vec<String> = covered
4295                    .iter()
4296                    .filter(|name| {
4297                        !kept.contains(**name)
4298                            || unit
4299                                .restriction
4300                                .as_ref()
4301                                .is_some_and(|restriction| restriction.tables.contains(name))
4302                    })
4303                    .map(|name| (*name).clone())
4304                    .collect();
4305                if tables.is_empty() || tables.len() == covered.len() {
4306                    continue;
4307                }
4308                tables.sort_by(|a, b| a.encode_utf16().cmp(b.encode_utf16()));
4309                restrictions.insert(unit.id, UnitRestriction { cursor, tables });
4310            }
4311            Some(restrictions)
4312        } else {
4313            None
4314        };
4315        reset_units(&self.conn, &keep_units, restrictions.as_ref())
4316            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4317        // The stop state is over: this client now ships a servable schema.
4318        self.stopped = false;
4319        self.schema_floor = None;
4320        self.delete_meta(SCHEMA_FLOOR_KEY)?;
4321        // Rewrite the marker LAST so a crash mid-reset re-runs the reset.
4322        self.persist_schema_version()?;
4323        self.save_unit_scope_schema()?;
4324        // §7.1/§7.4.4: the previous-version capture and audit above saw the
4325        // original pending intent; the synced tables now carry the new schema, so
4326        // recover intrinsically codec-invalid persisted values here, before the
4327        // final replay. A pending upsert that names a column the new schema
4328        // removed stays pending for the send-time `drop_incompatible_outbox`
4329        // prepass. The reset wrapper snapshots and restores the outbox,
4330        // rejections, and change queues, so a failed commit rolls this recovery
4331        // back with the rest of the reset.
4332        // §7.4.4: the enclosing reset observation owns the publication, so the
4333        // recovery contributes its rejection, outcome, and table domains to it.
4334        self.drop_unencodable_outbox(false, Some(batch))?;
4335        // Re-apply the surviving outbox optimistically over the empty tables.
4336        self.rebuild_overlay()?;
4337        Ok(())
4338    }
4339
4340    /// §7.4.4 send-time prepass: a persisted upsert whose values reference a
4341    /// column the current schema lacks (or a removed table) cannot be encoded
4342    /// (`commit_audit_operations` inspects upserts, so a value-free delete stays
4343    /// encodable and is validated by the server), nor can an insert-intent
4344    /// upsert that lacks a non-nullable column (§7.1, the audit's
4345    /// `missing-required-column`). Drop the commit, undo
4346    /// its optimistic projection, and raise a client-local
4347    /// `sync.outbox_incompatible` rejection.
4348    fn drop_incompatible_outbox(&mut self) -> Result<bool, String> {
4349        let incompatible = self
4350            .outbox
4351            .iter()
4352            .filter(|commit| {
4353                first_incompatibility(&self.schema, &commit_audit_operations(commit)).is_some()
4354            })
4355            .cloned()
4356            .collect::<Vec<_>>();
4357        if incompatible.is_empty() {
4358            return Ok(false);
4359        }
4360        self.drop_outbox_commits(
4361            &incompatible,
4362            OUTBOX_INCOMPATIBLE_CODE,
4363            "the persisted commit cannot encode under the current schema",
4364        )?;
4365        Ok(true)
4366    }
4367
4368    /// §7.4.4 incompatible-commit path, also taken by a refused blob upload
4369    /// (§5.9.7) with the server's code: reject every operation of `commits`,
4370    /// remove them from the outbox, and undo their optimistic rows.
4371    fn drop_outbox_commits(
4372        &mut self,
4373        incompatible: &[Arc<OutboxCommit>],
4374        code: &str,
4375        message: &str,
4376    ) -> Result<(), String> {
4377        // §7.4.4: one observation covers the durable outcome writes, the outbox
4378        // removal, the in-memory drop, and the overlay rebuild, so a storage
4379        // fault leaves the replica and the client unchanged and publishes no
4380        // change; a successful drop publishes exactly one batch carrying the
4381        // rejections, outcomes, status, and the tables whose projection changed.
4382        let rollback = self.capture();
4383        let owns_transaction = self.conn.is_autocommit();
4384        self.begin_observation("syncular_outbox_drop")?;
4385        let mut batch = ChangeAccumulator::default();
4386        let dropped = (|| -> Result<(), String> {
4387            let mut rejections = Vec::new();
4388            for commit in incompatible {
4389                let results = commit
4390                    .ops
4391                    .iter()
4392                    .enumerate()
4393                    .map(|(op_index, operation)| {
4394                        let rejection = RejectionRecord {
4395                            client_commit_id: commit.client_commit_id.clone(),
4396                            op_index: op_index as i32,
4397                            code: code.to_owned(),
4398                            message: message.to_owned(),
4399                            retryable: false,
4400                            details: None,
4401                            operation: Some(CommitOperation::from(operation)),
4402                        };
4403                        rejections.push(rejection.clone());
4404                        CommitOperationOutcome::Error { rejection }
4405                    })
4406                    .collect::<Vec<_>>();
4407                self.persist_commit_outcome(
4408                    &commit.client_commit_id,
4409                    CommitOutcomeStatus::Rejected,
4410                    &results,
4411                    Some(&commit.ops),
4412                )?;
4413                self.delete_outbox_persisted(&commit.client_commit_id)?;
4414                // A removed table has no local projection to invalidate; every
4415                // surviving table the dropped commit touched does.
4416                for operation in &commit.ops {
4417                    if self.schema.table(&operation.table).is_some() {
4418                        batch.table(&operation.table);
4419                    }
4420                }
4421            }
4422            self.prune_commit_outcomes()?;
4423            let incompatible_ids = incompatible
4424                .iter()
4425                .map(|commit| commit.client_commit_id.as_str())
4426                .collect::<BTreeSet<_>>();
4427            self.outbox
4428                .retain(|commit| !incompatible_ids.contains(commit.client_commit_id.as_str()));
4429            self.rejections.extend(rejections);
4430            // The dropped commits' purely-optimistic rows are undone: the
4431            // visible projection is re-derived from the surviving base + outbox.
4432            self.overlay_dirty.set(true);
4433            self.rebuild_overlay()?;
4434            batch.status = true;
4435            batch.rejections = true;
4436            batch.outcomes = true;
4437            Ok(())
4438        })();
4439        let result = dropped.and_then(|()| self.finish_observation("syncular_outbox_drop", batch));
4440        if let Err(error) = result {
4441            // An outer RELEASE is the COMMIT. Roll back that transaction
4442            // directly: retrying RELEASE after ROLLBACK TO still needs the
4443            // reader's lock. A nested observation only rolls back its savepoint.
4444            if owns_transaction {
4445                if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
4446                    let mut failure = self.storage_failure.borrow_mut();
4447                    let failure =
4448                        failure.get_or_insert_with(|| QueryReadFailure::from(error.clone()));
4449                    failure.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
4450                }
4451            } else {
4452                self.rollback_observation("syncular_outbox_drop");
4453            }
4454            self.restore(rollback);
4455            return Err(error);
4456        }
4457        Ok(())
4458    }
4459
4460    /// §7.4.3: (re)create the base + visible table pair for every synced
4461    /// table in the CURRENT schema (idempotent — `IF NOT EXISTS`).
4462    fn create_synced_tables(&self) -> Result<(), String> {
4463        for table in &self.schema.tables {
4464            // The base half + the visible half form the synced-table pair. An
4465            // index name is global in SQLite, so the base half's indexes are
4466            // name-prefixed (`_syncular_base_<index>`) to stay distinct.
4467            for (full, index_prefix) in [
4468                (base_table(&table.name), "_syncular_base_"),
4469                (visible_table(&table.name), ""),
4470            ] {
4471                let mut cols: Vec<String> =
4472                    table.columns.iter().map(|c| quote_ident(&c.name)).collect();
4473                cols.push("\"_syncular_version\" INTEGER NOT NULL".to_owned());
4474                cols.extend(
4475                    table
4476                        .sealed_columns()
4477                        .iter()
4478                        .map(|name| format!("{} TEXT", quote_ident(name))),
4479                );
4480                let sql = format!(
4481                    "CREATE TABLE IF NOT EXISTS {full} ({} , PRIMARY KEY ({}))",
4482                    cols.join(", "),
4483                    quote_ident(&table.primary_key)
4484                );
4485                self.conn
4486                    .execute(&sql, [])
4487                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4488                // §5.11: a replica created before sealed apply gains the
4489                // reserved columns in place, on both halves in the same order,
4490                // so data, indexes and the FTS projection stay.
4491                for name in table.sealed_columns() {
4492                    let present: bool = self
4493                        .conn
4494                        .query_row(
4495                            "SELECT EXISTS(SELECT 1 FROM pragma_table_info(?1) WHERE name = ?2)",
4496                            rusqlite::params![format!("{index_prefix}{}", table.name), name],
4497                            |row| row.get(0),
4498                        )
4499                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4500                    if !present {
4501                        self.conn
4502                            .execute(
4503                                &format!(
4504                                    "ALTER TABLE {full} ADD COLUMN {} TEXT",
4505                                    quote_ident(name)
4506                                ),
4507                                [],
4508                            )
4509                            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4510                    }
4511                }
4512                // The reveal at every round start reads the sealed rows of the
4513                // base half through this partial index.
4514                if table.has_encrypted_columns() && !index_prefix.is_empty() {
4515                    self.conn
4516                        .execute(
4517                            &format!(
4518                                "CREATE INDEX IF NOT EXISTS {} ON {full} (\"_sync_sealed_key\") WHERE \"_sync_sealed_key\" IS NOT NULL",
4519                                quote_ident(&format!("_syncular_sealed_{}", table.name))
4520                            ),
4521                            [],
4522                        )
4523                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4524                }
4525                // Local secondary indexes (CREATE INDEX subset). Created on
4526                // both halves so mirror reads hit an index on either. Runs on
4527                // both the initial create and the §7.4.3 reset recreate path.
4528                for index in &table.indexes {
4529                    let unique = if index.unique { "UNIQUE " } else { "" };
4530                    let index_name = quote_ident(&format!("{index_prefix}{}", index.name));
4531                    let cols_sql = index
4532                        .columns
4533                        .iter()
4534                        .map(|c| quote_ident(c))
4535                        .collect::<Vec<_>>()
4536                        .join(", ");
4537                    let index_sql = format!(
4538                        "CREATE {unique}INDEX IF NOT EXISTS {index_name} ON {full} ({cols_sql})"
4539                    );
4540                    self.conn
4541                        .execute(&index_sql, [])
4542                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4543                }
4544            }
4545        }
4546        // FTS exists only on the visible half. It is a local search
4547        // projection, never a synced/base table or a wire-schema member.
4548        for table in &self.schema.tables {
4549            for index in &table.fts_indexes {
4550                self.create_fts_projection(table, index)?;
4551            }
4552        }
4553        Ok(())
4554    }
4555
4556    fn fts_projection_exists(&self, index: &FtsIndexSchema) -> Result<bool, String> {
4557        let count: i64 = self
4558            .conn
4559            .query_row(
4560                "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?1",
4561                rusqlite::params![index.name],
4562                |row| row.get(0),
4563            )
4564            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4565        Ok(count > 0)
4566    }
4567
4568    fn drop_fts_triggers(&self, index: &FtsIndexSchema) -> Result<(), String> {
4569        for suffix in ["ai", "ad", "au"] {
4570            self.conn
4571                .execute(
4572                    &format!(
4573                        "DROP TRIGGER IF EXISTS {}",
4574                        quote_ident(&format!("{}_{suffix}", index.name))
4575                    ),
4576                    [],
4577                )
4578                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4579        }
4580        Ok(())
4581    }
4582
4583    fn create_fts_triggers(
4584        &self,
4585        table: &TableSchema,
4586        index: &FtsIndexSchema,
4587    ) -> Result<(), String> {
4588        let fts = quote_ident(&index.name);
4589        let source = visible_table(&table.name);
4590        let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
4591        let pk = quote_ident(&table.primary_key);
4592        let projection_columns = std::iter::once(source_id.clone())
4593            .chain(index.columns.iter().map(|column| quote_ident(column)))
4594            .collect::<Vec<_>>()
4595            .join(", ");
4596        let new_values = std::iter::once(format!("CAST(new.{pk} AS TEXT)"))
4597            .chain(
4598                index
4599                    .columns
4600                    .iter()
4601                    .map(|column| format!("new.{}", quote_ident(column))),
4602            )
4603            .collect::<Vec<_>>()
4604            .join(", ");
4605        let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
4606        let delete_new = format!("DELETE FROM {fts} WHERE rowid = (SELECT id FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)); DELETE FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)");
4607        let delete_old = format!("DELETE FROM {fts} WHERE rowid = (SELECT id FROM {mapping} WHERE source_id = CAST(old.{pk} AS TEXT)); DELETE FROM {mapping} WHERE source_id = CAST(old.{pk} AS TEXT)");
4608        let insert_new = format!("INSERT INTO {mapping}(source_id) VALUES (CAST(new.{pk} AS TEXT)); INSERT INTO {fts} (rowid, {projection_columns}) VALUES ((SELECT id FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)), {new_values})");
4609        let bi = quote_ident(&format!("{}_bi", index.name));
4610        let ai = quote_ident(&format!("{}_ai", index.name));
4611        let ad = quote_ident(&format!("{}_ad", index.name));
4612        let au = quote_ident(&format!("{}_au", index.name));
4613        let replacement_exists = format!("EXISTS (SELECT 1 FROM {source} WHERE {pk} = new.{pk})");
4614        let sql = format!(
4615            "DROP TRIGGER IF EXISTS {bi};
4616             DROP TRIGGER IF EXISTS {ai};
4617             DROP TRIGGER IF EXISTS {ad};
4618             DROP TRIGGER IF EXISTS {au};
4619             CREATE TRIGGER {bi} BEFORE INSERT ON {source} WHEN {replacement_exists} BEGIN {delete_new}; END;
4620             CREATE TRIGGER {ai} AFTER INSERT ON {source} BEGIN {insert_new}; END;
4621             CREATE TRIGGER {ad} AFTER DELETE ON {source} BEGIN {delete_old}; END;
4622             CREATE TRIGGER {au} AFTER UPDATE ON {source} BEGIN {delete_old}; {delete_new}; {insert_new}; END;",
4623        );
4624        self.conn
4625            .execute_batch(&sql)
4626            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
4627    }
4628
4629    fn rebuild_fts_projection(
4630        &self,
4631        table: &TableSchema,
4632        index: &FtsIndexSchema,
4633    ) -> Result<(), String> {
4634        let fts = quote_ident(&index.name);
4635        let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
4636        let pk = quote_ident(&table.primary_key);
4637        let indexed_columns = index
4638            .columns
4639            .iter()
4640            .map(|column| quote_ident(column))
4641            .collect::<Vec<_>>();
4642        let projection_columns = std::iter::once(source_id.clone())
4643            .chain(indexed_columns.iter().cloned())
4644            .collect::<Vec<_>>()
4645            .join(", ");
4646        let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
4647        let sql = format!(
4648            "DELETE FROM {fts}; DELETE FROM {mapping}; INSERT INTO {fts} ({projection_columns}) SELECT CAST({pk} AS TEXT), {columns} FROM {source}; INSERT INTO {mapping}(id, source_id) SELECT rowid, {source_id} FROM {fts};",
4649            columns = indexed_columns.join(", "),
4650            source = visible_table(&table.name),
4651        );
4652        self.conn
4653            .execute_batch(&sql)
4654            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
4655    }
4656
4657    fn create_fts_projection(
4658        &self,
4659        table: &TableSchema,
4660        index: &FtsIndexSchema,
4661    ) -> Result<(), String> {
4662        self.begin_observation("syncular_fts_schema")?;
4663        let result = (|| {
4664            let existed = self.fts_projection_exists(index)?;
4665            let tokenizer = index.tokenize.replace('\'', "''");
4666            let columns = index
4667                .columns
4668                .iter()
4669                .map(|column| quote_ident(column))
4670                .collect::<Vec<_>>()
4671                .join(", ");
4672            let sql = format!(
4673            "CREATE VIRTUAL TABLE IF NOT EXISTS {fts} USING fts5({source_id} UNINDEXED, {columns}, tokenize='{tokenizer}')",
4674            fts = quote_ident(&index.name),
4675            source_id = quote_ident(FTS_SOURCE_ID_COLUMN),
4676        );
4677            self.conn.execute(&sql, []).map_err(|error| {
4678                format!(
4679                    "cannot create local FTS5 projection {:?}: {error}",
4680                    index.name
4681                )
4682            })?;
4683            let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
4684            let mapped: bool = self
4685                .conn
4686                .query_row(
4687                    "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?1)",
4688                    [format!("_syncular_fts_{}", index.name)],
4689                    |row| row.get(0),
4690                )
4691                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4692            self.conn.execute_batch(&format!("CREATE TABLE IF NOT EXISTS {mapping}(id INTEGER PRIMARY KEY, source_id TEXT NOT NULL UNIQUE)")).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4693            if !mapped && existed {
4694                self.conn
4695                    .execute_batch(&format!(
4696                        "INSERT INTO {mapping}(id, source_id) SELECT rowid, {} FROM {}",
4697                        quote_ident(FTS_SOURCE_ID_COLUMN),
4698                        quote_ident(&index.name)
4699                    ))
4700                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4701            }
4702            self.create_fts_triggers(table, index)?;
4703            // Every visible row has exactly one mapping row, so a count
4704            // mismatch marks a projection an earlier release damaged; the
4705            // open rebuilds it.
4706            let stale: bool = !existed
4707                || self
4708                    .conn
4709                    .query_row(
4710                        &format!(
4711                            "SELECT (SELECT count(*) FROM {mapping}) != (SELECT count(*) FROM {})",
4712                            visible_table(&table.name)
4713                        ),
4714                        [],
4715                        |row| row.get(0),
4716                    )
4717                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4718            if stale {
4719                self.rebuild_fts_projection(table, index)?;
4720            }
4721            Ok(())
4722        })();
4723        if let Err(error) = result {
4724            self.rollback_observation("syncular_fts_schema");
4725            return Err(error);
4726        }
4727        if let Err(error) = self.conn.execute_batch("RELEASE syncular_fts_schema") {
4728            self.rollback_observation("syncular_fts_schema");
4729            return Err(error.to_string());
4730        }
4731        Ok(())
4732    }
4733
4734    // -- persistence write-through --------------------------------------------
4735
4736    fn persist_outbox_insert(&self, commit: &OutboxCommit) -> Result<(), String> {
4737        let ops: Vec<Value> = commit
4738            .ops
4739            .iter()
4740            .map(|op| {
4741                let mut entry = serde_json::json!({
4742                    "op": if op.upsert { "upsert" } else { "delete" },
4743                    "table": op.table,
4744                    "rowId": op.row_id,
4745                    "baseVersion": op.base_version,
4746                    "values": op.values.clone().map(Value::Object),
4747                });
4748                if op.insert {
4749                    entry["insert"] = Value::Bool(true);
4750                }
4751                entry
4752            })
4753            .collect();
4754        self.conn
4755            .execute(
4756                "INSERT OR REPLACE INTO _syncular_outbox (commit_id, ops_json, created_at_ms) VALUES (?1, ?2, ?3)",
4757                rusqlite::params![
4758                    commit.client_commit_id,
4759                    Value::Array(ops).to_string(),
4760                    self.clock_now_ms()
4761                ],
4762            )
4763            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4764        for operation in &commit.ops {
4765            let Some(values) = &operation.values else {
4766                continue;
4767            };
4768            let Some(table) = self.schema.table(&operation.table) else {
4769                continue;
4770            };
4771            for column in table
4772                .columns
4773                .iter()
4774                .filter(|column| column.ty == ColumnType::BlobRef)
4775            {
4776                let Some(Value::String(raw)) = values.get(&column.name) else {
4777                    continue;
4778                };
4779                let Ok(blob_ref) = serde_json::from_str::<Value>(raw) else {
4780                    continue;
4781                };
4782                let Some(blob_id) = blob_ref.get("blobId").and_then(Value::as_str) else {
4783                    continue;
4784                };
4785                self.conn
4786                    .execute(
4787                        "INSERT OR IGNORE INTO _syncular_blob_commit_refs(commit_id, blob_id)
4788                         SELECT ?1, ?2 WHERE EXISTS (SELECT 1 FROM _syncular_blobs WHERE blob_id = ?2)",
4789                        rusqlite::params![commit.client_commit_id, blob_id],
4790                    )
4791                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4792            }
4793        }
4794        Ok(())
4795    }
4796
4797    fn delete_outbox_persisted(&self, client_commit_id: &str) -> Result<(), String> {
4798        self.conn
4799            .execute(
4800                "DELETE FROM _syncular_blob_commit_refs WHERE commit_id = ?1",
4801                rusqlite::params![client_commit_id],
4802            )
4803            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4804        self.conn
4805            .execute(
4806                "DELETE FROM _syncular_outbox WHERE commit_id = ?1",
4807                rusqlite::params![client_commit_id],
4808            )
4809            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4810        self.conn.execute("DELETE FROM _syncular_row_deliveries WHERE NOT EXISTS(SELECT 1 FROM _syncular_outbox,json_each(ops_json) WHERE json_extract(json_each.value,'$.table')=tbl AND json_extract(json_each.value,'$.rowId')=id)", [])
4811            .map(|_| ())
4812            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
4813    }
4814
4815    fn outcome_status_name(status: CommitOutcomeStatus) -> &'static str {
4816        match status {
4817            CommitOutcomeStatus::Applied => "applied",
4818            CommitOutcomeStatus::Cached => "cached",
4819            CommitOutcomeStatus::Conflict => "conflict",
4820            CommitOutcomeStatus::Rejected => "rejected",
4821        }
4822    }
4823
4824    fn outcome_resolution_name(resolution: CommitOutcomeResolution) -> &'static str {
4825        match resolution {
4826            CommitOutcomeResolution::Active => "active",
4827            CommitOutcomeResolution::ResolvedKeepServer => "resolved_keep_server",
4828            CommitOutcomeResolution::Superseded => "superseded",
4829            CommitOutcomeResolution::Dismissed => "dismissed",
4830        }
4831    }
4832
4833    fn parse_outcome_status(value: &str) -> Result<CommitOutcomeStatus, String> {
4834        match value {
4835            "applied" => Ok(CommitOutcomeStatus::Applied),
4836            "cached" => Ok(CommitOutcomeStatus::Cached),
4837            "conflict" => Ok(CommitOutcomeStatus::Conflict),
4838            "rejected" => Ok(CommitOutcomeStatus::Rejected),
4839            _ => Err(format!("invalid persisted commit outcome status {value:?}")),
4840        }
4841    }
4842
4843    fn parse_outcome_resolution(value: &str) -> Result<CommitOutcomeResolution, String> {
4844        match value {
4845            "active" => Ok(CommitOutcomeResolution::Active),
4846            "resolved_keep_server" => Ok(CommitOutcomeResolution::ResolvedKeepServer),
4847            "superseded" => Ok(CommitOutcomeResolution::Superseded),
4848            "dismissed" => Ok(CommitOutcomeResolution::Dismissed),
4849            _ => Err(format!(
4850                "invalid persisted commit outcome resolution {value:?}"
4851            )),
4852        }
4853    }
4854
4855    fn persist_commit_outcome(
4856        &self,
4857        client_commit_id: &str,
4858        status: CommitOutcomeStatus,
4859        results: &[CommitOperationOutcome],
4860        operations: Option<&[OutboxOp]>,
4861    ) -> Result<(), String> {
4862        let results_json = serde_json::to_string(results).map_err(|error| error.to_string())?;
4863        let operations_json = operations
4864            .map(|items| {
4865                serde_json::to_string(&items.iter().map(CommitOperation::from).collect::<Vec<_>>())
4866            })
4867            .transpose()
4868            .map_err(|error| error.to_string())?;
4869        self.conn
4870            .execute(
4871                "INSERT INTO _syncular_commit_outcomes (
4872                   client_commit_id, status, recorded_at_ms, results_json,
4873                   operations_json, resolution
4874                 ) VALUES (?1, ?2, ?3, ?4, ?5, 'active')",
4875                rusqlite::params![
4876                    client_commit_id,
4877                    Self::outcome_status_name(status),
4878                    self.clock_now_ms(),
4879                    results_json,
4880                    operations_json
4881                ],
4882            )
4883            .map(|_| ())
4884            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
4885    }
4886
4887    /// §7.2: the journal holds a stored cell only in its canonical shape. A cell
4888    /// the byte-envelope rule refuses cannot be journaled verbatim.
4889    fn stored_cell_is_representable(cell: &Value) -> bool {
4890        cell.is_null()
4891            || cell.is_string()
4892            || cell.is_boolean()
4893            || cell.is_number()
4894            || is_canonical_bytes_envelope(cell)
4895    }
4896
4897    fn stored_row_is_representable(value: &Value) -> bool {
4898        value
4899            .as_object()
4900            .is_some_and(|row| row.values().all(Self::stored_cell_is_representable))
4901    }
4902
4903    fn stored_operation_is_representable(value: &Value) -> bool {
4904        value.get("table").is_some_and(Value::is_string)
4905            && value.get("rowId").is_some_and(Value::is_string)
4906            && value.get("baseVersion").is_none_or(|v| {
4907                v.as_i64()
4908                    .is_some_and(|n| n.unsigned_abs() <= MAX_JS_SAFE_INTEGER)
4909            })
4910            && match value.get("op").and_then(Value::as_str) {
4911                Some("upsert") => {
4912                    value
4913                        .get("values")
4914                        .is_some_and(Self::stored_row_is_representable)
4915                        && value
4916                            .get("insert")
4917                            .is_none_or(|insert| *insert == Value::Bool(true))
4918                }
4919                Some("delete") => value.get("values").is_none() && value.get("insert").is_none(),
4920                _ => false,
4921            }
4922    }
4923
4924    /// §7.2/§7.4.4: a dropped commit keeps its operation envelope in the journal
4925    /// only when the journal reader accepts the serialized operation, which is the
4926    /// rule `outcome_from_row` applies to this same row later.
4927    fn outbox_op_is_journalable(operation: &OutboxOp) -> bool {
4928        serde_json::to_value(CommitOperation::from(operation))
4929            .is_ok_and(|value| Self::stored_operation_is_representable(&value))
4930    }
4931
4932    fn outcome_from_row(row: StoredCommitOutcomeRow) -> Result<CommitOutcome, String> {
4933        let StoredCommitOutcomeRow {
4934            sequence,
4935            client_commit_id,
4936            status,
4937            recorded_at_ms,
4938            results_json,
4939            operations_json,
4940            resolution,
4941            resolved_at_ms,
4942            replacement_client_commit_id,
4943        } = row;
4944        if sequence.unsigned_abs() > MAX_JS_SAFE_INTEGER
4945            || recorded_at_ms.unsigned_abs() > MAX_JS_SAFE_INTEGER
4946            || resolved_at_ms.is_some_and(|value| value.unsigned_abs() > MAX_JS_SAFE_INTEGER)
4947        {
4948            return Err("sync.local_corrupt: persisted commit outcome is invalid".to_owned());
4949        }
4950        let invalid = || "sync.local_corrupt: persisted commit outcome is invalid".to_owned();
4951        let stored: Vec<Value> = serde_json::from_str(&results_json).map_err(|_| invalid())?;
4952        for result in &stored {
4953            let nested = match result.get("status").and_then(Value::as_str) {
4954                Some("conflict") => result.get("conflict"),
4955                Some("error") => result.get("rejection"),
4956                _ => None,
4957            };
4958            if nested
4959                .and_then(|record| record.get("operation"))
4960                .is_some_and(|op| !Self::stored_operation_is_representable(op))
4961                || (result.get("status").and_then(Value::as_str) == Some("conflict")
4962                    && nested
4963                        .and_then(|record| record.get("serverRow"))
4964                        .is_some_and(|row| !Self::stored_row_is_representable(row)))
4965            {
4966                return Err(invalid());
4967            }
4968        }
4969        let results: Vec<CommitOperationOutcome> =
4970            serde_json::from_value(Value::Array(stored)).map_err(|_| invalid())?;
4971        let has_conflict = results
4972            .iter()
4973            .any(|result| matches!(result, CommitOperationOutcome::Conflict { .. }));
4974        let has_error = results
4975            .iter()
4976            .any(|result| matches!(result, CommitOperationOutcome::Error { .. }));
4977        let status = Self::parse_outcome_status(&status).map_err(|_| invalid())?;
4978        let redacted = results.is_empty() && operations_json.is_none();
4979        if match status {
4980            CommitOutcomeStatus::Applied | CommitOutcomeStatus::Cached => has_conflict || has_error,
4981            CommitOutcomeStatus::Conflict => !(has_conflict || redacted),
4982            CommitOutcomeStatus::Rejected => has_conflict || !(has_error || redacted),
4983        } {
4984            return Err(invalid());
4985        }
4986        let operations: Option<Vec<CommitOperation>> = operations_json
4987            .map(|raw| {
4988                let stored: Vec<Value> = serde_json::from_str(&raw).map_err(|_| invalid())?;
4989                if !stored.iter().all(Self::stored_operation_is_representable) {
4990                    return Err(invalid());
4991                }
4992                serde_json::from_value(Value::Array(stored)).map_err(|_| invalid())
4993            })
4994            .transpose()?;
4995        for result in &results {
4996            match result {
4997                CommitOperationOutcome::Conflict { conflict }
4998                    if conflict.server_version.unsigned_abs() > MAX_JS_SAFE_INTEGER =>
4999                {
5000                    return Err("sync.local_corrupt: persisted commit outcome is invalid".to_owned())
5001                }
5002                CommitOperationOutcome::Error { rejection } => {
5003                    if let Some(details) = &rejection.details {
5004                        let encoded = serde_json::to_string(details).map_err(|_| {
5005                            "sync.local_corrupt: persisted commit outcome is invalid".to_owned()
5006                        })?;
5007                        RejectionDetails::parse(&encoded).map_err(|_| {
5008                            "sync.local_corrupt: persisted commit outcome is invalid".to_owned()
5009                        })?;
5010                    }
5011                }
5012                _ => {}
5013            }
5014        }
5015        Ok(CommitOutcome {
5016            sequence,
5017            client_commit_id,
5018            status,
5019            recorded_at_ms,
5020            results,
5021            operations,
5022            retained_rows: None,
5023            resolution: Self::parse_outcome_resolution(&resolution)?,
5024            resolved_at_ms,
5025            replacement_client_commit_id,
5026        })
5027    }
5028
5029    /// Resolve declared unique keys through SQLite, including affinity and NULL semantics.
5030    fn unique_conflicts(
5031        &self,
5032        table: &TableSchema,
5033        values: &Map<String, Value>,
5034        base: bool,
5035    ) -> Result<Vec<RetainedUniqueConflict>, String> {
5036        let source = if base {
5037            base_table(&table.name)
5038        } else {
5039            visible_table(&table.name)
5040        };
5041        let mut conflicts = Vec::new();
5042        for index in table.indexes.iter().filter(|index| index.unique) {
5043            let mut params = vec![json_param_to_sql(
5044                values.get(&table.primary_key).ok_or("sync.local_corrupt")?,
5045            )?];
5046            for name in &index.columns {
5047                let column = table
5048                    .columns
5049                    .iter()
5050                    .find(|column| column.name == *name)
5051                    .ok_or("sync.local_corrupt")?;
5052                let cell = json_to_column_value(column, values.get(name))?;
5053                params.push(owned_sql_value(RowParam::Cell(&cell))?);
5054            }
5055            let sql = format!(
5056                "SELECT * FROM {source} WHERE {} != ? AND {}",
5057                quote_ident(&table.primary_key),
5058                index
5059                    .columns
5060                    .iter()
5061                    .map(|column| format!("{} = ?", quote_ident(column)))
5062                    .collect::<Vec<_>>()
5063                    .join(" AND ")
5064            );
5065            let mut statement = self
5066                .conn
5067                .prepare_cached(&sql)
5068                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5069            let records = statement
5070                .query_map(rusqlite::params_from_iter(params), |record| {
5071                    let values = table
5072                        .columns
5073                        .iter()
5074                        .enumerate()
5075                        .map(|(index, column)| {
5076                            let value = sql_ref_to_json(column, record.get_ref(index)?);
5077                            json_to_column_value(column, Some(&value)).map_err(|message| {
5078                                rusqlite::Error::ToSqlConversionFailure(message.into())
5079                            })
5080                        })
5081                        .collect::<Result<Row, _>>()?;
5082                    Ok((values, record.get::<_, i64>(table.columns.len())?))
5083                })
5084                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5085            for record in records {
5086                let (row, version) =
5087                    record.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5088                let row_id = render_row_id_json(Some(&column_value_to_json(&row[table.pk_index])))?;
5089                conflicts.push(RetainedUniqueConflict {
5090                    index: index.name.clone(),
5091                    columns: index.columns.clone(),
5092                    row_id,
5093                    server_row: table
5094                        .columns
5095                        .iter()
5096                        .zip(&row)
5097                        .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
5098                        .collect(),
5099                    server_version: version,
5100                });
5101            }
5102        }
5103        Ok(conflicts)
5104    }
5105
5106    fn attach_retained_rows(&self, mut outcome: CommitOutcome) -> Result<CommitOutcome, String> {
5107        if let Some((commit, initial)) = self
5108            .failed_commits
5109            .iter()
5110            .find(|(commit, _)| commit.client_commit_id == outcome.client_commit_id)
5111        {
5112            let mut rows = Vec::new();
5113            for operation in &commit.ops {
5114                // A table the current schema removed has no rows to retain.
5115                let Some(table) = self.schema.table(&operation.table) else {
5116                    continue;
5117                };
5118                let key =
5119                    serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
5120                        .map_err(|_| "sync.local_corrupt".to_owned())?;
5121                let base = self.stored_row(table, &operation.row_id, true)?;
5122                rows.push(RetainedCommitRow {
5123                    table: operation.table.clone(),
5124                    row_id: operation.row_id.clone(),
5125                    unique_conflicts: if operation.upsert {
5126                        self.unique_conflicts(
5127                            table,
5128                            initial
5129                                .get(&key)
5130                                .and_then(Value::as_object)
5131                                .or(operation.values.as_ref())
5132                                .ok_or("sync.local_corrupt")?,
5133                            true,
5134                        )?
5135                    } else {
5136                        Vec::new()
5137                    },
5138                    local_row: if operation.upsert {
5139                        initial
5140                            .get(&key)
5141                            .and_then(Value::as_object)
5142                            .cloned()
5143                            .or_else(|| operation.values.clone())
5144                    } else {
5145                        None
5146                    },
5147                    server_row: base.as_ref().map(|(values, _)| {
5148                        table
5149                            .columns
5150                            .iter()
5151                            .zip(values)
5152                            .map(|(column, value)| {
5153                                (column.name.clone(), column_value_to_json(value))
5154                            })
5155                            .collect()
5156                    }),
5157                    server_version: base.map(|(_, version)| version),
5158                });
5159            }
5160            outcome.retained_rows = Some(rows);
5161        }
5162        Ok(outcome)
5163    }
5164
5165    fn commit_outcome(&self, client_commit_id: &str) -> Result<Option<CommitOutcome>, String> {
5166        let row = self
5167            .conn
5168            .query_row(
5169                "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
5170                        resolution, resolved_at_ms, replacement_client_commit_id
5171                   FROM _syncular_commit_outcomes WHERE client_commit_id = ?1",
5172                rusqlite::params![client_commit_id],
5173                |row| {
5174                    Ok(StoredCommitOutcomeRow {
5175                        sequence: row.get(0)?,
5176                        client_commit_id: row.get(1)?,
5177                        status: row.get(2)?,
5178                        recorded_at_ms: row.get(3)?,
5179                        results_json: row.get(4)?,
5180                        operations_json: row.get(5)?,
5181                        resolution: row.get(6)?,
5182                        resolved_at_ms: row.get(7)?,
5183                        replacement_client_commit_id: row.get(8)?,
5184                    })
5185                },
5186            )
5187            .optional()
5188            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5189        row.map(|row| self.attach_retained_rows(Self::outcome_from_row(row)?))
5190            .transpose()
5191    }
5192
5193    /// Internal host step. The host must run its idle drive before finishing.
5194    #[doc(hidden)]
5195    pub fn prepare_current_base(
5196        &self,
5197        client_commit_id: &str,
5198        sequence: i64,
5199    ) -> Result<CurrentBasePreparation, CurrentBaseProof> {
5200        let outcome = self
5201            .commit_outcome(client_commit_id)
5202            .map_err(|_| CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed))?;
5203        let valid = outcome.as_ref().is_some_and(|outcome| {
5204            outcome.sequence == sequence
5205                && outcome.resolution == CommitOutcomeResolution::Active
5206                && matches!(
5207                    outcome.status,
5208                    CommitOutcomeStatus::Conflict | CommitOutcomeStatus::Rejected
5209                )
5210                && outcome
5211                    .operations
5212                    .as_ref()
5213                    .zip(outcome.retained_rows.as_ref())
5214                    .is_some_and(|(operations, rows)| {
5215                        !rows.is_empty() && operations.len() == rows.len()
5216                    })
5217        });
5218        if !valid {
5219            return Err(CurrentBaseProof::failure(
5220                CurrentBaseProofReason::OutcomeUnavailable,
5221            ));
5222        }
5223        if !self.transport_enabled {
5224            return Err(CurrentBaseProof::failure(CurrentBaseProofReason::Offline));
5225        }
5226        Ok(CurrentBasePreparation {
5227            client_commit_id: client_commit_id.to_owned(),
5228            sequence,
5229            replacement: None,
5230            units: load_units(&self.conn)
5231                .map_err(|_| CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed))?,
5232            transport_epoch: self.transport_epoch,
5233            registration_epoch: self.registration_epoch,
5234            security_epoch: self.security_epoch,
5235        })
5236    }
5237
5238    /// Internal host step after the completed post-rejection idle drive.
5239    #[doc(hidden)]
5240    pub fn finish_current_base(
5241        &mut self,
5242        prepared: CurrentBasePreparation,
5243        drive: SyncOutcome,
5244    ) -> CurrentBaseProof {
5245        if !self.transport_enabled || self.transport_epoch != prepared.transport_epoch {
5246            return CurrentBaseProof::failure(CurrentBaseProofReason::Offline);
5247        }
5248        if self.registration_epoch != prepared.registration_epoch {
5249            return CurrentBaseProof::failure(CurrentBaseProofReason::MissingCoverage);
5250        }
5251        if self.security_preflight || self.security_epoch != prepared.security_epoch {
5252            return CurrentBaseProof::failure(CurrentBaseProofReason::SyncFailed);
5253        }
5254        let report = match drive {
5255            SyncOutcome::Ok(report) => report,
5256            SyncOutcome::BudgetExhausted(_) => {
5257                return CurrentBaseProof::failure(CurrentBaseProofReason::BudgetExhausted)
5258            }
5259            _ => return CurrentBaseProof::failure(CurrentBaseProofReason::SyncFailed),
5260        };
5261        if report.schema_floor.is_some() {
5262            return CurrentBaseProof::failure(CurrentBaseProofReason::SchemaFloor);
5263        }
5264        if self
5265            .conn
5266            .execute_batch("SAVEPOINT syncular_base_proof")
5267            .is_err()
5268        {
5269            return CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed);
5270        }
5271        let proof = (|| {
5272            let snapshot = snapshot_read_connection(
5273                &self.conn,
5274                &SnapshotReadRequest {
5275                    units: prepared.units.iter().map(|unit| unit.id.clone()).collect(),
5276                    commit_ids: vec![prepared.client_commit_id.clone()],
5277                    ..Default::default()
5278                },
5279            )
5280            .map_err(|_| CurrentBaseProofReason::SnapshotFailed)?;
5281            let current = self
5282                .commit_outcome(&prepared.client_commit_id)
5283                .map_err(|_| CurrentBaseProofReason::SnapshotFailed)?;
5284            let unavailable = if report.revoked.is_empty() {
5285                CurrentBaseProofReason::OutcomeUnavailable
5286            } else {
5287                CurrentBaseProofReason::Revoked
5288            };
5289            let Some(outcome) = current else {
5290                return Err(unavailable);
5291            };
5292            if !matches!(
5293                snapshot.deliveries.first(),
5294                Some(CommitDelivery::Known { .. })
5295            ) || outcome.sequence != prepared.sequence
5296                || outcome.resolution != CommitOutcomeResolution::Active
5297                || !matches!(
5298                    outcome.status,
5299                    CommitOutcomeStatus::Conflict | CommitOutcomeStatus::Rejected
5300                )
5301            {
5302                return Err(unavailable);
5303            }
5304            let (Some(operations), Some(rows)) = (outcome.operations, outcome.retained_rows) else {
5305                return Err(unavailable);
5306            };
5307            if rows.is_empty() || operations.len() != rows.len() {
5308                return Err(unavailable);
5309            }
5310            let after =
5311                load_units(&self.conn).map_err(|_| CurrentBaseProofReason::SnapshotFailed)?;
5312            let complete: HashSet<&str> = snapshot
5313                .units
5314                .iter()
5315                .filter_map(|unit| match unit {
5316                    UnitCatchup::Known {
5317                        id,
5318                        bootstrap_complete: true,
5319                        ..
5320                    } => Some(id.as_str()),
5321                    _ => None,
5322                })
5323                .collect();
5324            // A unit id names one scope map (§4.1), so the same id before and
5325            // after the drive is the same registration.
5326            let candidates: Vec<&UnitRecord> = after
5327                .iter()
5328                .filter(|unit| {
5329                    complete.contains(unit.id.as_str())
5330                        && prepared.units.iter().any(|before| before.id == unit.id)
5331                })
5332                .collect();
5333            let covered = |table_name: &str, row: &Map<String, Value>| {
5334                let Some(table) = self.schema.table(table_name) else {
5335                    return false;
5336                };
5337                candidates.iter().any(|unit| {
5338                    covers(table, &unit.scopes)
5339                        && pair_scopes(&unit.scopes, table_name, unit.echo.as_deref()).is_some_and(
5340                            |scopes| {
5341                                scopes.iter().all(|(variable, allowed)| {
5342                                    table
5343                                        .scope_column(variable)
5344                                        .and_then(|column| match row.get(column) {
5345                                            Some(Value::String(value)) => Some(value.clone()),
5346                                            Some(Value::Bool(value)) => Some(value.to_string()),
5347                                            Some(Value::Number(value)) => {
5348                                                value.as_f64().map(|number| {
5349                                                    ryu_js::Buffer::new()
5350                                                        .format_finite(number)
5351                                                        .to_owned()
5352                                                })
5353                                            }
5354                                            _ => None,
5355                                        })
5356                                        .is_some_and(|value| {
5357                                            !value.is_empty() && allowed.contains(&value)
5358                                        })
5359                                })
5360                            },
5361                        )
5362                })
5363            };
5364            for row in &rows {
5365                let Some(base) = row.server_row.as_ref().or(row.local_row.as_ref()) else {
5366                    return Err(if report.revoked.is_empty() {
5367                        CurrentBaseProofReason::MissingCoverage
5368                    } else {
5369                        CurrentBaseProofReason::Revoked
5370                    });
5371                };
5372                if !covered(&row.table, base)
5373                    || row
5374                        .unique_conflicts
5375                        .iter()
5376                        .any(|conflict| !covered(&row.table, &conflict.server_row))
5377                {
5378                    return Err(if report.revoked.is_empty() {
5379                        CurrentBaseProofReason::MissingCoverage
5380                    } else {
5381                        CurrentBaseProofReason::Revoked
5382                    });
5383                }
5384            }
5385            Ok(rows)
5386        })();
5387        if self
5388            .conn
5389            .execute_batch("RELEASE syncular_base_proof")
5390            .is_err()
5391        {
5392            return CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed);
5393        }
5394        match proof {
5395            Ok(rows) => CurrentBaseProof {
5396                proven: true,
5397                rows: Some(rows),
5398                reason: None,
5399            },
5400            Err(reason) => CurrentBaseProof::failure(reason),
5401        }
5402    }
5403
5404    pub fn prove_current_base(
5405        &mut self,
5406        transport: &mut dyn Transport,
5407        client_commit_id: &str,
5408        sequence: i64,
5409        max_rounds: Option<u32>,
5410    ) -> CurrentBaseProof {
5411        let prepared = match self.prepare_current_base(client_commit_id, sequence) {
5412            Ok(prepared) => prepared,
5413            Err(proof) => return proof,
5414        };
5415        let drive = self.sync_until_idle(transport, max_rounds);
5416        self.finish_current_base(prepared, drive)
5417    }
5418
5419    pub fn commit_outcomes(&self, query: CommitOutcomeQuery) -> Result<Vec<CommitOutcome>, String> {
5420        if query.limit == Some(0) {
5421            return Err("sync.invalid_request: commit outcome limit must be positive".to_owned());
5422        }
5423        let mut sql = String::from(
5424            "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
5425                    resolution, resolved_at_ms, replacement_client_commit_id
5426               FROM _syncular_commit_outcomes",
5427        );
5428        if query.active_only {
5429            sql.push_str(" WHERE resolution = 'active' AND status IN ('conflict', 'rejected')");
5430        }
5431        sql.push_str(" ORDER BY seq DESC");
5432        if let Some(limit) = query.limit {
5433            sql.push_str(&format!(" LIMIT {limit}"));
5434        }
5435        let mut stmt = self
5436            .conn
5437            .prepare(&sql)
5438            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5439        let rows = stmt
5440            .query_map([], |row| {
5441                Ok(StoredCommitOutcomeRow {
5442                    sequence: row.get(0)?,
5443                    client_commit_id: row.get(1)?,
5444                    status: row.get(2)?,
5445                    recorded_at_ms: row.get(3)?,
5446                    results_json: row.get(4)?,
5447                    operations_json: row.get(5)?,
5448                    resolution: row.get(6)?,
5449                    resolved_at_ms: row.get(7)?,
5450                    replacement_client_commit_id: row.get(8)?,
5451                })
5452            })
5453            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5454        let mut outcomes = Vec::new();
5455        for row in rows {
5456            outcomes.push(self.attach_retained_rows(Self::outcome_from_row(
5457                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
5458            )?)?);
5459        }
5460        Ok(outcomes)
5461    }
5462
5463    fn prune_commit_outcomes(&self) -> Result<(), String> {
5464        #[cfg(test)]
5465        self.outcome_prune_count
5466            .set(self.outcome_prune_count.get() + 1);
5467        let max_entries = self.limits.outcome_retention_max_entries.unwrap_or(1_000);
5468        let count = self
5469            .conn
5470            .query_row(
5471                "SELECT COUNT(*) FROM _syncular_commit_outcomes",
5472                [],
5473                |row| row.get::<_, i64>(0),
5474            )
5475            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
5476            as usize;
5477        let excess = count.saturating_sub(max_entries);
5478        if excess == 0 {
5479            return Ok(());
5480        }
5481        let mut stmt = self
5482            .conn
5483            .prepare(
5484                "SELECT seq FROM _syncular_commit_outcomes
5485                  WHERE status IN ('applied', 'cached') OR resolution != 'active'
5486                  ORDER BY seq ASC LIMIT ?1",
5487            )
5488            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5489        let rows = stmt
5490            .query_map(rusqlite::params![excess as i64], |row| row.get::<_, i64>(0))
5491            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5492        let sequences = rows
5493            .collect::<Result<Vec<_>, _>>()
5494            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5495        drop(stmt);
5496        for sequence in sequences {
5497            self.conn
5498                .execute(
5499                    "DELETE FROM _syncular_commit_outcomes WHERE seq = ?1",
5500                    rusqlite::params![sequence],
5501                )
5502                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5503        }
5504        Ok(())
5505    }
5506
5507    // -- units (§4.1) ------------------------------------------------------------
5508
5509    /// Every registered unit in registration order (§4.1).
5510    pub fn units(&self) -> Result<Vec<UnitRecord>, String> {
5511        self.stored(load_units(&self.conn))
5512    }
5513
5514    pub fn unit(&self, id: &str) -> Result<Option<UnitRecord>, String> {
5515        self.stored(get_unit(&self.conn, id))
5516    }
5517
5518    /// Register one unit (§4.1) and return its id.
5519    pub fn subscribe(&mut self, scopes: UnitScopes) -> Result<String, String> {
5520        Ok(self.subscribe_many(vec![scopes])?.remove(0))
5521    }
5522
5523    /// Register several units in one local transaction and return their ids:
5524    /// one commit for all of them (on OPFS a commit costs milliseconds, so a
5525    /// cold start with hundreds of units pays it once). An error rolls back
5526    /// every registration of the call. Registering an active unit again
5527    /// changes nothing; a revoked unit re-arms (§3.3). A call that registers or
5528    /// re-arms a unit raises sync-needed with an interactive intent.
5529    pub fn subscribe_many(&mut self, units: Vec<UnitScopes>) -> Result<Vec<String>, String> {
5530        self.begin_observation("syncular_subscribe")?;
5531        let mut changed = false;
5532        let registered = (|| {
5533            let mut ids = Vec::with_capacity(units.len());
5534            for scopes in units {
5535                let (id, held) = self.hold_unit(scopes, true)?;
5536                changed |= held;
5537                ids.push(id);
5538            }
5539            self.conn
5540                .execute_batch("RELEASE syncular_subscribe")
5541                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5542            Ok(ids)
5543        })();
5544        let ids = match registered {
5545            Ok(ids) => ids,
5546            Err(error) => {
5547                self.rollback_observation("syncular_subscribe");
5548                return Err(error);
5549            }
5550        };
5551        if changed {
5552            self.registration_epoch = self.registration_epoch.wrapping_add(1);
5553            self.set_sync_needed(true, true);
5554        }
5555        Ok(ids)
5556    }
5557
5558    /// Release the explicit reference to a unit. The unit leaves the
5559    /// registration list when no window base holds it (§4.1); its rows stay.
5560    pub fn unsubscribe(&mut self, id: &str) -> Result<(), String> {
5561        let Some(unit) = self.stored(get_unit(&self.conn, id))? else {
5562            return Ok(());
5563        };
5564        if !unit.explicit {
5565            return Ok(());
5566        }
5567        if !self.stored(window_refs(&self.conn, id))?.is_empty() {
5568            return self.stored(save_unit(
5569                &self.conn,
5570                &UnitRecord {
5571                    explicit: false,
5572                    ..unit
5573                },
5574            ));
5575        }
5576        self.conn
5577            .execute("DELETE FROM _syncular_units WHERE id = ?1", [id])
5578            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5579        self.registration_epoch = self.registration_epoch.wrapping_add(1);
5580        if unit.status == UnitStatus::Active {
5581            self.set_sync_needed(true, true);
5582        }
5583        Ok(())
5584    }
5585
5586    /// Take one reference on a unit (§4.1): validate the scope map, register a
5587    /// new unit catching up with cursor `-1`, re-arm a revoked one (§3.3).
5588    /// Returns the id and whether the registration list changed.
5589    fn hold_unit(&mut self, scopes: UnitScopes, explicit: bool) -> Result<(String, bool), String> {
5590        if scopes.is_empty()
5591            || scopes.values().any(|value| value == "*")
5592            || !self
5593                .schema
5594                .tables
5595                .iter()
5596                .any(|table| covers(table, &scopes))
5597        {
5598            return Err("sync.invalid_subscription: a unit names one non-wildcard value per variable and covers a table".to_owned());
5599        }
5600        let id = ssp2::registration::unit_id(&scope_pairs(&scopes));
5601        let existing = self.stored(get_unit(&self.conn, &id))?;
5602        if let Some(existing) = existing
5603            .as_ref()
5604            .filter(|unit| unit.status != UnitStatus::Revoked)
5605        {
5606            if explicit && !existing.explicit {
5607                self.stored(save_unit(
5608                    &self.conn,
5609                    &UnitRecord {
5610                        explicit,
5611                        ..existing.clone()
5612                    },
5613                ))?;
5614            }
5615            return Ok((id, false));
5616        }
5617        self.delete_pending_evict(&id)?;
5618        self.stored(save_unit(
5619            &self.conn,
5620            &UnitRecord {
5621                id: id.clone(),
5622                scopes,
5623                explicit: explicit || existing.is_some_and(|unit| unit.explicit),
5624                status: UnitStatus::Active,
5625                reason_code: None,
5626                cursor: Some(-1),
5627                bootstrap_state: None,
5628                seeded: Vec::new(),
5629                echo: None,
5630                restriction: None,
5631            },
5632        ))?;
5633        Ok((id, true))
5634    }
5635
5636    // -- windows (§4.8) -----------------------------------------------------------
5637
5638    /// §4.8: set the windowed-in values of a base, one unit per value. Added
5639    /// values take a reference on their unit (a new unit bootstraps on the
5640    /// next sync); removed values release theirs, and a unit no reference
5641    /// holds leaves the registration list fused with its first eviction chunk
5642    /// (E1–E4). Idempotent; re-entry cancels a deferred eviction.
5643    pub fn set_window(
5644        &mut self,
5645        base: &WindowBase,
5646        units: &[String],
5647    ) -> Result<CommandEffects, String> {
5648        let base_key = window_base_key(base);
5649        let live = self.load_window_units(&base_key)?;
5650        let mut changed = false;
5651        // §7.8: the added values register in one transaction.
5652        let added: Vec<&String> = units
5653            .iter()
5654            .filter(|value| !live.iter().any(|(held, _)| held == *value))
5655            .collect();
5656        if !added.is_empty() {
5657            let rollback = self.capture();
5658            self.begin_observation("syncular_window")?;
5659            let widened = (|| {
5660                let mut batch = ChangeAccumulator::default();
5661                for value in &added {
5662                    let (id, registered) =
5663                        self.hold_unit(window_unit_scopes(base, value), false)?;
5664                    self.sync_needed |= registered;
5665                    self.conn
5666                        .execute(
5667                            "INSERT OR REPLACE INTO _syncular_windows(base, unit, sub_id) VALUES (?1, ?2, ?3)",
5668                            rusqlite::params![base_key, value, id],
5669                        )
5670                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5671                    batch.window(&base_key, value);
5672                }
5673                batch.status = true;
5674                self.finish_observation("syncular_window", batch)
5675            })();
5676            if let Err(error) = widened {
5677                self.rollback_observation("syncular_window");
5678                self.restore(rollback);
5679                return Err(error);
5680            }
5681            changed = true;
5682            self.registration_epoch = self.registration_epoch.wrapping_add(1);
5683        }
5684        for (value, id) in live {
5685            if units.contains(&value) {
5686                continue;
5687            }
5688            self.rebuild_overlay_if_dirty()?;
5689            let rollback = self.capture();
5690            self.begin_observation("syncular_window")?;
5691            let remaining = self.release_window_unit(&base_key, &value, &id);
5692            let remaining = match remaining {
5693                Ok(remaining) => remaining,
5694                Err(error) => {
5695                    self.rollback_observation("syncular_window");
5696                    self.restore(rollback);
5697                    return Err(error);
5698                }
5699            };
5700            changed = true;
5701            self.registration_epoch = self.registration_epoch.wrapping_add(1);
5702            if remaining {
5703                self.drain_pending_evictions()?;
5704            }
5705        }
5706        Ok(if changed && self.transport_enabled {
5707            CommandEffects::interactive()
5708        } else {
5709            CommandEffects::none()
5710        })
5711    }
5712
5713    /// §4.8 completeness oracle: the windowed-in values of a base and those
5714    /// whose unit is not complete. A value is complete once its unit is active
5715    /// and no longer bootstrapping; registration alone is not completeness.
5716    pub fn window_state(&self, base: &WindowBase) -> Result<WindowState, String> {
5717        persisted_window_state(&self.conn, base).map_err(|failure| failure.message)
5718    }
5719
5720    /// Registry rows for a base, ordered by value: (value, unit id).
5721    fn load_window_units(&self, base_key: &str) -> Result<Vec<(String, String)>, String> {
5722        let mut statement = self
5723            .conn
5724            .prepare_cached(
5725                "SELECT unit, sub_id FROM _syncular_windows WHERE base = ?1 ORDER BY unit ASC",
5726            )
5727            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5728        let rows = statement
5729            .query_map([base_key], |row| Ok((row.get(0)?, row.get(1)?)))
5730            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5731        rows.collect::<Result<_, _>>()
5732            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5733    }
5734
5735    /// Every window registry row: (base key, value).
5736    fn load_window_registry(&self) -> Result<Vec<(String, String)>, String> {
5737        let mut statement = self
5738            .conn
5739            .prepare_cached("SELECT base, unit FROM _syncular_windows ORDER BY base, unit")
5740            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5741        let rows = statement
5742            .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))
5743            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5744        rows.collect::<Result<_, _>>()
5745            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5746    }
5747
5748    /// §4.8: release a window value's reference inside the caller's
5749    /// observation. When no reference holds the unit, unregister it fused with
5750    /// its first eviction chunk (E3): delete the rows of every covered table
5751    /// matching the persisted echo, except rows pinned by the outbox (E1) or
5752    /// owned by another unit (§3.3), and park the per-table scopes of what
5753    /// remains in the pending-eviction registry. Fails closed without a local
5754    /// scope-column mapping. Returns whether unpinned rows remain.
5755    fn release_window_unit(
5756        &mut self,
5757        base_key: &str,
5758        value: &str,
5759        id: &str,
5760    ) -> Result<bool, String> {
5761        self.conn
5762            .execute(
5763                "DELETE FROM _syncular_windows WHERE base = ?1 AND unit = ?2",
5764                rusqlite::params![base_key, value],
5765            )
5766            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5767        let mut batch = ChangeAccumulator::default();
5768        batch.window(base_key, value);
5769        batch.status = true;
5770        let unit = self.stored(get_unit(&self.conn, id))?;
5771        let mut remaining = false;
5772        if let Some(unit) = unit.filter(|unit| !unit.explicit) {
5773            if self.stored(window_refs(&self.conn, id))?.is_empty() {
5774                let units = self.stored(load_units(&self.conn))?;
5775                self.conn
5776                    .execute("DELETE FROM _syncular_units WHERE id = ?1", [id])
5777                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5778                self.sync_needed = true;
5779                for table in self.schema.tables.clone() {
5780                    let Some(effective) = unit
5781                        .echo
5782                        .as_deref()
5783                        .filter(|_| covers(&table, &unit.scopes))
5784                        .and_then(|echo| pair_scopes(&unit.scopes, &table.name, Some(echo)))
5785                    else {
5786                        continue;
5787                    };
5788                    let pinned = self.pinned_row_ids(&table.name);
5789                    let owners = unit_owners(&units, &table, id);
5790                    let (more, deferred) =
5791                        self.evict_scope_rows(&table.name, &effective, &owners, &pinned)?;
5792                    remaining |= more;
5793                    if deferred {
5794                        self.save_pending_evict(id, &table.name, &effective)?;
5795                    }
5796                    self.record_scope_map(&mut batch, &table.name, &effective);
5797                }
5798            }
5799        }
5800        self.finish_observation("syncular_window", batch)?;
5801        Ok(remaining)
5802    }
5803
5804    /// §4.8 E1: delete base rows owned by the effective scopes
5805    /// ([`local_scope_predicate`]) EXCEPT pinned primary keys. Returns (more
5806    /// unpinned work, any scoped rows remain). A missing scope-column mapping
5807    /// fails without deleting rows.
5808    fn evict_scope_rows(
5809        &self,
5810        table_name: &str,
5811        effective: &[(String, Vec<String>)],
5812        owners: &[Vec<(String, Vec<String>)>],
5813        pinned: &std::collections::HashSet<String>,
5814    ) -> Result<(bool, bool), String> {
5815        let table = self.schema.table(table_name).ok_or("sync.unknown_table")?;
5816        let Some((scope, mut params)) = local_scope_predicate(table, effective, owners)
5817            .map_err(|()| "sync.scope_revoked".to_owned())?
5818        else {
5819            return Ok((false, false));
5820        };
5821        let scope_params = params.len();
5822        let pk = quote_ident(&table.primary_key);
5823        // Pinned ids are rowId text; the text form matches every key type.
5824        let predicate = if pinned.is_empty() {
5825            scope.clone()
5826        } else {
5827            params.extend(pinned.iter().cloned().map(SqlValue::Text));
5828            let holes = vec!["?"; pinned.len()].join(", ");
5829            format!("{scope} AND CAST({pk} AS TEXT) NOT IN ({holes})")
5830        };
5831        let base = base_table(table_name);
5832        let visible = visible_table(table_name);
5833        let mut ids: Vec<SqlValue> = Vec::new();
5834        for source in [&base, &visible] {
5835            let mut stmt = self
5836                .conn
5837                .prepare_cached(&format!(
5838                    "SELECT {pk} FROM {source} WHERE {predicate} LIMIT 1024"
5839                ))
5840                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5841            ids = stmt
5842                .query_map(rusqlite::params_from_iter(&params), |row| row.get(0))
5843                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
5844                .collect::<Result<_, _>>()
5845                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5846            if !ids.is_empty() {
5847                break;
5848            }
5849        }
5850        if !ids.is_empty() {
5851            for id in &ids {
5852                self.conn
5853                    .execute(
5854                        "DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2",
5855                        rusqlite::params![table_name, id],
5856                    )
5857                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5858            }
5859            let holes = vec!["?"; ids.len()].join(", ");
5860            for source in [&base, &visible] {
5861                self.conn
5862                    .execute(
5863                        &format!("DELETE FROM {source} WHERE {pk} IN ({holes})"),
5864                        rusqlite::params_from_iter(&ids),
5865                    )
5866                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5867            }
5868        }
5869        let mut remaining = false;
5870        let mut deferred = false;
5871        for source in [&base, &visible] {
5872            remaining |= self
5873                .conn
5874                .query_row(
5875                    &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {predicate})"),
5876                    rusqlite::params_from_iter(&params),
5877                    |row| row.get::<_, bool>(0),
5878                )
5879                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5880            deferred |= self
5881                .conn
5882                .query_row(
5883                    &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {scope})"),
5884                    rusqlite::params_from_iter(&params[..scope_params]),
5885                    |row| row.get::<_, bool>(0),
5886                )
5887                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5888        }
5889        Ok((remaining, deferred))
5890    }
5891
5892    /// §4.8 E1: continue deferred evictions after the outbox drains or between
5893    /// chunks, retaining outbox pins and their durable marker. A unit that
5894    /// re-entered the window has no pending entry left.
5895    fn drain_pending_evictions(&mut self) -> Result<(), String> {
5896        let pending = self.load_pending_evictions()?;
5897        if pending.is_empty() {
5898            return Ok(());
5899        }
5900        self.rebuild_overlay_if_dirty()?;
5901        for (unit_id, table_name, effective) in pending {
5902            let Some(table) = self.schema.table(&table_name).cloned() else {
5903                self.delete_pending_evict_table(&unit_id, &table_name)?;
5904                continue;
5905            };
5906            loop {
5907                std::thread::yield_now();
5908                self.begin_observation("syncular_evict")?;
5909                let chunk = (|| {
5910                    let pinned = self.pinned_row_ids(&table_name);
5911                    let owners =
5912                        unit_owners(&self.stored(load_units(&self.conn))?, &table, &unit_id);
5913                    let (remaining, deferred) =
5914                        self.evict_scope_rows(&table_name, &effective, &owners, &pinned)?;
5915                    if !deferred {
5916                        self.delete_pending_evict_table(&unit_id, &table_name)?;
5917                    }
5918                    let mut batch = ChangeAccumulator::default();
5919                    self.record_scope_map(&mut batch, &table_name, &effective);
5920                    self.finish_observation("syncular_evict", batch)?;
5921                    Ok(remaining)
5922                })();
5923                match chunk {
5924                    Ok(true) => {}
5925                    Ok(false) => break,
5926                    Err(error) => {
5927                        self.rollback_observation("syncular_evict");
5928                        return Err(error);
5929                    }
5930                }
5931            }
5932        }
5933        Ok(())
5934    }
5935
5936    /// §4.8 E1: primary keys of `table` referenced by a pending outbox
5937    /// commit — rows that MUST NOT be evicted until the commit drains.
5938    fn pinned_row_ids(&self, table: &str) -> std::collections::HashSet<String> {
5939        let mut pinned = std::collections::HashSet::new();
5940        for commit in self
5941            .outbox
5942            .iter()
5943            .map(|commit| &**commit)
5944            .chain(self.failed_commits.iter().map(|(commit, _)| commit))
5945        {
5946            for op in &commit.ops {
5947                if op.table == table {
5948                    pinned.insert(op.row_id.clone());
5949                }
5950            }
5951        }
5952        pinned
5953    }
5954
5955    fn save_pending_evict(
5956        &self,
5957        unit_id: &str,
5958        table: &str,
5959        effective: &[(String, Vec<String>)],
5960    ) -> Result<(), String> {
5961        self.conn.execute(
5962            "INSERT OR REPLACE INTO _syncular_pending_evictions(unit_id, tbl, effective_scopes) VALUES (?1, ?2, ?3)",
5963            rusqlite::params![unit_id, table, scope_map_to_json(effective).to_string()],
5964        ).map(|_| ()).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5965    }
5966
5967    /// Re-entry cancels every pending eviction of the unit.
5968    fn delete_pending_evict(&self, unit_id: &str) -> Result<(), String> {
5969        self.conn
5970            .execute(
5971                "DELETE FROM _syncular_pending_evictions WHERE unit_id = ?1",
5972                [unit_id],
5973            )
5974            .map(|_| ())
5975            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5976    }
5977
5978    fn delete_pending_evict_table(&self, unit_id: &str, table: &str) -> Result<(), String> {
5979        self.conn
5980            .execute(
5981                "DELETE FROM _syncular_pending_evictions WHERE unit_id = ?1 AND tbl = ?2",
5982                [unit_id, table],
5983            )
5984            .map(|_| ())
5985            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5986    }
5987
5988    fn load_pending_evictions(&self) -> Result<Vec<PendingEvict>, String> {
5989        let mut stmt = self
5990            .conn
5991            .prepare("SELECT unit_id, tbl, effective_scopes FROM _syncular_pending_evictions")
5992            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5993        let rows = stmt
5994            .query_map([], |row| {
5995                Ok((
5996                    row.get::<_, String>(0)?,
5997                    row.get::<_, String>(1)?,
5998                    row.get::<_, String>(2)?,
5999                ))
6000            })
6001            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6002        rows.map(|row| {
6003            let (unit_id, table, json) =
6004                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6005            let value: Value = serde_json::from_str(&json)
6006                .map_err(|_| "sync.local_corrupt: invalid pending eviction scopes".to_owned())?;
6007            let effective = json_to_scope_map(&value)
6008                .map_err(|_| "sync.local_corrupt: invalid pending eviction scopes".to_owned())?;
6009            Ok((unit_id, table, effective))
6010        })
6011        .collect()
6012    }
6013
6014    /// Select failed-intent retention; reload durable retained aggregates.
6015    pub fn set_retain_failed_commits(&mut self, enabled: bool) -> Result<(), String> {
6016        self.retain_failed_commits = enabled;
6017        for (commit, _) in self.failed_commits.iter() {
6018            for operation in &commit.ops {
6019                self.overlay_dirty.table(&operation.table);
6020            }
6021        }
6022        self.failed_commits = self.load_failed_commits()?.into();
6023        for (commit, _) in self.failed_commits.iter() {
6024            for operation in &commit.ops {
6025                self.overlay_dirty.table(&operation.table);
6026            }
6027        }
6028        Ok(())
6029    }
6030
6031    fn load_failed_commits(&self) -> Result<Vec<FailedCommit>, String> {
6032        let mut statement = self.conn.prepare("SELECT client_commit_id, operations_json, initial_json FROM _syncular_failed_commits ORDER BY seq")
6033            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6034        let rows = statement
6035            .query_map([], |row| {
6036                Ok((
6037                    row.get::<_, String>(0)?,
6038                    row.get::<_, String>(1)?,
6039                    row.get::<_, String>(2)?,
6040                ))
6041            })
6042            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6043        rows.map(|row| {
6044            let (id, operations, initial) =
6045                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6046            let operations: Vec<CommitOperation> =
6047                serde_json::from_str(&operations).map_err(|_| "sync.local_corrupt".to_owned())?;
6048            let initial =
6049                serde_json::from_str(&initial).map_err(|_| "sync.local_corrupt".to_owned())?;
6050            Ok((
6051                OutboxCommit {
6052                    client_commit_id: id,
6053                    ops: operations
6054                        .into_iter()
6055                        .map(|op| OutboxOp {
6056                            upsert: op.op == "upsert",
6057                            table: op.table,
6058                            row_id: op.row_id,
6059                            base_version: op.base_version,
6060                            values: op.values,
6061                            insert: op.insert,
6062                        })
6063                        .collect(),
6064                },
6065                initial,
6066            ))
6067        })
6068        .collect()
6069    }
6070
6071    fn retain_failed_commit(&mut self, id: &str, operations: &[OutboxOp]) -> Result<(), String> {
6072        let mut initial = Map::new();
6073        for operation in operations {
6074            // A table the current schema removed has no rows to retain.
6075            let Some(table) = self.schema.table(&operation.table) else {
6076                continue;
6077            };
6078            let initial_row = match self.stored_row(table, &operation.row_id, true)? {
6079                Some(row) => Some(row),
6080                None => self.visible_row(table, &operation.row_id)?,
6081            };
6082            if let Some((values, _)) = initial_row {
6083                let mut intended: Map<String, Value> = table
6084                    .columns
6085                    .iter()
6086                    .zip(values.iter())
6087                    .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
6088                    .collect();
6089                if let Some(patch) = &operation.values {
6090                    intended.extend(patch.clone());
6091                }
6092                initial.insert(
6093                    serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
6094                        .map_err(|_| "sync.local_corrupt".to_owned())?,
6095                    Value::Object(intended),
6096                );
6097            }
6098        }
6099        self.conn.execute("INSERT INTO _syncular_failed_commits(client_commit_id, operations_json, initial_json) VALUES (?1, ?2, ?3)", rusqlite::params![id,
6100            serde_json::to_string(&operations.iter().map(CommitOperation::from).collect::<Vec<_>>()).map_err(|_| "sync.local_corrupt".to_owned())?,
6101            serde_json::to_string(&initial).map_err(|_| "sync.local_corrupt".to_owned())?])
6102            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6103        self.failed_commits.push((
6104            OutboxCommit {
6105                client_commit_id: id.to_owned(),
6106                ops: operations.to_vec(),
6107            },
6108            initial,
6109        ));
6110        Ok(())
6111    }
6112
6113    fn drop_failed_commits(
6114        &mut self,
6115        matches: impl Fn(&str, &Map<String, Value>) -> bool,
6116    ) -> Result<Vec<OutboxCommit>, String> {
6117        let mut doomed = self.drop_acknowledged_rows(&matches)?;
6118        for (commit, initial) in self.failed_commits.iter() {
6119            for operation in &commit.ops {
6120                let key =
6121                    serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
6122                        .map_err(|_| "sync.local_corrupt".to_owned())?;
6123                if initial
6124                    .get(&key)
6125                    .and_then(Value::as_object)
6126                    .is_some_and(|values| matches(&operation.table, values))
6127                {
6128                    doomed.push(commit.clone());
6129                    break;
6130                }
6131            }
6132        }
6133        for commit in &doomed {
6134            self.conn
6135                .execute(
6136                    "DELETE FROM _syncular_failed_commits WHERE client_commit_id = ?1",
6137                    rusqlite::params![commit.client_commit_id],
6138                )
6139                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6140        }
6141        for commit in &doomed {
6142            self.conn.execute("UPDATE _syncular_commit_outcomes SET operations_json = NULL, results_json = '[]' WHERE client_commit_id = ?1", rusqlite::params![commit.client_commit_id]).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6143        }
6144        self.conflicts.retain(|record| {
6145            !doomed
6146                .iter()
6147                .any(|commit| commit.client_commit_id == record.client_commit_id)
6148        });
6149        self.rejections.retain(|record| {
6150            !doomed
6151                .iter()
6152                .any(|commit| commit.client_commit_id == record.client_commit_id)
6153        });
6154        self.failed_commits.retain(|(commit, _)| {
6155            !doomed
6156                .iter()
6157                .any(|dropped| dropped.client_commit_id == commit.client_commit_id)
6158        });
6159        for commit in &doomed {
6160            for operation in &commit.ops {
6161                self.overlay_dirty.table(&operation.table);
6162            }
6163        }
6164        Ok(doomed)
6165    }
6166
6167    /// Record one commit of local mutations. The returned commit ID is the
6168    /// §2.3 idempotency key. Failures are structured so callers read
6169    /// `code`/`message`/`details` instead of parsing a string prefix.
6170    pub fn mutate(&mut self, mutations: Vec<Mutation>) -> Result<String, ClientError> {
6171        let prior = self.storage_failure.borrow_mut().take();
6172        let outcome = self
6173            .prepare_mutations(mutations)
6174            .and_then(|ops| Ok(self.record_outbox_commit(ops, None)?));
6175        self.finish_authoring(prior, outcome)
6176    }
6177
6178    /// §7.1: validate one commit's mutations into outbox operations.
6179    fn prepare_mutations(&self, mutations: Vec<Mutation>) -> Result<Vec<OutboxOp>, ClientError> {
6180        {
6181            if mutations.is_empty() {
6182                return Err(
6183                    "sync.invalid_request: a commit must contain at least one operation (§6.1)"
6184                        .to_owned()
6185                        .into(),
6186                );
6187            }
6188            let mut ops = Vec::with_capacity(mutations.len());
6189            // §7.1: a patch reads the row as the commit's earlier operations
6190            // leave it: the values an earlier upsert wrote, or None after an
6191            // earlier delete. A patch changes neither presence nor a scope
6192            // column, so it records nothing.
6193            let mut staged: HashMap<(String, String), Option<Map<String, Value>>> = HashMap::new();
6194            for mutation in mutations {
6195                match mutation {
6196                    Mutation::Patch {
6197                        table,
6198                        values,
6199                        base_version,
6200                    } => {
6201                        let schema_table = self
6202                            .schema
6203                            .table(&table)
6204                            .ok_or_else(|| "sync.unknown_table".to_owned())?;
6205                        let values = normalize_values_casing(schema_table, values)?;
6206                        let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
6207                        let base = staged.get(&(table.clone(), row_id.clone()));
6208                        ops.push(self.prepare_patch(
6209                            &table,
6210                            &row_id,
6211                            values,
6212                            base_version,
6213                            base,
6214                        )?);
6215                    }
6216                    Mutation::Upsert {
6217                        table,
6218                        values,
6219                        base_version,
6220                    } => {
6221                        let schema_table = self.schema.table(&table).ok_or_else(|| {
6222                            format!("sync.unknown_table: unknown table {table:?}")
6223                        })?;
6224                        // §5: value keys are accepted in snake_case AND the
6225                        // generated row types' camelCase; normalize to SQL truth
6226                        // before the pk lookup / codec see them.
6227                        let values = normalize_values_casing(schema_table, values)?;
6228                        let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
6229                        // §6.7 `mutate` marks every column present: fill missing
6230                        // nullable columns with NULL before the payload is built.
6231                        let values = full_row_values(schema_table, values)?;
6232                        self.refuse_sealed_row(schema_table, &row_id)?;
6233                        // §5.11: validate the payload encodes with the current
6234                        // codec. A key-selection or unknown-key failure is a
6235                        // durable rejection at the push seam (§10.3), never an
6236                        // author-time error. Full-row `mutate` presents every
6237                        // column, so no stored fallback is needed.
6238                        self.validate_author_encode(schema_table, &row_id, &values)?;
6239                        // §7.1 insert intent: the row is absent from the
6240                        // visible replica at call time, before this batch
6241                        // applies anything.
6242                        let insert = !self.visible_row_exists(schema_table, &row_id)?;
6243                        staged.insert((table.clone(), row_id.clone()), Some(values.clone()));
6244                        ops.push(OutboxOp {
6245                            upsert: true,
6246                            table,
6247                            row_id,
6248                            base_version,
6249                            values: Some(values),
6250                            insert,
6251                        });
6252                    }
6253                    Mutation::Delete {
6254                        table,
6255                        row_id,
6256                        base_version,
6257                    } => {
6258                        let schema_table = self.schema.table(&table).ok_or_else(|| {
6259                            format!("sync.unknown_table: unknown table {table:?}")
6260                        })?;
6261                        self.refuse_sealed_row(schema_table, &row_id)?;
6262                        // §2.4: the server rejects a rowId holding U+0000.
6263                        if row_id.contains('\0') {
6264                            return Err("sync.invalid_request: rowId holds U+0000 (§2.4)"
6265                                .to_owned()
6266                                .into());
6267                        }
6268                        staged.insert((table.clone(), row_id.clone()), None);
6269                        ops.push(OutboxOp {
6270                            upsert: false,
6271                            table,
6272                            row_id,
6273                            base_version,
6274                            values: None,
6275                            insert: false,
6276                        });
6277                    }
6278                }
6279            }
6280            Ok(ops)
6281        }
6282    }
6283
6284    /// Scope the retained `storage_failure` diagnostic to this call so an
6285    /// earlier operation's failure cannot classify a new one, and surface the
6286    /// call's own classified failure with its details. A fresh classified
6287    /// failure replaces the retained diagnostic; otherwise the earlier one is
6288    /// kept for the next sync.
6289    fn finish_authoring(
6290        &self,
6291        prior: Option<QueryReadFailure>,
6292        outcome: Result<String, ClientError>,
6293    ) -> Result<String, ClientError> {
6294        let fresh = self.storage_failure.borrow_mut().take();
6295        *self.storage_failure.borrow_mut() = fresh.clone().or(prior);
6296        outcome.map_err(|error| fresh.map(ClientError::from).unwrap_or(error))
6297    }
6298
6299    /// `decides` names the active outcome this commit replaces (§7.2.2); the
6300    /// outcome records it as its pending decision in the same transaction.
6301    fn record_outbox_commit(
6302        &mut self,
6303        ops: Vec<OutboxOp>,
6304        decides: Option<&str>,
6305    ) -> Result<String, String> {
6306        let commit = OutboxCommit {
6307            client_commit_id: uuid::Uuid::new_v4().to_string(),
6308            ops,
6309        };
6310        self.begin_observation("syncular_mutation")?;
6311        let mut batch = ChangeAccumulator::default();
6312        for op in &commit.ops {
6313            let mut precise = self.record_row_scopes(&mut batch, &op.table, &op.row_id, false);
6314            if let Some(values) = &op.values {
6315                if let Some(table) = self.schema.table(&op.table) {
6316                    for scope in &table.scope_variables {
6317                        if let Some(Value::String(value)) = values.get(&scope.column) {
6318                            batch.scope(&op.table, format!("{}:{value}", scope.prefix));
6319                            precise = true;
6320                        }
6321                    }
6322                }
6323            }
6324            if !precise {
6325                batch.table(&op.table);
6326            }
6327        }
6328        let persisted = self.persist_outbox_insert(&commit).and_then(|()| {
6329            let Some(outcome) = decides else {
6330                return Ok(());
6331            };
6332            batch.outcomes = true;
6333            let updated = self
6334                .conn
6335                .execute(
6336                    "UPDATE _syncular_commit_outcomes SET replacement_client_commit_id = ?1
6337                      WHERE client_commit_id = ?2 AND resolution = 'active'
6338                        AND replacement_client_commit_id IS NULL",
6339                    rusqlite::params![commit.client_commit_id, outcome],
6340                )
6341                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6342            if updated == 1 {
6343                Ok(())
6344            } else {
6345                Err(REPLACEMENT_PENDING.to_owned())
6346            }
6347        });
6348        if let Err(error) = persisted {
6349            self.rollback_observation("syncular_mutation");
6350            return Err(error);
6351        }
6352        // A clean overlay contains the FIFO fold of every previous commit, so
6353        // the new commit applies over it. §7.1: an operation that does not
6354        // apply (a secondary unique collision) fails the whole commit; the
6355        // savepoint rollback removes the outbox entry and every visible write.
6356        // The append undoes itself: a rollback capture would copy the outbox.
6357        let was_dirty = self.overlay_dirty.snapshot();
6358        let id = commit.client_commit_id.clone();
6359        let appended = self.rebuild_overlay_if_dirty().and_then(|()| {
6360            commit
6361                .ops
6362                .iter()
6363                .try_for_each(|op| self.apply_outbox_op(op))?;
6364            self.outbox.push(Arc::new(commit));
6365            batch.status = true;
6366            self.finish_observation("syncular_mutation", batch)
6367                .inspect_err(|_| drop(self.outbox.pop()))
6368        });
6369        if let Err(error) = appended {
6370            self.rollback_observation("syncular_mutation");
6371            self.overlay_dirty.restore(was_dirty);
6372            return Err(error);
6373        }
6374        Ok(id)
6375    }
6376
6377    /// Record one §6.1 sparse upsert: the primary key plus the supplied
6378    /// non-scope columns are present; every other column is absent and stays
6379    /// untouched on the server and in the local overlay. An absent local base
6380    /// refuses the whole batch with sync.row_missing before enqueueing (§7.1).
6381    /// Failures are structured so callers read `code`/`message`/`details`.
6382    pub fn patch(
6383        &mut self,
6384        table: &str,
6385        row_id: &str,
6386        partial: Map<String, Value>,
6387        base_version: Option<i64>,
6388    ) -> Result<String, ClientError> {
6389        let prior = self.storage_failure.borrow_mut().take();
6390        let outcome = self.perform_patch(table, row_id, partial, base_version);
6391        self.finish_authoring(prior, outcome)
6392    }
6393
6394    /// The unscoped patch body. Public authoring entry points scope
6395    /// `storage_failure` around the whole operation, including any pre-read.
6396    fn perform_patch(
6397        &mut self,
6398        table: &str,
6399        row_id: &str,
6400        partial: Map<String, Value>,
6401        base_version: Option<i64>,
6402    ) -> Result<String, ClientError> {
6403        let operation = self.prepare_patch(table, row_id, partial, base_version, None)?;
6404        Ok(self.record_outbox_commit(vec![operation], None)?)
6405    }
6406
6407    /// `staged` is the row as the batch's earlier operations leave it
6408    /// (`Some(None)` after a delete); `None` reads the visible replica.
6409    fn prepare_patch(
6410        &self,
6411        table: &str,
6412        row_id: &str,
6413        partial: Map<String, Value>,
6414        base_version: Option<i64>,
6415        staged: Option<&Option<Map<String, Value>>>,
6416    ) -> Result<OutboxOp, ClientError> {
6417        let schema_table = self
6418            .schema
6419            .table(table)
6420            .ok_or_else(|| format!("sync.unknown_table: unknown table {table:?}"))?;
6421        let partial = normalize_values_casing(schema_table, partial)?;
6422        if let Some(pk) = partial.get(&schema_table.primary_key) {
6423            if render_row_id_json(Some(pk)).ok().as_deref() != Some(row_id) {
6424                return Err(invalid_authoring(
6425                    "patch cannot change the primary key",
6426                    json!({ "table": table }),
6427                ));
6428            }
6429        }
6430        let exists = match staged {
6431            Some(row) => row.is_some(),
6432            None => self.visible_row_exists(schema_table, row_id)?,
6433        };
6434        if !exists {
6435            return Err("sync.row_missing: a sparse patch requires a local row"
6436                .to_owned()
6437                .into());
6438        }
6439        self.refuse_sealed_row(schema_table, row_id)?;
6440        // §3.4 rule 5 / §6.2: scope columns are immutable on update. The
6441        // server accepts a present scope column whose value equals the
6442        // stored row and applies it as a no-op, so a decoded envelope
6443        // round-trips. The local row is the only value the client can prove
6444        // equality against; when the row is absent locally the client cannot
6445        // prove equality against the server's stored row and fails closed.
6446        // A primary key that is also a scope column needs no stored-row
6447        // read: the primary key in a sparse payload is by construction the
6448        // row id being patched (§6.1), so its value is proven equal already.
6449        for scope in &schema_table.scope_variables {
6450            if !partial.contains_key(&scope.column) || scope.column == schema_table.primary_key {
6451                continue;
6452            }
6453            let Some(column) = schema_table
6454                .columns
6455                .iter()
6456                .find(|column| column.name == scope.column)
6457            else {
6458                continue;
6459            };
6460            let stored = match staged {
6461                Some(row) => row.as_ref().and_then(|row| {
6462                    json_to_column_value(column, row.get(&scope.column))
6463                        .ok()
6464                        .map(|value| column_value_to_json(&value))
6465                }),
6466                None => self
6467                    .conn
6468                    .query_row(
6469                        &format!(
6470                            "SELECT {} FROM {} WHERE {}",
6471                            quote_ident(&scope.column),
6472                            visible_table(&schema_table.name),
6473                            row_id_predicate(schema_table)
6474                        ),
6475                        rusqlite::params![row_id],
6476                        |row| Ok(sql_ref_to_json(column, row.get_ref(0)?)),
6477                    )
6478                    .optional()
6479                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
6480            };
6481            // Decode the supplied value through the same column-type path the
6482            // rest of the patch encoding uses, so a value that only equals
6483            // the stored one in another number representation still compares
6484            // equal.
6485            let incoming = json_to_column_value(column, partial.get(&scope.column))
6486                .ok()
6487                .map(|value| column_value_to_json(&value));
6488            // A stored blob never equals a patch value, matching the server's
6489            // scope-column comparison (§3.4): bytes are unstorable as scope
6490            // values.
6491            let equal = stored.as_ref().is_some_and(|stored| {
6492                stored.get("$bytes").is_none() && incoming.as_ref() == Some(stored)
6493            });
6494            if !equal {
6495                return Err(invalid_authoring(
6496                    "patch cannot write a scope column (§3.4)",
6497                    json!({ "table": table, "column": scope.column }),
6498                ));
6499            }
6500        }
6501        let mut values = partial;
6502        // Presence-set semantics: a proven-equal scope column is dropped, so
6503        // the patch leaves the stored value untouched (§6.2).
6504        for scope in &schema_table.scope_variables {
6505            values.remove(&scope.column);
6506        }
6507        let pk_value = match schema_table.columns[schema_table.pk_index].ty {
6508            // §2.4: the rowId of an integer key is its canonical decimal form.
6509            ColumnType::Integer => match row_id.parse::<i64>() {
6510                Ok(v)
6511                    if v.to_string() == row_id
6512                        && (-ssp2::primitives::I64_SAFE_MAX..=ssp2::primitives::I64_SAFE_MAX)
6513                            .contains(&v) =>
6514                {
6515                    Value::from(v)
6516                }
6517                _ => {
6518                    return Err(format!(
6519                    "sync.invalid_request: table {table:?}: invalid integer primary key {row_id:?}"
6520                )
6521                    .into())
6522                }
6523            },
6524            ColumnType::Boolean => match row_id {
6525                "true" => Value::Bool(true),
6526                "false" => Value::Bool(false),
6527                _ => {
6528                    return Err(format!(
6529                    "sync.invalid_request: table {table:?}: invalid boolean primary key {row_id:?}"
6530                )
6531                    .into())
6532                }
6533            },
6534            _ => Value::from(row_id.to_owned()),
6535        };
6536        values.insert(schema_table.primary_key.clone(), pk_value);
6537        // §6.1: a patch cannot null a non-nullable column.
6538        if let Some(column) = schema_table
6539            .columns
6540            .iter()
6541            .find(|column| !column.nullable && values.get(&column.name) == Some(&Value::Null))
6542        {
6543            return Err(invalid_authoring(
6544                "a non-nullable column is null (§6.1)",
6545                json!({ "table": table, "column": column.name }),
6546            ));
6547        }
6548        // §5.11: structural validation only; an absent key-id selector is
6549        // resolved at the push seam from the stored local row, and a
6550        // key-selection or unknown-key failure becomes a durable
6551        // `client.encrypt_failed` rejection there (§10.3), never an
6552        // author-time error.
6553        self.validate_author_encode(schema_table, row_id, &values)?;
6554        Ok(OutboxOp {
6555            upsert: true,
6556            table: table.to_owned(),
6557            row_id: row_id.to_owned(),
6558            base_version,
6559            values: Some(values),
6560            insert: false,
6561        })
6562    }
6563
6564    /// §5.11 key-id fallback: the stored local row's selector value as a
6565    /// positional row (only the selector slot filled). Returns None when no
6566    /// selector is configured, the row is locally absent, or the stored
6567    /// value is not a usable key id. A failed storage read is an error.
6568    fn stored_key_fallback(
6569        &self,
6570        table: &TableSchema,
6571        row_id: &str,
6572    ) -> Result<Option<Row>, String> {
6573        let Some(selector) = self.encryption.key_id_columns.get(&table.name) else {
6574            return Ok(None);
6575        };
6576        let Some(index) = table
6577            .columns
6578            .iter()
6579            .position(|column| &column.name == selector)
6580        else {
6581            return Ok(None);
6582        };
6583        let sql = format!(
6584            "SELECT {} FROM {} WHERE {}",
6585            quote_ident(selector),
6586            visible_table(&table.name),
6587            row_id_predicate(table)
6588        );
6589        let key = self
6590            .conn
6591            .query_row(&sql, rusqlite::params![row_id], |row| {
6592                row.get::<_, Option<String>>(0)
6593            })
6594            .optional()
6595            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
6596            .flatten()
6597            .filter(|key| !key.is_empty());
6598        let Some(key) = key else {
6599            return Ok(None);
6600        };
6601        let mut fallback: Row = vec![None; table.columns.len()];
6602        fallback[index] = Some(ColumnValue::String(key));
6603        Ok(Some(fallback))
6604    }
6605
6606    /// §5.11/§10.3 author-time encode check: a structural failure (a
6607    /// non-nullable NULL, a value the codec rejects) is an author-time
6608    /// error; a key-selection or unknown-key failure is deferred to the push
6609    /// seam, where it becomes a durable `client.encrypt_failed` rejection.
6610    fn validate_author_encode(
6611        &self,
6612        table: &TableSchema,
6613        row_id: &str,
6614        values: &Map<String, Value>,
6615    ) -> Result<(), String> {
6616        match encode_sparse_row_json(table, row_id, values, &self.encryption, None) {
6617            Err(error) if error.starts_with(ENCRYPT_FAILED_CODE) => Ok(()),
6618            result => result.map(|_| ()),
6619        }
6620    }
6621
6622    /// §5.11/§10.3: drop every pending commit whose push encode fails with
6623    /// `client.encrypt_failed` (an unresolvable key id or an unknown key) and
6624    /// raise one durable rejection per commit. `sync()` never aborts for an
6625    /// encode failure. Mirrors §7.4.4's `drop_incompatible_outbox`, including
6626    /// undoing the dropped commit's purely-optimistic rows.
6627    /// §7.1/§7.4.4: drop pending commits whose values the current codec refuses
6628    /// (an envelope or a type an earlier version accepted). `with_encryption`
6629    /// additionally drops commits whose §5.11 encryption cannot resolve a key;
6630    /// the startup boundary leaves those to the send seam, where hosts configure
6631    /// keys after open.
6632    /// Recovery runs as one observation. An enclosing observation (the guarded
6633    /// schema reset) passes its `batch`, so the rejection, outcome, and table
6634    /// domains join that change event instead of publishing a second one.
6635    fn drop_unencodable_outbox(
6636        &mut self,
6637        with_encryption: bool,
6638        batch: Option<&mut ChangeAccumulator>,
6639    ) -> Result<bool, String> {
6640        let mut failures = Vec::new();
6641        for commit in self.outbox.iter() {
6642            for op in &commit.ops {
6643                let Some(table) = self.schema.table(&op.table).filter(|_| op.upsert) else {
6644                    continue;
6645                };
6646                // An insert lacking a non-nullable column is left to
6647                // `drop_incompatible_outbox` (§7.4.4).
6648                let Some(values) = op.presented_values(table) else {
6649                    continue;
6650                };
6651                let fallback = self.stored_key_fallback(table, &op.row_id)?;
6652                let failure = match encode_sparse_row_json(
6653                    table,
6654                    &op.row_id,
6655                    &values,
6656                    &self.encryption,
6657                    fallback.as_ref(),
6658                ) {
6659                    Err(error) if error.starts_with(ENCRYPT_FAILED_CODE) => {
6660                        with_encryption.then_some((error, ENCRYPT_FAILED_CODE))
6661                    }
6662                    // §7.1/§7.4.4: a value the current codec refuses was
6663                    // persisted by a version that accepted it; it leaves
6664                    // through the incompatible-outbox path, so the FIFO
6665                    // queue never wedges.
6666                    Err(error) if error.starts_with(INVALID_REQUEST_CODE) => Some((
6667                        LEGACY_VALUE_REFUSAL_MESSAGE.to_owned(),
6668                        OUTBOX_INCOMPATIBLE_CODE,
6669                    )),
6670                    _ => None,
6671                };
6672                if let Some((error, code)) = failure {
6673                    failures.push((commit.clone(), error, code));
6674                    break;
6675                }
6676            }
6677        }
6678        if failures.is_empty() {
6679            return Ok(false);
6680        }
6681        self.reject_outbox_commits(
6682            failures
6683                .into_iter()
6684                .map(|(commit, message, code)| {
6685                    let details = (code == OUTBOX_INCOMPATIBLE_CODE).then(|| RejectionDetails {
6686                        field_paths: None,
6687                        reason: Some(INVALID_STORED_VALUES_REASON.to_owned()),
6688                        required_action: None,
6689                        references: None,
6690                    });
6691                    (commit, message, code, details)
6692                })
6693                .collect(),
6694            batch,
6695        )?;
6696        Ok(true)
6697    }
6698
6699    /// §7.4.4 client-local terminal drop of pending commits, each journaled as a
6700    /// rejection with `code` and `details`. Every code except
6701    /// `sync.outbox_incompatible` keeps the retained intent.
6702    #[allow(clippy::type_complexity)]
6703    fn reject_outbox_commits(
6704        &mut self,
6705        failures: Vec<(
6706            Arc<OutboxCommit>,
6707            String,
6708            &'static str,
6709            Option<RejectionDetails>,
6710        )>,
6711        batch: Option<&mut ChangeAccumulator>,
6712    ) -> Result<(), String> {
6713        // §7.4.4: one observation covers the durable outcome writes, the outbox
6714        // removal, the in-memory drop, and the overlay rebuild, so a storage fault
6715        // leaves the replica and the client unchanged, a successful recovery
6716        // publishes one batch with the rejection, outcome, and table domains, and
6717        // an ordinary open with nothing to recover publishes nothing.
6718        let rollback = self.capture();
6719        let owns_observation = batch.is_none();
6720        let owns_transaction = self.conn.is_autocommit();
6721        let mut domains = ChangeAccumulator::default();
6722        if owns_observation {
6723            self.begin_observation("syncular_value_recovery")?;
6724        }
6725        let dropped = (|| -> Result<(), String> {
6726            for (commit, message, code, details) in failures {
6727                if self.retain_failed_commits && code != OUTBOX_INCOMPATIBLE_CODE {
6728                    self.retain_failed_commit(&commit.client_commit_id, &commit.ops)?;
6729                }
6730                let representable = commit.ops.iter().all(Self::outbox_op_is_journalable);
6731                let rejection = RejectionRecord {
6732                    client_commit_id: commit.client_commit_id.clone(),
6733                    op_index: 0,
6734                    code: code.to_owned(),
6735                    message,
6736                    retryable: false,
6737                    details,
6738                    operation: representable
6739                        .then(|| commit.ops.first())
6740                        .flatten()
6741                        .map(CommitOperation::from),
6742                };
6743                self.persist_commit_outcome(
6744                    &commit.client_commit_id,
6745                    CommitOutcomeStatus::Rejected,
6746                    &[CommitOperationOutcome::Error {
6747                        rejection: rejection.clone(),
6748                    }],
6749                    representable.then_some(commit.ops.as_slice()),
6750                )?;
6751                self.delete_outbox_persisted(&commit.client_commit_id)?;
6752                self.outbox
6753                    .retain(|candidate| candidate.client_commit_id != commit.client_commit_id);
6754                self.rejections.push(rejection);
6755                for operation in &commit.ops {
6756                    domains.table(operation.table.as_str());
6757                }
6758            }
6759            domains.status = true;
6760            domains.rejections = true;
6761            domains.outcomes = true;
6762            self.prune_commit_outcomes()?;
6763            // The visible projection is re-derived from the surviving base + outbox.
6764            self.overlay_dirty.set(true);
6765            self.rebuild_overlay()?;
6766            Ok(())
6767        })();
6768        let result = match batch {
6769            Some(batch) => {
6770                if dropped.is_ok() {
6771                    for table in domains.tables.keys() {
6772                        batch.table(table.as_str());
6773                    }
6774                    batch.status |= domains.status;
6775                    batch.rejections |= domains.rejections;
6776                    batch.outcomes |= domains.outcomes;
6777                }
6778                dropped
6779            }
6780            None => {
6781                dropped.and_then(|()| self.finish_observation("syncular_value_recovery", domains))
6782            }
6783        };
6784        if let Err(error) = result {
6785            if owns_observation {
6786                if owns_transaction {
6787                    if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
6788                        let mut failure = self.storage_failure.borrow_mut();
6789                        let failure =
6790                            failure.get_or_insert_with(|| QueryReadFailure::from(error.clone()));
6791                        failure.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
6792                    }
6793                } else {
6794                    self.rollback_observation("syncular_value_recovery");
6795                }
6796            }
6797            self.restore(rollback);
6798            return Err(error);
6799        }
6800        Ok(())
6801    }
6802
6803    pub fn pending_commit_ids(&self) -> Vec<String> {
6804        self.outbox
6805            .iter()
6806            .map(|c| c.client_commit_id.clone())
6807            .collect()
6808    }
6809
6810    /// §7.1: every pending commit in FIFO order, as the TypeScript client's
6811    /// `pendingCommits()` returns it.
6812    pub fn pending_commits(&self) -> Result<Vec<PendingCommit>, String> {
6813        let mut stmt = self
6814            .conn
6815            .prepare_cached(
6816                "SELECT seq, commit_id, created_at_ms, ops_json FROM _syncular_outbox ORDER BY seq",
6817            )
6818            .map_err(|error| error.to_string())?;
6819        let rows = stmt
6820            .query_map([], |row| {
6821                Ok((
6822                    row.get(0)?,
6823                    row.get(1)?,
6824                    row.get(2)?,
6825                    row.get::<_, String>(3)?,
6826                ))
6827            })
6828            .map_err(|error| error.to_string())?;
6829        rows.map(|row| {
6830            let (seq, client_commit_id, created_at_ms, raw) =
6831                row.map_err(|error| error.to_string())?;
6832            let operations = serde_json::from_str(&raw).map_err(|error| {
6833                format!("invalid persisted outbox {client_commit_id:?}: {error}")
6834            })?;
6835            Ok(PendingCommit {
6836                seq,
6837                client_commit_id,
6838                created_at_ms,
6839                operations,
6840            })
6841        })
6842        .collect()
6843    }
6844
6845    /// §6.1: the sparse push payload bytes of every pending upsert operation,
6846    /// FIFO — exactly what the next push round would put on the wire. Used by
6847    /// conformance B.21(h) to pin cross-core byte identity.
6848    pub fn pending_payloads(&self) -> Result<Vec<Vec<u8>>, String> {
6849        let mut payloads = Vec::new();
6850        for commit in self.outbox.iter() {
6851            for op in &commit.ops {
6852                let Some(table) = self.schema.table(&op.table).filter(|_| op.upsert) else {
6853                    continue;
6854                };
6855                let Some(values) = op.presented_values(table) else {
6856                    continue;
6857                };
6858                let fallback = self.stored_key_fallback(table, &op.row_id)?;
6859                if let Ok(payload) = encode_sparse_row_json(
6860                    table,
6861                    &op.row_id,
6862                    &values,
6863                    &self.encryption,
6864                    fallback.as_ref(),
6865                ) {
6866                    payloads.push(payload);
6867                }
6868            }
6869        }
6870        Ok(payloads)
6871    }
6872
6873    pub fn conflicts(&self) -> &[ConflictRecord] {
6874        &self.conflicts
6875    }
6876
6877    pub fn rejections(&self) -> &[RejectionRecord] {
6878        &self.rejections
6879    }
6880
6881    pub fn resolve_commit_outcome(
6882        &mut self,
6883        input: ResolveCommitOutcomeInput,
6884    ) -> Result<CommitOutcome, String> {
6885        let current = self
6886            .commit_outcome(&input.client_commit_id)?
6887            .ok_or_else(|| {
6888                format!(
6889                    "sync.outcome_not_found: no durable outcome exists for {:?}",
6890                    input.client_commit_id
6891                )
6892            })?;
6893        if current.resolution != CommitOutcomeResolution::Active {
6894            return Ok(current);
6895        }
6896        if current.replacement_client_commit_id.is_some() {
6897            return Err(REPLACEMENT_PENDING.to_owned());
6898        }
6899        if input.resolution == CommitOutcomeResolution::Active {
6900            return Err("sync.invalid_request: resolution must leave active state".to_owned());
6901        }
6902        match input.resolution {
6903            CommitOutcomeResolution::Superseded => {
6904                let replacement = input
6905                    .replacement_client_commit_id
6906                    .as_deref()
6907                    .filter(|value| !value.is_empty() && *value != input.client_commit_id)
6908                    .ok_or_else(|| {
6909                        "sync.invalid_request: superseded outcomes require a distinct replacementClientCommitId"
6910                            .to_owned()
6911                    })?;
6912                let _ = replacement;
6913            }
6914            _ if input.replacement_client_commit_id.is_some() => {
6915                return Err(
6916                    "sync.invalid_request: replacementClientCommitId is valid only for superseded outcomes"
6917                        .to_owned(),
6918                );
6919            }
6920            _ => {}
6921        }
6922        let allowed = match current.status {
6923            CommitOutcomeStatus::Conflict => matches!(
6924                input.resolution,
6925                CommitOutcomeResolution::ResolvedKeepServer | CommitOutcomeResolution::Superseded
6926            ),
6927            CommitOutcomeStatus::Rejected => {
6928                matches!(
6929                    input.resolution,
6930                    CommitOutcomeResolution::Superseded
6931                        | CommitOutcomeResolution::ResolvedKeepServer
6932                )
6933            }
6934            CommitOutcomeStatus::Applied | CommitOutcomeStatus::Cached => {
6935                input.resolution == CommitOutcomeResolution::Dismissed
6936            }
6937        };
6938        if !allowed {
6939            return Err(format!(
6940                "sync.invalid_request: resolution {:?} is invalid for {:?} outcome",
6941                input.resolution, current.status
6942            ));
6943        }
6944
6945        let rollback = self.capture();
6946        self.begin_observation("syncular_outcome_resolution")?;
6947        let mut batch = ChangeAccumulator::default();
6948        let result = self
6949            .release_outcome(
6950                &current,
6951                input.resolution,
6952                input.replacement_client_commit_id.as_deref(),
6953                &mut batch,
6954            )
6955            .and_then(|()| self.rebuild_overlay_if_dirty())
6956            .and_then(|()| self.finish_observation("syncular_outcome_resolution", batch));
6957        if let Err(error) = result {
6958            self.rollback_observation("syncular_outcome_resolution");
6959            self.restore(rollback);
6960            return Err(error);
6961        }
6962        self.commit_outcome(&current.client_commit_id)?
6963            .ok_or_else(|| "sync.outcome_not_found: outcome disappeared".to_owned())
6964    }
6965
6966    /// §7.2.2: record a decision against an active conflict or rejection as a
6967    /// held replacement commit and return its client commit id. Without
6968    /// `replacement` the retained envelope is kept ("keep mine"); every
6969    /// operation on a retained row takes that row's base version.
6970    pub fn replace_commit_outcome(
6971        &mut self,
6972        client_commit_id: &str,
6973        sequence: i64,
6974        replacement: Option<Vec<Mutation>>,
6975    ) -> Result<String, ClientError> {
6976        let prior = self.storage_failure.borrow_mut().take();
6977        let outcome = (|| -> Result<String, ClientError> {
6978            let Some(current) = self.commit_outcome(client_commit_id)? else {
6979                return Err(ClientError {
6980                    code: "sync.outcome_not_found".to_owned(),
6981                    message: "no durable outcome exists for the commit".to_owned(),
6982                    details: Some(json!({ "clientCommitId": client_commit_id })),
6983                    retryable: false,
6984                });
6985            };
6986            if current.resolution == CommitOutcomeResolution::Active
6987                && current.replacement_client_commit_id.is_some()
6988            {
6989                return Err(ClientError {
6990                    code: "client.replacement_pending".to_owned(),
6991                    message: "the outcome has a pending replacement".to_owned(),
6992                    details: Some(json!({ "clientCommitId": client_commit_id })),
6993                    retryable: false,
6994                });
6995            }
6996            let (Some(envelope), Some(rows)) = (current.operations, current.retained_rows) else {
6997                return Err(invalid_authoring(
6998                    "the outcome cannot be replaced",
6999                    json!({ "clientCommitId": client_commit_id }),
7000                ));
7001            };
7002            if current.sequence != sequence
7003                || current.resolution != CommitOutcomeResolution::Active
7004                || !matches!(
7005                    current.status,
7006                    CommitOutcomeStatus::Conflict | CommitOutcomeStatus::Rejected
7007                )
7008                || rows.is_empty()
7009                || envelope.len() != rows.len()
7010            {
7011                return Err(invalid_authoring(
7012                    "the outcome cannot be replaced",
7013                    json!({ "clientCommitId": client_commit_id }),
7014                ));
7015            }
7016            let ops = match replacement {
7017                None => envelope
7018                    .into_iter()
7019                    .zip(&rows)
7020                    .map(|(operation, row)| OutboxOp {
7021                        base_version: Some(row.server_version.unwrap_or(0)),
7022                        ..OutboxOp::from(operation)
7023                    })
7024                    .collect(),
7025                Some(mutations) => {
7026                    let mut ops = self.prepare_mutations(mutations)?;
7027                    for op in &mut ops {
7028                        let Some(row) = rows
7029                            .iter()
7030                            .find(|row| row.table == op.table && row.row_id == op.row_id)
7031                        else {
7032                            continue;
7033                        };
7034                        if op.base_version.is_some() {
7035                            return Err(invalid_authoring(
7036                                "an operation on a retained row takes its base version",
7037                                json!({ "table": op.table }),
7038                            ));
7039                        }
7040                        op.base_version = Some(row.server_version.unwrap_or(0));
7041                    }
7042                    ops
7043                }
7044            };
7045            Ok(self.record_outbox_commit(ops, Some(client_commit_id))?)
7046        })();
7047        self.finish_authoring(prior, outcome)
7048    }
7049
7050    /// Internal host step at the start of an idle drive (§7.2.2): capture
7051    /// every pending decision with the units and epochs of this drive.
7052    #[doc(hidden)]
7053    pub fn prepare_replacements(&self) -> Result<Vec<CurrentBasePreparation>, String> {
7054        let mut stmt = self
7055            .conn
7056            .prepare_cached(
7057                "SELECT client_commit_id, seq, replacement_client_commit_id
7058                   FROM _syncular_commit_outcomes
7059                  WHERE resolution = 'active' AND replacement_client_commit_id IS NOT NULL
7060                  ORDER BY seq ASC",
7061            )
7062            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
7063        let decisions = stmt
7064            .query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))
7065            .and_then(|rows| rows.collect::<Result<Vec<(String, i64, String)>, _>>())
7066            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
7067        if decisions.is_empty() {
7068            return Ok(Vec::new());
7069        }
7070        let units = self.stored(load_units(&self.conn))?;
7071        Ok(decisions
7072            .into_iter()
7073            .map(
7074                |(client_commit_id, sequence, replacement)| CurrentBasePreparation {
7075                    client_commit_id,
7076                    sequence,
7077                    replacement: Some(replacement),
7078                    units: units.clone(),
7079                    transport_epoch: self.transport_epoch,
7080                    registration_epoch: self.registration_epoch,
7081                    security_epoch: self.security_epoch,
7082                },
7083            )
7084            .collect())
7085    }
7086
7087    /// Internal host step when an idle drive reaches idle (§7.2.2): write,
7088    /// refuse, or keep each decision captured at the drive's start. Returns
7089    /// whether a decision was written, so the drive runs another round.
7090    #[doc(hidden)]
7091    pub fn settle_replacements(
7092        &mut self,
7093        decisions: &[CurrentBasePreparation],
7094        drive: &SyncOutcome,
7095    ) -> Result<bool, String> {
7096        if !self.transport_enabled {
7097            return Ok(false);
7098        }
7099        let mut written = false;
7100        for decision in decisions {
7101            // A registration change during the drive says nothing about the
7102            // base; the decision waits for the next drive.
7103            if decision.registration_epoch != self.registration_epoch {
7104                continue;
7105            }
7106            let Some(replacement) = decision.replacement.as_deref() else {
7107                continue;
7108            };
7109            let Some(current) =
7110                self.commit_outcome(&decision.client_commit_id)?
7111                    .filter(|current| {
7112                        current.resolution == CommitOutcomeResolution::Active
7113                            && current.replacement_client_commit_id.as_deref() == Some(replacement)
7114                    })
7115            else {
7116                continue;
7117            };
7118            let held = self
7119                .outbox
7120                .iter()
7121                .find(|commit| commit.client_commit_id == replacement)
7122                .cloned();
7123            let refused = match &held {
7124                // A revocation drop (§3.3) or a crash already ended the
7125                // replacement; its own journal entry records what happened.
7126                None => None,
7127                Some(commit) => match self.finish_current_base(decision.clone(), drive.clone()) {
7128                    CurrentBaseProof {
7129                        rows: Some(rows), ..
7130                    } => {
7131                        let proven = commit.ops.iter().all(|op| {
7132                            rows.iter()
7133                                .find(|row| row.table == op.table && row.row_id == op.row_id)
7134                                .is_none_or(|row| {
7135                                    op.base_version == Some(row.server_version.unwrap_or(0))
7136                                })
7137                        });
7138                        written |= proven;
7139                        (!proven).then(|| "base_moved".to_owned())
7140                    }
7141                    CurrentBaseProof {
7142                        reason:
7143                            Some(
7144                                reason @ (CurrentBaseProofReason::SchemaFloor
7145                                | CurrentBaseProofReason::MissingCoverage
7146                                | CurrentBaseProofReason::Revoked
7147                                | CurrentBaseProofReason::OutcomeUnavailable),
7148                            ),
7149                        ..
7150                    } => Some(
7151                        serde_json::to_value(reason)
7152                            .ok()
7153                            .and_then(|reason| reason.as_str().map(str::to_owned))
7154                            .ok_or("sync.local_corrupt")?,
7155                    ),
7156                    _ => continue,
7157                },
7158            };
7159            // One transaction: the original outcome is superseded first, so
7160            // the refused replacement retains its intent over the restored
7161            // server base.
7162            let rollback = self.capture();
7163            self.begin_observation("syncular_replacement")?;
7164            let mut batch = ChangeAccumulator::default();
7165            let result = self
7166                .release_outcome(
7167                    &current,
7168                    CommitOutcomeResolution::Superseded,
7169                    Some(replacement),
7170                    &mut batch,
7171                )
7172                .and_then(|()| self.rebuild_overlay_if_dirty())
7173                .and_then(|()| match (refused, held) {
7174                    (Some(reason), Some(commit)) => self.reject_outbox_commits(
7175                        vec![(
7176                            commit,
7177                            "the replacement could not be written on a proven base".to_owned(),
7178                            REPLACEMENT_REFUSED_CODE,
7179                            Some(RejectionDetails {
7180                                field_paths: None,
7181                                reason: Some(reason),
7182                                required_action: None,
7183                                references: None,
7184                            }),
7185                        )],
7186                        Some(&mut batch),
7187                    ),
7188                    _ => Ok(()),
7189                })
7190                .and_then(|()| self.finish_observation("syncular_replacement", batch));
7191            if let Err(error) = result {
7192                self.rollback_observation("syncular_replacement");
7193                self.restore(rollback);
7194                return Err(error);
7195            }
7196        }
7197        if written {
7198            self.set_sync_needed(true, true);
7199        }
7200        Ok(written)
7201    }
7202
7203    /// §7.2.1 explicit resolution inside the caller's observation: record the
7204    /// resolution, release the retained aggregate and mark its tables for the
7205    /// overlay rebuild, which restores the server base and replays what
7206    /// remains over it.
7207    fn release_outcome(
7208        &mut self,
7209        current: &CommitOutcome,
7210        resolution: CommitOutcomeResolution,
7211        replacement: Option<&str>,
7212        batch: &mut ChangeAccumulator,
7213    ) -> Result<(), String> {
7214        self.conn
7215            .execute(
7216                "UPDATE _syncular_commit_outcomes
7217                    SET resolution = ?1, resolved_at_ms = ?2,
7218                        replacement_client_commit_id = ?3
7219                  WHERE client_commit_id = ?4 AND resolution = 'active'",
7220                rusqlite::params![
7221                    Self::outcome_resolution_name(resolution),
7222                    self.clock_now_ms(),
7223                    replacement,
7224                    current.client_commit_id
7225                ],
7226            )
7227            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
7228        self.conn
7229            .execute(
7230                "DELETE FROM _syncular_failed_commits WHERE client_commit_id = ?1",
7231                rusqlite::params![current.client_commit_id],
7232            )
7233            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
7234        for (commit, _) in self.failed_commits.iter() {
7235            if commit.client_commit_id == current.client_commit_id {
7236                for operation in &commit.ops {
7237                    self.overlay_dirty.table(&operation.table);
7238                }
7239            }
7240        }
7241        self.failed_commits
7242            .retain(|(commit, _)| commit.client_commit_id != current.client_commit_id);
7243        self.prune_commit_outcomes()?;
7244        self.conflicts
7245            .retain(|record| record.client_commit_id != current.client_commit_id);
7246        self.rejections
7247            .retain(|record| record.client_commit_id != current.client_commit_id);
7248        batch.conflicts |= current.status == CommitOutcomeStatus::Conflict;
7249        batch.rejections |= current.status == CommitOutcomeStatus::Rejected;
7250        batch.outcomes = true;
7251        for operation in current.operations.iter().flatten() {
7252            batch.table(&operation.table);
7253        }
7254        Ok(())
7255    }
7256
7257    pub fn schema_floor(&self) -> Option<&SchemaFloor> {
7258        self.schema_floor.as_ref()
7259    }
7260
7261    /// §7.3.5: the client's opaque auth-lease state, if any.
7262    pub fn lease_state(&self) -> Option<&LeaseState> {
7263        self.lease_state.as_ref()
7264    }
7265
7266    /// §7.3.5: record a request-level lease error (stop-and-surface). Only
7267    /// the two lease codes set it; other errors leave leaseState untouched.
7268    fn record_lease_error(&mut self, code: &str) -> Result<(), String> {
7269        if code != "sync.auth_lease_required" && code != "sync.auth_lease_revoked" {
7270            return Ok(());
7271        }
7272        let mut next = self.lease_state.clone().unwrap_or_default();
7273        next.error_code = Some(code.to_owned());
7274        self.set_lease_state(Some(next))
7275    }
7276
7277    fn set_lease_state(&mut self, next: Option<LeaseState>) -> Result<(), String> {
7278        if self.lease_state == next {
7279            return Ok(());
7280        }
7281        self.begin_observation("syncular_lease")?;
7282        let rollback = self.capture();
7283        self.lease_state = next;
7284        let persisted = match &self.lease_state {
7285            Some(lease) => serde_json::to_string(lease)
7286                .map_err(|error| error.to_string())
7287                .and_then(|json| self.set_meta(LEASE_STATE_KEY, &json)),
7288            None => self.delete_meta(LEASE_STATE_KEY),
7289        };
7290        let batch = ChangeAccumulator {
7291            status: true,
7292            ..ChangeAccumulator::default()
7293        };
7294        if let Err(error) =
7295            persisted.and_then(|()| self.finish_observation("syncular_lease", batch))
7296        {
7297            self.rollback_observation("syncular_lease");
7298            self.restore(rollback);
7299            return Err(error);
7300        }
7301        Ok(())
7302    }
7303
7304    fn set_schema_floor(&mut self, next: Option<SchemaFloor>) -> Result<(), String> {
7305        if self.schema_floor == next {
7306            return Ok(());
7307        }
7308        self.begin_observation("syncular_schema_floor")?;
7309        let rollback = self.capture();
7310        self.stopped = next.is_some();
7311        self.schema_floor = next;
7312        let persisted = match &self.schema_floor {
7313            Some(floor) => serde_json::to_string(floor)
7314                .map_err(|error| error.to_string())
7315                .and_then(|json| self.set_meta(SCHEMA_FLOOR_KEY, &json)),
7316            None => self.delete_meta(SCHEMA_FLOOR_KEY),
7317        };
7318        let batch = ChangeAccumulator {
7319            status: true,
7320            ..ChangeAccumulator::default()
7321        };
7322        if let Err(error) =
7323            persisted.and_then(|()| self.finish_observation("syncular_schema_floor", batch))
7324        {
7325            self.rollback_observation("syncular_schema_floor");
7326            self.restore(rollback);
7327            return Err(error);
7328        }
7329        Ok(())
7330    }
7331
7332    fn set_upgrading(&mut self, value: bool) {
7333        if self.upgrading == value {
7334            return;
7335        }
7336        if self.begin_observation("syncular_upgrading").is_err() {
7337            return;
7338        }
7339        self.upgrading = value;
7340        self.upgrade_kept_tables &= value;
7341        let batch = ChangeAccumulator {
7342            status: true,
7343            ..ChangeAccumulator::default()
7344        };
7345        if self
7346            .finish_observation("syncular_upgrading", batch)
7347            .is_err()
7348        {
7349            self.rollback_observation("syncular_upgrading");
7350        }
7351    }
7352
7353    pub fn sync_needed(&self) -> bool {
7354        self.sync_needed
7355    }
7356
7357    pub fn read_rows(&self, table: &str) -> Result<Vec<RowState>, String> {
7358        let schema_table = self
7359            .schema
7360            .table(table)
7361            .ok_or_else(|| format!("unknown table {table:?}"))?;
7362        let sql = format!(
7363            "SELECT * FROM {} ORDER BY {} ASC",
7364            visible_table(table),
7365            quote_ident(&schema_table.primary_key)
7366        );
7367        let mut stmt = self
7368            .conn
7369            .prepare(&sql)
7370            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
7371        let mut rows = stmt
7372            .query([])
7373            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
7374        let mut out = Vec::new();
7375        while let Some(row) = rows
7376            .next()
7377            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
7378        {
7379            let mut values = Map::new();
7380            for (i, column) in schema_table.columns.iter().enumerate() {
7381                let value = row
7382                    .get_ref(i)
7383                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
7384                values.insert(column.name.clone(), sql_ref_to_json(column, value));
7385            }
7386            let version: i64 = row
7387                .get(schema_table.columns.len())
7388                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
7389            let row_id = match values.get(&schema_table.primary_key) {
7390                Some(Value::String(s)) => s.clone(),
7391                Some(Value::Number(n)) => n.to_string(),
7392                Some(Value::Bool(b)) => b.to_string(),
7393                other => format!("{}", other.cloned().unwrap_or(Value::Null)),
7394            };
7395            let sealed_key = if schema_table.has_encrypted_columns() {
7396                row.get(schema_table.columns.len() + 1)
7397                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
7398            } else {
7399                None
7400            };
7401            out.push(RowState {
7402                row_id,
7403                version,
7404                values,
7405                sealed_key,
7406            });
7407        }
7408        Ok(out)
7409    }
7410
7411    // -- §5.10.5 native CRDT (the `crdt-yjs` feature) --------------------------
7412    //
7413    // The Rust face of the §5.10.4 client model: a local crdt edit loads the
7414    // current stored (server-merged ⊕ pending-overlay) column bytes, applies
7415    // the op with `yrs`, re-encodes the whole doc state, and records a
7416    // baseVersion-less sparse upsert carrying only the primary key and the
7417    // crdt column (§5.10.3, §6.6). No local merge — merging is server-side;
7418    // the overlay re-materializes the edit immediately (optimistic apply,
7419    // §7.1) and the server-merged bytes arrive on the next pull, idempotently.
7420    // Byte-compatible with `@syncular/crdt-yjs`.
7421
7422    /// The current stored value of a `crdt` column for one row — the visible
7423    /// (optimistic) bytes, or `None` when the row is absent or the column is
7424    /// NULL (the empty document, §5.10.1). Errors if the column is not a
7425    /// `crdt` column (guards the app against a typo'd column name).
7426    #[cfg(feature = "crdt-yjs")]
7427    fn crdt_column_bytes(
7428        &self,
7429        table: &str,
7430        row_id: &str,
7431        column: &str,
7432    ) -> Result<Option<Vec<u8>>, String> {
7433        let schema_table = self
7434            .schema
7435            .table(table)
7436            .ok_or_else(|| format!("unknown table {table:?}"))?;
7437        let col = schema_table
7438            .columns
7439            .iter()
7440            .find(|c| c.name == column)
7441            .ok_or_else(|| format!("table {table:?} has no column {column:?}"))?;
7442        if col.ty != ColumnType::Crdt {
7443            return Err(format!("column {column:?} is not a crdt column (§5.10.1)"));
7444        }
7445        let sql = format!(
7446            "SELECT {} FROM {} WHERE {}",
7447            quote_ident(column),
7448            visible_table(table),
7449            row_id_predicate(schema_table)
7450        );
7451        let bytes: Option<Vec<u8>> = self
7452            .conn
7453            .prepare_cached(&sql)
7454            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
7455            .query_row(rusqlite::params![row_id], |row| {
7456                row.get::<_, Option<Vec<u8>>>(0)
7457            })
7458            .map_err(|e| match e {
7459                rusqlite::Error::QueryReturnedNoRows => "no such row".to_owned(),
7460                other => Self::sqlite_failure(&self.storage_failure, other),
7461            })?;
7462        Ok(bytes)
7463    }
7464
7465    /// §5.10.4 materialize: the collaborative text of a `crdt` column, decoded
7466    /// from the stored bytes with `yrs` — `YjsColumn.text(name).toString()`.
7467    /// An absent row / NULL column is the empty document (empty string).
7468    #[cfg(feature = "crdt-yjs")]
7469    pub fn crdt_text(
7470        &self,
7471        table: &str,
7472        row_id: &str,
7473        column: &str,
7474        name: &str,
7475    ) -> Result<String, String> {
7476        let bytes = self
7477            .crdt_column_bytes(table, row_id, column)?
7478            .unwrap_or_default();
7479        crate::crdt::text(&bytes, name)
7480    }
7481
7482    /// §5.10.4 push-an-update: apply a text insert to a `crdt` column and
7483    /// record the resulting crdt-only sparse operation. Returns the enqueued
7484    /// `clientCommitId`.
7485    #[cfg(feature = "crdt-yjs")]
7486    pub fn crdt_insert_text(
7487        &mut self,
7488        table: &str,
7489        row_id: &str,
7490        column: &str,
7491        name: &str,
7492        index: u32,
7493        value: &str,
7494    ) -> Result<String, ClientError> {
7495        let prior = self.storage_failure.borrow_mut().take();
7496        let outcome = (|| -> Result<String, ClientError> {
7497            let current = self
7498                .crdt_column_bytes(table, row_id, column)?
7499                .unwrap_or_default();
7500            let update = crate::crdt::insert_text(&current, name, index, value)?;
7501            self.perform_crdt_push_update(table, row_id, column, &update)
7502        })();
7503        self.finish_authoring(prior, outcome)
7504    }
7505
7506    /// §5.10.4 push-an-update: apply a text delete to a `crdt` column and
7507    /// record the resulting crdt-only sparse operation. Returns the enqueued
7508    /// `clientCommitId`.
7509    #[cfg(feature = "crdt-yjs")]
7510    pub fn crdt_delete_text(
7511        &mut self,
7512        table: &str,
7513        row_id: &str,
7514        column: &str,
7515        name: &str,
7516        index: u32,
7517        len: u32,
7518    ) -> Result<String, ClientError> {
7519        let prior = self.storage_failure.borrow_mut().take();
7520        let outcome = (|| -> Result<String, ClientError> {
7521            let current = self
7522                .crdt_column_bytes(table, row_id, column)?
7523                .unwrap_or_default();
7524            let update = crate::crdt::delete_text(&current, name, index, len)?;
7525            self.perform_crdt_push_update(table, row_id, column, &update)
7526        })();
7527        self.finish_authoring(prior, outcome)
7528    }
7529
7530    /// §5.10.4 generic escape hatch: apply an arbitrary Yjs update onto a
7531    /// `crdt` column's current state and record the resulting crdt-only sparse
7532    /// operation. The app authored the update with its own `yrs` model.
7533    /// Returns the enqueued `clientCommitId`.
7534    #[cfg(feature = "crdt-yjs")]
7535    pub fn crdt_apply_update(
7536        &mut self,
7537        table: &str,
7538        row_id: &str,
7539        column: &str,
7540        update: &[u8],
7541    ) -> Result<String, ClientError> {
7542        let prior = self.storage_failure.borrow_mut().take();
7543        let outcome = (|| -> Result<String, ClientError> {
7544            let current = self
7545                .crdt_column_bytes(table, row_id, column)?
7546                .unwrap_or_default();
7547            let next = crate::crdt::apply_update(&current, update)?;
7548            self.perform_crdt_push_update(table, row_id, column, &next)
7549        })();
7550        self.finish_authoring(prior, outcome)
7551    }
7552
7553    /// Shared tail of the crdt edit methods: record a §6.1 sparse crdt-only
7554    /// upsert — the primary key plus the crdt column, no `baseVersion` — and
7555    /// enqueue it. The server merges the crdt column and leaves every other
7556    /// column untouched (§5.10.3), so a baseVersion-less crdt edit cannot
7557    /// clobber a concurrent edit to another column. A locally absent row
7558    /// records the same partial operation; the server answers per §6.2.
7559    #[cfg(feature = "crdt-yjs")]
7560    fn perform_crdt_push_update(
7561        &mut self,
7562        table: &str,
7563        row_id: &str,
7564        column: &str,
7565        crdt_bytes: &[u8],
7566    ) -> Result<String, ClientError> {
7567        let mut bytes_obj = Map::new();
7568        bytes_obj.insert("$bytes".to_owned(), Value::from(bytes_to_hex(crdt_bytes)));
7569        let mut partial = Map::new();
7570        partial.insert(column.to_owned(), Value::Object(bytes_obj));
7571        self.perform_patch(table, row_id, partial, None)
7572    }
7573
7574    /// Run an arbitrary read-only SQL query against the local database and
7575    /// return each row as a `column-name → JSON value` map. This is the seam
7576    /// the React `useSyncQuery` live-query API needs (it takes app-authored
7577    /// SQL over the visible tables/views, not a fixed table read like
7578    /// [`read_rows`]).
7579    ///
7580    /// Bound `params` are the driver value forms: JSON strings/numbers/bools/
7581    /// null bind directly; a `{"$bytes": hex}` object binds as a BLOB — the
7582    /// same envelope the command surface uses everywhere else. Output BLOB
7583    /// columns come back as `{"$bytes": hex}` to round-trip cleanly.
7584    ///
7585    /// The result column typing is dynamic (SQLite's stored affinity), because
7586    /// arbitrary SQL can alias, join, and compute — there is no schema column
7587    /// to consult per output cell, unlike [`read_rows`].
7588    pub fn query(&self, sql: &str, params: &[QueryValue]) -> Result<Vec<QueryRow>, String> {
7589        query_connection(&self.conn, sql, params).map_err(|failure| failure.to_string())
7590    }
7591
7592    /// Repository benchmark and conformance access, present only with `bench-internals`.
7593    #[cfg(feature = "bench-internals")]
7594    #[doc(hidden)]
7595    pub fn benchmark_connection(&mut self) -> &mut Connection {
7596        &mut self.conn
7597    }
7598
7599    /// Private benchmark intervals; absent from ordinary builds and diagnostics.
7600    #[cfg(feature = "bench-internals")]
7601    #[doc(hidden)]
7602    pub fn benchmark_phases(
7603        &self,
7604        enabled: Option<bool>,
7605        reset: bool,
7606    ) -> Result<Option<Value>, String> {
7607        self.benchmark_phases.configure(enabled, reset)?;
7608        Ok(self.benchmark_phases.snapshot())
7609    }
7610
7611    /// One serialized, read-only authority observation, including durable coverage.
7612    pub fn authority_snapshot(&self) -> Result<Value, String> {
7613        validate_authority_reads(&self.schema, &self.limits.authority_reads)?;
7614        if self.limits.authority_reads.is_empty() {
7615            return Err("client.authority_read_forbidden: no authority reads were declared at client creation".into());
7616        }
7617        self.conn
7618            .execute_batch("SAVEPOINT authority_snapshot")
7619            .map_err(|e| e.to_string())?;
7620        let result = (|| {
7621            let revision = self
7622                .get_meta(LOCAL_REVISION_KEY)?
7623                .ok_or("sync.local_corrupt: authority revision is missing")?;
7624            let parsed_revision = revision
7625                .parse::<u64>()
7626                .map_err(|_| "sync.local_corrupt: authority revision is invalid")?;
7627            if parsed_revision.to_string() != revision {
7628                return Err("sync.local_corrupt: authority revision is invalid".into());
7629            }
7630            let mut tables = Vec::new();
7631            for read in &self.limits.authority_reads {
7632                let table = self
7633                    .schema
7634                    .table(&read.table)
7635                    .ok_or("client.authority_read_forbidden: authority table is not declared")?;
7636                let scalar = |column: &str, value: &Value| -> Result<String, String> {
7637                    let ty = table
7638                        .columns
7639                        .iter()
7640                        .find(|c| c.name == column)
7641                        .expect("validated authority column")
7642                        .ty;
7643                    if ty == ColumnType::Boolean {
7644                        if let Some(value) = value.as_bool() {
7645                            return Ok(value.to_string());
7646                        }
7647                        return match value.as_i64() {
7648                            Some(0) => Ok("false".into()),
7649                            Some(1) => Ok("true".into()),
7650                            _ => Err("sync.local_corrupt: authority boolean is invalid".into()),
7651                        };
7652                    }
7653                    match value {
7654                        Value::String(s) => Ok(s.clone()),
7655                        Value::Number(n) => Ok(n.to_string()),
7656                        Value::Object(o) if ty == ColumnType::Integer && o.len() == 1 => o
7657                            .get("$bigint")
7658                            .and_then(Value::as_str)
7659                            .map(str::to_owned)
7660                            .ok_or("sync.local_corrupt: authority integer is invalid".into()),
7661                        _ => {
7662                            Err("sync.local_corrupt: authority scalar is missing or invalid".into())
7663                        }
7664                    }
7665                };
7666                let matches = |row: &QueryRow| -> Result<bool, String> {
7667                    for (variable, values) in &read.scopes {
7668                        let column = table
7669                            .scope_column(variable)
7670                            .expect("validated authority scope");
7671                        let value = row
7672                            .get(column)
7673                            .ok_or("sync.local_corrupt: authority scope column is missing")?;
7674                        if value.is_null() || !values.contains(&scalar(column, value)?) {
7675                            return Ok(false);
7676                        }
7677                    }
7678                    Ok(true)
7679                };
7680                let selected = read
7681                    .columns
7682                    .iter()
7683                    .map(|c| quote_ident(c))
7684                    .collect::<Vec<_>>()
7685                    .join(",");
7686                let mut intent = BTreeSet::new();
7687                let mut relevant = BTreeSet::new();
7688                for row in query_connection(&self.conn,
7689                    "SELECT id,values_json FROM (SELECT json_extract(value,'$.table') AS tbl,json_extract(value,'$.rowId') AS id,json_extract(value,'$.values') AS values_json FROM _syncular_outbox,json_each(ops_json) UNION ALL SELECT json_extract(value,'$.table'),json_extract(value,'$.rowId'),json_extract(value,'$.values') FROM _syncular_failed_commits,json_each(operations_json) UNION ALL SELECT tbl,id,intent_json FROM _syncular_acked_rows) WHERE tbl=?",
7690                    &[Value::String(read.table.clone())]).map_err(|e| e.to_string())? {
7691                    let id = row.get("id").and_then(Value::as_str).ok_or("sync.local_corrupt: authority intent id is invalid")?.to_owned();
7692                    intent.insert(id.clone());
7693                    if let Some(raw) = row.get("values_json").and_then(Value::as_str) {
7694                        let values: QueryRow = serde_json::from_str(raw).map_err(|_| "sync.local_corrupt: authority intent values are invalid")?;
7695                        if read.scopes.keys().all(|v| values.contains_key(table.scope_column(v).expect("validated scope"))) && matches(&values)? {relevant.insert(id);}
7696                    }
7697                }
7698                let mut rows = BTreeMap::new();
7699                for full in [base_table(&read.table), visible_table(&read.table)] {
7700                    for mut values in query_connection(
7701                        &self.conn,
7702                        &format!(
7703                            "SELECT {selected},_syncular_version AS authority_version FROM {full}"
7704                        ),
7705                        &[],
7706                    )
7707                    .map_err(|e| e.to_string())?
7708                    {
7709                        if !matches(&values)? {
7710                            continue;
7711                        }
7712                        let id = scalar(
7713                            &table.primary_key,
7714                            values
7715                                .get(&table.primary_key)
7716                                .expect("selected primary key"),
7717                        )?;
7718                        relevant.insert(id.clone());
7719                        if full == base_table(&read.table) {
7720                            let version = values
7721                                .remove("authority_version")
7722                                .expect("selected version");
7723                            let version = version
7724                                .as_i64()
7725                                .ok_or("sync.local_corrupt: authority server version is invalid")?;
7726                            if version >= 0 {
7727                                rows.insert(id.clone(), json!({"values": values, "version": version, "hasLocalIntent": intent.contains(&id)}));
7728                            }
7729                        }
7730                    }
7731                }
7732                intent.retain(|id| relevant.contains(id));
7733                let rows = rows.into_values().collect::<Vec<_>>();
7734                // Every unit covering the table, as its (unit, table) pair (§4.4).
7735                let replica = self.stored(replica_cursor(&self.conn))?;
7736                let persisted = self
7737                    .stored(load_units(&self.conn))?
7738                    .into_iter()
7739                    .filter(|unit| covers(table, &unit.scopes))
7740                    .map(|unit| {
7741                        let effective = unit.echo.as_deref().and_then(|echo| {
7742                            pair_scopes(&unit.scopes, &read.table, Some(echo))
7743                        });
7744                        let requested = read
7745                            .scopes
7746                            .keys()
7747                            .map(|v| (v.clone(), json!(unit.scopes.get(v).into_iter().collect::<Vec<_>>())))
7748                            .collect::<Map<_, _>>();
7749                        let complete = effective
7750                            .as_ref()
7751                            .is_some_and(|scopes| scopes.len() == read.scopes.len())
7752                            && unit.status == UnitStatus::Active
7753                            && !unit.bootstrapping();
7754                        let exposed = effective.map(|scopes| {
7755                            read.scopes
7756                                .keys()
7757                                .map(|v| {
7758                                    let held = scopes.iter().find(|(variable, _)| variable == v);
7759                                    (v.clone(), json!(held.map_or(&Vec::new(), |(_, values)| values)))
7760                                })
7761                                .collect::<Map<_, _>>()
7762                        });
7763                        json!({"requestedScopes": requested, "status": unit.status, "cursor": unit.cursor.unwrap_or(replica), "effectiveScopes": exposed, "complete": complete})
7764                    })
7765                    .collect::<Vec<_>>();
7766                fn covered(selectors: &[(&String, &Vec<String>)], candidates: &[&Value]) -> bool {
7767                    match selectors.split_first() {
7768                        None => !candidates.is_empty(),
7769                        Some(((variable, values), rest)) => values.iter().all(|value| {
7770                            covered(
7771                                rest,
7772                                &candidates
7773                                    .iter()
7774                                    .copied()
7775                                    .filter(|p| {
7776                                        p["effectiveScopes"][variable.as_str()]
7777                                            .as_array()
7778                                            .is_some_and(|held| {
7779                                                held.contains(&Value::String(value.clone()))
7780                                                    || held.contains(&Value::String("*".into()))
7781                                            })
7782                                    })
7783                                    .collect::<Vec<_>>(),
7784                            )
7785                        }),
7786                    }
7787                }
7788                let coverage = if covered(
7789                    &read.scopes.iter().collect::<Vec<_>>(),
7790                    &persisted
7791                        .iter()
7792                        .filter(|p| p["complete"] == true)
7793                        .collect::<Vec<_>>(),
7794                ) {
7795                    "complete"
7796                } else if persisted.iter().any(|p| {
7797                    p["status"] == "active"
7798                        && read.scopes.iter().all(|(v, values)| {
7799                            values.iter().any(|value| {
7800                                p["requestedScopes"][v].as_array().is_some_and(|held| {
7801                                    held.contains(&Value::String(value.clone()))
7802                                })
7803                            })
7804                        })
7805                }) {
7806                    "pending"
7807                } else {
7808                    "missing"
7809                };
7810                tables.push(json!({"table": read.table, "rows": rows, "localIntentRowIds": intent, "scopes": read.scopes, "coverage": coverage, "persisted": persisted}));
7811            }
7812            Ok(
7813                json!({"revision": revision, "complete": tables.iter().all(|t| t["coverage"] == "complete"), "tables": tables}),
7814            )
7815        })();
7816        match result {
7817            Ok(snapshot) => {
7818                self.conn
7819                    .execute_batch("RELEASE authority_snapshot")
7820                    .map_err(|e| e.to_string())?;
7821                Ok(snapshot)
7822            }
7823            Err(error) => {
7824                self.conn
7825                    .execute_batch("ROLLBACK TO authority_snapshot; RELEASE authority_snapshot")
7826                    .map_err(|e| format!("{error}; rollback failed: {e}"))?;
7827                Err(error)
7828            }
7829        }
7830    }
7831
7832    /// Rows, coverage, and local revision from one SQLite read snapshot.
7833    /// §7.5 atomic snapshot read. An `owner` records a failure in
7834    /// `diagnostics_snapshot().query_failures` until its next successful read.
7835    pub fn query_snapshot(
7836        &mut self,
7837        sql: &str,
7838        params: &[QueryValue],
7839        coverage: &[WindowCoverage],
7840        owner: Option<QueryOwner<'_>>,
7841    ) -> Result<QuerySnapshot, String> {
7842        let result = snapshot_connection(&self.conn, sql, params, coverage);
7843        if let Some(owner) = owner {
7844            self.record_query_read(owner, result.as_ref().err());
7845        }
7846        result.map_err(|failure| failure.to_string())
7847    }
7848
7849    /// §7.6: record the outcome of one owned snapshot read, including reads a
7850    /// host ran on a separate connection. A success removes the owner's entry;
7851    /// an identical failure leaves it unchanged; a changed failure moves last.
7852    pub fn record_query_read(&mut self, owner: QueryOwner<'_>, failure: Option<&QueryReadFailure>) {
7853        let position = self
7854            .query_failures
7855            .iter()
7856            .position(|entry| entry.id == owner.id);
7857        let Some(failure) = failure else {
7858            if let Some(index) = position {
7859                self.query_failures.remove(index);
7860            }
7861            return;
7862        };
7863        let tables: Vec<String> = BTreeSet::from_iter(owner.tables.iter().copied())
7864            .into_iter()
7865            .map(str::to_owned)
7866            .collect();
7867        let code = failure.code.unwrap_or("client.query_failed").to_owned();
7868        if position.is_some_and(|index| {
7869            let entry = &self.query_failures[index];
7870            entry.tables == tables && entry.code == code && entry.sqlite_code == failure.sqlite_code
7871        }) {
7872            return;
7873        }
7874        if let Some(index) = position {
7875            self.query_failures.remove(index);
7876        }
7877        self.query_failures.push(DiagnosticQueryFailure {
7878            id: owner.id.to_owned(),
7879            tables,
7880            code,
7881            sqlite_code: failure.sqlite_code,
7882            at_ms: self.clock_now_ms(),
7883        });
7884        if self.query_failures.len() > MAX_DIAGNOSTIC_QUERY_FAILURES {
7885            self.query_failures.remove(0);
7886        }
7887    }
7888
7889    // -- request building ---------------------------------------------------------
7890
7891    fn build_request(&self, url_capable: bool) -> Result<(Message, RequestMeta), Box<SyncOutcome>> {
7892        #[cfg(feature = "bench-internals")]
7893        let _phase = self.benchmark_phases.start(Phase::RequestPrepare);
7894        let log_epoch = self.get_meta(LOG_EPOCH_KEY).map_err(|message| {
7895            Box::new(SyncOutcome::Failed {
7896                error_code: "storage.failed".into(),
7897                message,
7898                details: None,
7899                recommended_action: None,
7900            })
7901        })?;
7902        let max_commits = self.limits.max_push_commits_per_request;
7903        let max_ops = self
7904            .limits
7905            .max_push_operations_per_request
7906            .unwrap_or(PUSH_OPS_PER_REQUEST);
7907        let byte_cap = self.limits.max_push_request_bytes;
7908        // §4.2/§5.4: bit 3 is advertised iff the transport can fetch a
7909        // bare URL — capability negotiation, decided per transport.
7910        let accept = self.limits.accept.unwrap_or(if url_capable {
7911            DEFAULT_ACCEPT | ACCEPT_SIGNED_URLS
7912        } else {
7913            DEFAULT_ACCEPT
7914        });
7915        let storage_failed = |message: String| {
7916            Box::new(SyncOutcome::Failed {
7917                error_code: "storage.failed".into(),
7918                message,
7919                details: None,
7920                recommended_action: None,
7921            })
7922        };
7923        let header = Frame::ReqHeader {
7924            client_id: self.client_id.clone(),
7925            schema_version: self.schema.version,
7926            log_epoch: log_epoch.clone(),
7927            outbox_id: self.get_meta(OUTBOX_ID_KEY).map_err(storage_failed)?,
7928        };
7929        // §1.5: the lead frames are REQ_HEADER plus the optional ROUND_FAILURE,
7930        // which stays ahead of every push, pull, unit and progress frame; the
7931        // byte-capacity probes below measure this exact sequence.
7932        let mut lead = vec![header];
7933        lead.extend(
7934            self.round_failure
7935                .iter()
7936                .map(|code| Frame::RoundFailure { code: code.clone() }),
7937        );
7938        // §2.1: a replica without an epoch holds nothing bound to one.
7939        if log_epoch.is_none() {
7940            lead.push(Frame::AdoptEpoch);
7941        }
7942        let registered: Vec<UnitRecord> = self
7943            .stored(load_units(&self.conn))
7944            .map_err(storage_failed)?
7945            .into_iter()
7946            .filter(|unit| unit.status == UnitStatus::Active)
7947            .collect();
7948        let registration_digest = ssp2::registration::registration_digest(
7949            registered
7950                .iter()
7951                .map(|unit| scope_pairs(&unit.scopes))
7952                .collect::<Vec<_>>()
7953                .iter()
7954                .map(Vec::as_slice),
7955        );
7956        let echo_digest = self
7957            .get_meta(ECHO_DIGEST_KEY)
7958            .map_err(storage_failed)?
7959            .filter(|hex| !hex.is_empty())
7960            .map(|hex| {
7961                crate::values::hex_to_bytes(&hex)
7962                    .ok()
7963                    .and_then(|bytes| <[u8; 32]>::try_from(bytes).ok())
7964                    .ok_or_else(|| {
7965                        "sync.local_corrupt: persisted echo digest is invalid".to_owned()
7966                    })
7967            })
7968            .transpose()
7969            .map_err(storage_failed)?;
7970        let pull = Frame::PullHeader {
7971            limit_commits: self.limits.limit_commits.unwrap_or(0),
7972            limit_snapshot_rows: self.limits.limit_snapshot_rows.unwrap_or(0),
7973            max_snapshot_pages: self.limits.max_snapshot_pages.unwrap_or(0),
7974            accept,
7975            cursor: self
7976                .stored(replica_cursor(&self.conn))
7977                .map_err(storage_failed)?,
7978            registration_digest,
7979            echo_digest,
7980        };
7981        let registration_digest = bytes_to_hex(&registration_digest);
7982        // §4.1: the list travels when it changed since the server stored it,
7983        // after `unregistered` and after a reset (both clear the stored digest).
7984        let travels = self
7985            .get_meta(REGISTERED_DIGEST_KEY)
7986            .map_err(storage_failed)?
7987            .as_deref()
7988            != Some(registration_digest.as_str());
7989        let mut sub_frames: Vec<Frame> = Vec::new();
7990        if travels {
7991            sub_frames.extend(registered.iter().map(|unit| Frame::Unit {
7992                scopes: scope_pairs(&unit.scopes),
7993            }));
7994        }
7995        // §4.3: one PROGRESS per catching-up unit, in registration order, each
7996        // followed by its RESTRICT while the unit holds a restriction.
7997        for unit in &registered {
7998            let Some(cursor) = unit.cursor else {
7999                continue;
8000            };
8001            sub_frames.push(Frame::Progress {
8002                id: unit.id.clone(),
8003                cursor,
8004                bootstrap_state: unit.bootstrap_state.clone().map(RawJson),
8005            });
8006            sub_frames.extend(
8007                unit.restriction
8008                    .as_ref()
8009                    .map(|restriction| Frame::Restrict {
8010                        cursor: restriction.cursor,
8011                        tables: restriction.tables.clone(),
8012                    }),
8013            );
8014        }
8015        // The fixed frames (order-independent size) and the header+pull probe
8016        // both encode canonically, so the byte budget is the exact wire size.
8017        let fixed_bytes = match byte_cap {
8018            Some(_) => {
8019                let mut frames = lead.clone();
8020                frames.push(pull.clone());
8021                frames.extend(sub_frames.iter().cloned());
8022                encode_message(&Message {
8023                    wire_version: WIRE_VERSION,
8024                    msg_kind: MsgKind::Request,
8025                    frames,
8026                })
8027                .len()
8028            }
8029            None => 0,
8030        };
8031        if let Some(cap) = byte_cap {
8032            if fixed_bytes > cap {
8033                return Err(Self::push_capacity_outcome("bytes", cap, fixed_bytes, None));
8034            }
8035        }
8036        let base_bytes = match byte_cap {
8037            Some(_) => {
8038                let mut frames = lead.clone();
8039                frames.push(pull.clone());
8040                encode_message(&Message {
8041                    wire_version: WIRE_VERSION,
8042                    msg_kind: MsgKind::Request,
8043                    frames,
8044                })
8045                .len()
8046            }
8047            None => 0,
8048        };
8049        let total_outbox = if log_epoch.is_some() {
8050            self.outbox.len()
8051        } else {
8052            0
8053        };
8054        let mut push_frames: Vec<Frame> = Vec::new();
8055        let mut pushed_ids = Vec::new();
8056        let mut ops_in_request = 0usize;
8057        let mut bytes_in_request = 0usize;
8058        // §2.1: an epoch-less request withholds the whole outbox until the
8059        // next, epoch-bound request.
8060        let mut deferred_commits = self.outbox.len() - total_outbox;
8061        // §7.2.2: a pending replacement is held; the push stops before it
8062        // without deferring work.
8063        let held: HashSet<String> = self
8064            .prepare_replacements()
8065            .map_err(storage_failed)?
8066            .into_iter()
8067            .filter_map(|decision| decision.replacement)
8068            .collect();
8069        #[cfg(feature = "bench-internals")]
8070        let outbox_phase = self.benchmark_phases.start(Phase::OutboxEncode);
8071        for (index, commit) in self.outbox.iter().take(total_outbox).enumerate() {
8072            if held.contains(&commit.client_commit_id) {
8073                break;
8074            }
8075            // §6.1/§7.1: the commit-count and operation caps are hard. A
8076            // first commit over the operation cap is a typed capacity error
8077            // and stays queued; a later one defers with the complete suffix.
8078            if max_commits.is_some_and(|cap| push_frames.len() >= cap) {
8079                deferred_commits = total_outbox - index;
8080                break;
8081            }
8082            if ops_in_request + commit.ops.len() > max_ops {
8083                if push_frames.is_empty() {
8084                    return Err(Self::push_capacity_outcome(
8085                        "operations",
8086                        max_ops,
8087                        commit.ops.len(),
8088                        Some(commit.client_commit_id.as_str()),
8089                    ));
8090                }
8091                deferred_commits = total_outbox - index;
8092                break;
8093            }
8094            // §7.1 chained base: the commit waits for the change of the commit
8095            // it chains on, which supplies its rebased version.
8096            if self.outbox[..index]
8097                .iter()
8098                .flat_map(|earlier| &earlier.ops)
8099                .any(|earlier| commit.ops.iter().any(|op| chains_on(earlier, op)))
8100            {
8101                deferred_commits = total_outbox - index;
8102                break;
8103            }
8104            let operations = commit
8105                .ops
8106                .iter()
8107                .map(|op| {
8108                    let payload = match self
8109                        .schema
8110                        .table(&op.table)
8111                        .and_then(|table| Some((table, op.presented_values(table)?)))
8112                    {
8113                        Some((table, values)) => {
8114                            // §0: outbox entries encode at send time with the
8115                            // current codec (validated at mutate()). §6.1: the
8116                            // values map's keys are the sparse presence set; an
8117                            // insert-intent operation presents every column (§7.1).
8118                            // §5.11: encrypted columns are encrypted here, with
8119                            // an absent key-id selector read from the stored row.
8120                            // `sync_inner` runs `drop_unencodable_outbox` first,
8121                            // so an encrypt failure cannot reach this encode; the
8122                            // author seam validated every other failure.
8123                            let fallback =
8124                                self.stored_key_fallback(table, &op.row_id)
8125                                    .map_err(|message| {
8126                                        Box::new(SyncOutcome::Failed {
8127                                            error_code: "storage.failed".into(),
8128                                            message,
8129                                            details: None,
8130                                            recommended_action: None,
8131                                        })
8132                                    })?;
8133                            encode_sparse_row_json(
8134                                table,
8135                                &op.row_id,
8136                                &values,
8137                                &self.encryption,
8138                                fallback.as_ref(),
8139                            )
8140                            .ok()
8141                        }
8142                        None => None,
8143                    };
8144                    Ok(ssp2::model::Operation {
8145                        table: op.table.clone(),
8146                        row_id: op.row_id.clone(),
8147                        op: if op.upsert { Op::Upsert } else { Op::Delete },
8148                        base_version: op.base_version,
8149                        payload,
8150                    })
8151                })
8152                .collect::<Result<Vec<_>, Box<SyncOutcome>>>()?;
8153            // §6.3.2, §7.1: the commit's outbox sequence.
8154            let client_seq = self
8155                .conn
8156                .prepare_cached("SELECT seq FROM _syncular_outbox WHERE commit_id = ?1")
8157                .and_then(|mut statement| {
8158                    statement.query_row([&commit.client_commit_id], |row| row.get(0))
8159                })
8160                .map_err(|error| {
8161                    storage_failed(Self::sqlite_failure(&self.storage_failure, error))
8162                })?;
8163            let frame = Frame::PushCommit {
8164                client_commit_id: commit.client_commit_id.clone(),
8165                client_seq,
8166                operations,
8167            };
8168            let frame_bytes = match byte_cap {
8169                Some(_) => {
8170                    let mut frames = lead.clone();
8171                    frames.push(frame.clone());
8172                    frames.push(pull.clone());
8173                    encode_message(&Message {
8174                        wire_version: WIRE_VERSION,
8175                        msg_kind: MsgKind::Request,
8176                        frames,
8177                    })
8178                    .len()
8179                        - base_bytes
8180                }
8181                None => 0,
8182            };
8183            // The head is always encoded so the caller can report the full
8184            // projected request size; a later commit defers once it no longer
8185            // fits.
8186            if let Some(cap) = byte_cap {
8187                if !push_frames.is_empty() && fixed_bytes + bytes_in_request + frame_bytes > cap {
8188                    deferred_commits = total_outbox - index;
8189                    break;
8190                }
8191            }
8192            push_frames.push(frame);
8193            pushed_ids.push(commit.client_commit_id.clone());
8194            ops_in_request += commit.ops.len();
8195            bytes_in_request += frame_bytes;
8196        }
8197        #[cfg(feature = "bench-internals")]
8198        drop(outbox_phase);
8199        if let Some(cap) = byte_cap {
8200            if fixed_bytes + bytes_in_request > cap {
8201                // The head alone does not fit this budget; the commit stays
8202                // queued and no transport round is attempted.
8203                let blocked = if push_frames.is_empty() {
8204                    None
8205                } else {
8206                    self.outbox.first().map(|c| c.client_commit_id.as_str())
8207                };
8208                return Err(Self::push_capacity_outcome(
8209                    "bytes",
8210                    cap,
8211                    fixed_bytes + bytes_in_request,
8212                    blocked,
8213                ));
8214            }
8215        }
8216        let mut frames = lead;
8217        frames.extend(push_frames);
8218        frames.push(pull);
8219        frames.extend(sub_frames);
8220        let message = Message {
8221            wire_version: WIRE_VERSION,
8222            msg_kind: MsgKind::Request,
8223            frames,
8224        };
8225        Ok((
8226            message,
8227            RequestMeta {
8228                pushed_ids,
8229                registration_digest,
8230                accept,
8231                deferred_commits,
8232                applied: BTreeMap::new(),
8233                deferred: BTreeSet::new(),
8234            },
8235        ))
8236    }
8237
8238    /// §7.1: a configured push capacity limit cannot carry this request.
8239    fn push_capacity_outcome(
8240        kind: &str,
8241        limit: usize,
8242        size: usize,
8243        client_commit_id: Option<&str>,
8244    ) -> Box<SyncOutcome> {
8245        let mut details = Map::new();
8246        details.insert("kind".to_owned(), Value::from(kind));
8247        details.insert("limit".to_owned(), Value::from(limit as u64));
8248        details.insert("size".to_owned(), Value::from(size as u64));
8249        if let Some(id) = client_commit_id {
8250            details.insert("clientCommitId".to_owned(), Value::from(id));
8251        }
8252        Box::new(SyncOutcome::Failed {
8253            error_code: "client.push_request_too_large".into(),
8254            message: "the push request exceeds a configured capacity limit".into(),
8255            details: Some(Value::Object(details)),
8256            recommended_action: None,
8257        })
8258    }
8259
8260    // -- sync -------------------------------------------------------------------
8261
8262    /// Cloneable observation handle; no client lock is needed to read it.
8263    pub fn progress(&self) -> ProgressObserver {
8264        self.progress.clone()
8265    }
8266
8267    pub fn sync(&mut self, transport: &mut dyn Transport) -> SyncOutcome {
8268        self.sync_bounded(transport, None)
8269    }
8270
8271    /// One round; `idle_bound` is the idle drive's round bound (§7.8).
8272    fn sync_bounded(
8273        &mut self,
8274        transport: &mut dyn Transport,
8275        idle_bound: Option<(u64, u32)>,
8276    ) -> SyncOutcome {
8277        let mut prepared = match self.prepare_sync_round(transport.supports_url_fetch()) {
8278            Ok(prepared) => prepared,
8279            Err(outcome) => return *outcome,
8280        };
8281        prepared.idle_bound = idle_bound;
8282        // The round's network deadline rides `PreparedSyncRound` and is scoped
8283        // to each exchange, so uploads, continuations, the main request, and
8284        // every segment fetch share one monotonic anchor without leaving
8285        // deadline state on a reused transport.
8286        let mut next = prepared;
8287        loop {
8288            match self.apply_sync_round(next.exchange(transport)) {
8289                crate::AppliedSyncRound::Continue(prepared) => next = prepared,
8290                crate::AppliedSyncRound::Complete {
8291                    outcome, controls, ..
8292                } => {
8293                    if self.transport_enabled {
8294                        for text in controls {
8295                            let _ = transport.realtime_send(&text);
8296                        }
8297                    } else {
8298                        self.disconnect_realtime(transport);
8299                    }
8300                    return outcome;
8301                }
8302            }
8303        }
8304    }
8305
8306    fn finish_sync_round(
8307        &mut self,
8308        started_at_ms: i64,
8309        attempt: u64,
8310        mut outcome: SyncOutcome,
8311    ) -> SyncOutcome {
8312        if let SyncOutcome::Failed {
8313            error_code,
8314            message,
8315            details,
8316            ..
8317        } = &mut outcome
8318        {
8319            if let Some((failure, code)) = self
8320                .storage_failure
8321                .borrow()
8322                .as_ref()
8323                .and_then(|failure| failure.code.map(|code| (failure, code)))
8324            {
8325                *error_code = code.to_owned();
8326                *message = match code {
8327                    "client.storage_busy" => "local SQLite storage is busy",
8328                    "client.storage_full" => "local SQLite storage is full",
8329                    "client.storage_io" => "local SQLite storage I/O failed",
8330                    "client.storage_corrupt" => "local SQLite storage is corrupt",
8331                    _ => unreachable!("unrecognized classified storage failure"),
8332                }
8333                .to_owned();
8334                *details = failure.details();
8335            }
8336        }
8337        let details = match &outcome {
8338            SyncOutcome::Failed { details, .. } => details.as_ref(),
8339            _ => None,
8340        };
8341        let cause_kind = details
8342            .and_then(|details| details.get("causeKind"))
8343            .and_then(Value::as_str)
8344            .and_then(|cause| match cause {
8345                "network" => Some(crate::TransportCauseKind::Network),
8346                "connect" => Some(crate::TransportCauseKind::Connect),
8347                "tls" => Some(crate::TransportCauseKind::Tls),
8348                "io" => Some(crate::TransportCauseKind::Io),
8349                "request" => Some(crate::TransportCauseKind::Request),
8350                "protocol" => Some(crate::TransportCauseKind::Protocol),
8351                "timeout" => Some(crate::TransportCauseKind::Timeout),
8352                "round_deadline" => Some(crate::TransportCauseKind::RoundDeadline),
8353                "aborted" => Some(crate::TransportCauseKind::Aborted),
8354                "body" => Some(crate::TransportCauseKind::Body),
8355                "status" => Some(crate::TransportCauseKind::Status),
8356                "redirect" => Some(crate::TransportCauseKind::Redirect),
8357                "unknown" => Some(crate::TransportCauseKind::Unknown),
8358                _ => None,
8359            });
8360        let http_status = if matches!(
8361            cause_kind,
8362            Some(
8363                crate::TransportCauseKind::Status
8364                    | crate::TransportCauseKind::Body
8365                    | crate::TransportCauseKind::Redirect
8366            )
8367        ) {
8368            details
8369                .and_then(|details| details.get("httpStatus"))
8370                .and_then(Value::as_f64)
8371                .filter(|status| status.fract() == 0.0 && (100.0..=599.0).contains(status))
8372                .map(|status| status as u16)
8373        } else {
8374            None
8375        };
8376        let retry_delay_ms = self.round_retry_delay_ms;
8377        self.progress.update(|p| match &outcome {
8378            SyncOutcome::Ok(report) | SyncOutcome::BudgetExhausted(report)
8379                if report.failed.is_empty() =>
8380            {
8381                p.state = ProgressState::Complete
8382            }
8383            SyncOutcome::Ok(_) | SyncOutcome::BudgetExhausted(_) => {
8384                p.state = ProgressState::Failed;
8385                p.error_code = Some("sync.scope_revoked".into());
8386            }
8387            SyncOutcome::Failed { error_code, .. } => {
8388                p.state = ProgressState::Failed;
8389                p.error_code = Some(Self::diagnostic_code(error_code));
8390                p.cause_kind = cause_kind;
8391                p.http_status = http_status;
8392                p.retry_delay_ms = retry_delay_ms;
8393            }
8394            SyncOutcome::RealtimeUnavailable { .. } => {
8395                p.state = ProgressState::Failed;
8396                p.error_code = Some(REALTIME_UNAVAILABLE_CODE.to_owned());
8397                p.retry_delay_ms = retry_delay_ms;
8398            }
8399        });
8400        let completed_at_ms = self.clock_now_ms();
8401        self.last_round = Some(match &outcome {
8402            SyncOutcome::Ok(report) | SyncOutcome::BudgetExhausted(report) => DiagnosticLastRound {
8403                epoch: self.epoch.clone(),
8404                attempt,
8405                status: "succeeded".to_owned(),
8406                started_at_ms,
8407                completed_at_ms,
8408                duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
8409                counters: Some(DiagnosticRoundCounters {
8410                    pushed: report.pushed,
8411                    applied: report.applied.len(),
8412                    rejected: report.rejected.len(),
8413                    retryable: report.retryable.len(),
8414                    conflicts: report.conflicts,
8415                    commits_applied: report.commits_applied,
8416                    segment_rows_applied: report.segment_rows_applied,
8417                    bootstrapping: report.bootstrapping.len(),
8418                    resets: report.resets.len(),
8419                    revoked: report.revoked.len(),
8420                    failed: report.failed.len(),
8421                    deferred_commits: report.deferred_commits,
8422                }),
8423                error_code: None,
8424                cause_kind: None,
8425                http_status: None,
8426            },
8427            SyncOutcome::Failed { error_code, .. } => DiagnosticLastRound {
8428                epoch: self.epoch.clone(),
8429                attempt,
8430                status: "failed".to_owned(),
8431                started_at_ms,
8432                completed_at_ms,
8433                duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
8434                counters: None,
8435                error_code: Some(Self::diagnostic_code(error_code)),
8436                cause_kind,
8437                http_status,
8438            },
8439            SyncOutcome::RealtimeUnavailable { .. } => DiagnosticLastRound {
8440                epoch: self.epoch.clone(),
8441                attempt,
8442                status: "failed".to_owned(),
8443                started_at_ms,
8444                completed_at_ms,
8445                duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
8446                counters: None,
8447                error_code: Some(REALTIME_UNAVAILABLE_CODE.to_owned()),
8448                cause_kind: None,
8449                http_status: None,
8450            },
8451        });
8452        outcome
8453    }
8454
8455    /// Capture one request on the client owner. Mutations made after this
8456    /// call remain in the outbox for a later round; no network I/O occurs here.
8457    pub fn prepare_sync_round(
8458        &mut self,
8459        url_capable: bool,
8460    ) -> Result<crate::PreparedSyncRound, Box<SyncOutcome>> {
8461        if !self.transport_enabled {
8462            return Err(Box::new(SyncOutcome::Failed {
8463                error_code: "sync.offline".into(),
8464                message: "transport gate is closed".into(),
8465                details: None,
8466                recommended_action: None,
8467            }));
8468        }
8469        if self.active_round.is_some() {
8470            return Err(Box::new(SyncOutcome::Failed {
8471                error_code: "client.round_in_progress".into(),
8472                message: "a sync round is already in flight".into(),
8473                details: None,
8474                recommended_action: None,
8475            }));
8476        }
8477        self.storage_failure.borrow_mut().take();
8478        let attempt = self.progress.start(&self.epoch);
8479        self.round_retry_delay_ms = None;
8480        let started_at_ms = self.clock_now_ms();
8481        let prepared = (|| {
8482            if self.stopped {
8483                return Err(Box::new(SyncOutcome::Ok(SyncReport {
8484                    schema_floor: self.schema_floor.clone(),
8485                    ..SyncReport::default()
8486                })));
8487            }
8488            if self.security_preflight {
8489                return Err(Box::new(SyncOutcome::Failed {
8490                    error_code: SECURITY_PREFLIGHT_REQUIRED_CODE.into(),
8491                    message: "security preflight must complete before syncing".into(),
8492                    details: None,
8493                    recommended_action: None,
8494                }));
8495            }
8496            self.set_sync_needed(false, false);
8497            self.drain_pending_evictions()
8498                .and_then(|_| self.drop_incompatible_outbox())
8499                .and_then(|_| self.drop_unencodable_outbox(true, None))
8500                .map_err(|message| {
8501                    Box::new(SyncOutcome::Failed {
8502                        error_code: "storage.failed".into(),
8503                        message,
8504                        details: None,
8505                        recommended_action: None,
8506                    })
8507                })?;
8508            // §5.11: reveal rows sealed under a key the keyring gained since.
8509            self.reveal_sealed_rows().map_err(|error| {
8510                Box::new(SyncOutcome::Failed {
8511                    error_code: error.code,
8512                    message: error.message,
8513                    details: error.details,
8514                    recommended_action: None,
8515                })
8516            })?;
8517            let has_log_epoch = self.get_meta(LOG_EPOCH_KEY).map_err(|message| {
8518                Box::new(SyncOutcome::Failed {
8519                    error_code: "storage.failed".into(),
8520                    message,
8521                    details: None,
8522                    recommended_action: None,
8523                })
8524            })?;
8525            let uploads = if has_log_epoch.is_some() && self.schema_has_blobs() {
8526                self.pending_blob_uploads().map_err(|e| {
8527                    Box::new(SyncOutcome::Failed {
8528                        error_code: e.code,
8529                        message: e.message,
8530                        details: e.details,
8531                        recommended_action: None,
8532                    })
8533                })?
8534            } else {
8535                Vec::new()
8536            };
8537            if self.realtime_state != RealtimeState::Connected
8538                && self.realtime_policy == RealtimePolicy::Required
8539            {
8540                let retry_delay_ms = self.schedule_background_retry();
8541                self.realtime_retry_delay_ms = Some(retry_delay_ms);
8542                return Err(Box::new(SyncOutcome::RealtimeUnavailable {
8543                    state: self.realtime_state,
8544                    reason_code: self.realtime_reason_code.clone(),
8545                    retry_delay_ms,
8546                }));
8547            }
8548            let (message, mut meta) = self.build_request(url_capable)?;
8549            let units = self.stored(load_units(&self.conn)).map_err(|message| {
8550                Box::new(SyncOutcome::Failed {
8551                    error_code: "storage.failed".into(),
8552                    message,
8553                    details: None,
8554                    recommended_action: None,
8555                })
8556            })?;
8557            for unit in &units {
8558                let applied =
8559                    self.stored(applied_rows(&self.conn, &unit.id))
8560                        .map_err(|message| {
8561                            Box::new(SyncOutcome::Failed {
8562                                error_code: "storage.failed".into(),
8563                                message,
8564                                details: None,
8565                                recommended_action: None,
8566                            })
8567                        })?;
8568                if !applied.is_empty() {
8569                    meta.applied.insert(unit.id.clone(), applied);
8570                }
8571            }
8572            let id = uuid::Uuid::new_v4();
8573            self.active_round = Some((id, started_at_ms, attempt));
8574            let phase = if uploads.is_empty() {
8575                crate::round::RoundPhase::Sync
8576            } else {
8577                crate::round::RoundPhase::UploadGrant
8578            };
8579            Ok(crate::PreparedSyncRound {
8580                id,
8581                started_at_ms,
8582                attempt,
8583                message,
8584                meta,
8585                uploads: uploads.into(),
8586                phase,
8587                encoded: None,
8588                continuation: None,
8589                completed: None,
8590                realtime: self.realtime_state == RealtimeState::Connected,
8591                // §5.5: the owning unit's scope map, each value a one-element list.
8592                scopes: units
8593                    .into_iter()
8594                    .map(|unit| {
8595                        (
8596                            unit.id,
8597                            canonical_scope_json(&unit_scope_lists(&unit.scopes)),
8598                        )
8599                    })
8600                    .collect(),
8601                #[cfg(feature = "bench-internals")]
8602                benchmark_phases: self.benchmark_phases.clone(),
8603                progress: self.progress.clone(),
8604                fixed_now: self
8605                    .now_ms
8606                    .as_ref()
8607                    .map(|clock| clock.load(std::sync::atomic::Ordering::Relaxed)),
8608                round_deadline_at: None,
8609                idle_bound: None,
8610                network_started_at: None,
8611            })
8612        })();
8613        prepared
8614            .map_err(|outcome| Box::new(self.finish_sync_round(started_at_ms, attempt, *outcome)))
8615    }
8616
8617    /// Discard the reply of an old authorization or registration context. Its
8618    /// submitted commits stay durable and can be retried with the same ids.
8619    /// §7.6: an abandoned in-flight round ends its attempt as failed.
8620    pub fn cancel_sync_round(&mut self) {
8621        if let Some((_, started_at_ms, attempt)) = self.active_round.take() {
8622            // A storage failure recorded since the round started is not its cause.
8623            self.storage_failure.borrow_mut().take();
8624            self.finish_sync_round(
8625                started_at_ms,
8626                attempt,
8627                SyncOutcome::Failed {
8628                    error_code: "client.round_cancelled".into(),
8629                    message: "sync round belongs to a released client context".into(),
8630                    details: None,
8631                    recommended_action: None,
8632                },
8633            );
8634        }
8635    }
8636
8637    /// Apply captured network results on the owner, preserving all normal
8638    /// version, schema, content-address and cursor checks. No network I/O.
8639    pub fn apply_sync_round(
8640        &mut self,
8641        completed: crate::CompletedSyncRound,
8642    ) -> crate::AppliedSyncRound {
8643        let mut prepared = completed.prepared;
8644        if self.active_round.map(|(id, ..)| id) != Some(prepared.id) || self.security_preflight {
8645            return crate::AppliedSyncRound::Complete {
8646                outcome: SyncOutcome::Failed {
8647                    error_code: "client.round_cancelled".into(),
8648                    message: "sync round belongs to a released client context".into(),
8649                    details: None,
8650                    recommended_action: None,
8651                },
8652                controls: Vec::new(),
8653                more: false,
8654                bootstrap_advanced: false,
8655            };
8656        }
8657        self.storage_failure.borrow_mut().take();
8658        let (response, mut downloads, transport_failed) = match completed.exchange {
8659            crate::round::ExchangeResult::Upload { id, result } => {
8660                let result = match result {
8661                    // §5.9.7: a non-retryable server refusal (`blob.too_large`)
8662                    // is the body's final outcome. The commits naming it take
8663                    // the incompatible-commit path with the server's code and
8664                    // leave this round's request; the round continues.
8665                    Err(error)
8666                        if error
8667                            .details
8668                            .as_ref()
8669                            .and_then(|details| details.get("retryable"))
8670                            == Some(&Value::Bool(false)) =>
8671                    {
8672                        self.conn
8673                            .prepare(
8674                                "SELECT commit_id FROM _syncular_blob_commit_refs WHERE blob_id = ?",
8675                            )
8676                            .and_then(|mut statement| {
8677                                statement
8678                                    .query_map(rusqlite::params![id], |row| row.get::<_, String>(0))?
8679                                    .collect::<Result<HashSet<_>, _>>()
8680                            })
8681                            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
8682                            .and_then(|refused| {
8683                                let commits = self
8684                                    .outbox
8685                                    .iter()
8686                                    .filter(|commit| refused.contains(&commit.client_commit_id))
8687                                    .cloned()
8688                                    .collect::<Vec<_>>();
8689                                if !commits.is_empty() {
8690                                    self.drop_outbox_commits(&commits, &error.code, &error.message)?;
8691                                }
8692                                prepared.message.frames.retain(|frame| {
8693                                    !matches!(frame, Frame::PushCommit { client_commit_id, .. }
8694                                        if refused.contains(client_commit_id))
8695                                });
8696                                prepared.meta.pushed_ids.retain(|id| !refused.contains(id));
8697                                prepared.continuation = Some(crate::round::RoundPhase::UploadGrant);
8698                                Ok(())
8699                            })
8700                            .map_err(|message| TransportError::new("client.failed", message))
8701                    }
8702                    result => result,
8703                };
8704                let result = result.and_then(|()| {
8705                    self.conn
8706                        .execute(
8707                            "DELETE FROM _syncular_blob_uploads WHERE blob_id = ?",
8708                            rusqlite::params![id],
8709                        )
8710                        .map(|_| ())
8711                        .map_err(|error| {
8712                            TransportError::new(
8713                                "client.failed",
8714                                Self::sqlite_failure(&self.storage_failure, error),
8715                            )
8716                        })
8717                });
8718                match result {
8719                    Ok(()) => return crate::AppliedSyncRound::Continue(prepared),
8720                    Err(TransportError {
8721                        code,
8722                        message,
8723                        details,
8724                    }) => {
8725                        self.active_round = None;
8726                        if Self::unreachable_transport(&details) {
8727                            // The round's work is still owed; the host's
8728                            // connectivity resume runs it.
8729                            self.set_sync_needed(true, false);
8730                        } else if Self::retryable_failure_code(&code) {
8731                            self.schedule_background_retry();
8732                        }
8733                        let recommended_action = recommended_action_of(&details);
8734                        let outcome = self.finish_sync_round(
8735                            prepared.started_at_ms,
8736                            prepared.attempt,
8737                            SyncOutcome::Failed {
8738                                error_code: code,
8739                                message,
8740                                details,
8741                                recommended_action,
8742                            },
8743                        );
8744                        return crate::AppliedSyncRound::Complete {
8745                            outcome,
8746                            controls: Vec::new(),
8747                            more: false,
8748                            bootstrap_advanced: false,
8749                        };
8750                    }
8751                }
8752            }
8753            crate::round::ExchangeResult::Reply {
8754                response,
8755                downloads,
8756                transport_failed,
8757            } => (response, downloads, transport_failed),
8758        };
8759        self.active_round = None;
8760        let before = self.resume_tokens();
8761        let outcome = (|| {
8762            let response = match response {
8763                Ok(response) => {
8764                    // §1.5: a complete, decodable response ends the report, an
8765                    // in-band ERROR included (the server already recorded it).
8766                    self.round_failure = None;
8767                    response
8768                }
8769                Err(TransportError {
8770                    code,
8771                    message,
8772                    details,
8773                }) => {
8774                    // §1.5: the server cannot see a round whose transport
8775                    // failed or whose response never decoded, so its code rides
8776                    // the following requests. A code the frame grammar rejects
8777                    // is not reportable, and the previous code stays; the
8778                    // client-local drive timeout never goes on the wire (§7.8).
8779                    if code != "sync.idle_timeout" && ssp2::model::valid_round_failure_code(&code) {
8780                        self.round_failure = Some(code.clone());
8781                    }
8782                    if transport_failed {
8783                        if prepared.realtime {
8784                            self.set_realtime_state(RealtimeState::Lost, Some(&code));
8785                        }
8786                        if let Err(message) = self.record_lease_error(&code) {
8787                            return SyncOutcome::Failed {
8788                                error_code: "storage.failed".to_owned(),
8789                                message,
8790                                details: None,
8791                                recommended_action: None,
8792                            };
8793                        }
8794                    }
8795                    if Self::retryable_failure_code(&code) && !Self::unreachable_transport(&details)
8796                    {
8797                        let delay = self.schedule_background_retry();
8798                        if prepared.realtime && transport_failed {
8799                            self.realtime_retry_delay_ms = Some(delay);
8800                        }
8801                    }
8802                    let recommended_action = recommended_action_of(&details);
8803                    return SyncOutcome::Failed {
8804                        error_code: code,
8805                        message,
8806                        details,
8807                        recommended_action,
8808                    };
8809                }
8810            };
8811            let mut outcome = self.process_response(&mut downloads, response, &prepared.meta);
8812            if let SyncOutcome::Ok(report) = &mut outcome {
8813                report.deferred_commits = prepared.meta.deferred_commits;
8814            }
8815            match &outcome {
8816                SyncOutcome::Ok(_) => self.reset_background_retry(),
8817                SyncOutcome::Failed {
8818                    error_code,
8819                    details,
8820                    ..
8821                } if Self::retryable_failure_code(error_code)
8822                    && !Self::unreachable_transport(details) =>
8823                {
8824                    self.schedule_background_retry();
8825                }
8826                _ => {}
8827            }
8828            if prepared.meta.deferred_commits > 0 {
8829                self.set_sync_needed(true, true);
8830            }
8831            self.previous_version_lifetime_check();
8832            outcome
8833        })();
8834        match &outcome {
8835            // §7.1: a unit still catching up needs another round, also when the
8836            // drive that started it ends early.
8837            SyncOutcome::Ok(report) if !report.bootstrapping.is_empty() => {
8838                self.set_sync_needed(true, false)
8839            }
8840            // The round's work is still owed; the host's connectivity resume
8841            // runs it.
8842            SyncOutcome::Failed { details, .. } if Self::unreachable_transport(details) => {
8843                self.set_sync_needed(true, false)
8844            }
8845            _ => {}
8846        }
8847        let more = self.transport_enabled
8848            && matches!(&outcome, SyncOutcome::Ok(report)
8849                if report.schema_floor.is_none()
8850                    && (!report.bootstrapping.is_empty()
8851                        || report.commits_applied > 0
8852                        || report.segment_rows_applied > 0
8853                        || !report.resets.is_empty()
8854                        || report.deferred_commits > 0
8855                        || self.sync_needed));
8856        let bootstrap_advanced = matches!(&outcome, SyncOutcome::Ok(report) if report.segment_rows_applied > 0 && !report.bootstrapping.is_empty())
8857            && before != self.resume_tokens();
8858        crate::AppliedSyncRound::Complete {
8859            outcome: self.finish_sync_round(prepared.started_at_ms, prepared.attempt, outcome),
8860            controls: if self.transport_enabled {
8861                downloads.controls
8862            } else {
8863                Vec::new()
8864            },
8865            more,
8866            bootstrap_advanced,
8867        }
8868    }
8869
8870    /// Every unit's resume token and applied-segment record: a bootstrap
8871    /// that advanced changes one (§4.7), also when it applied images of a
8872    /// section it deferred (§5.3, §5.6).
8873    #[allow(clippy::type_complexity)]
8874    fn resume_tokens(&self) -> Option<Vec<(String, Option<String>, Option<String>)>> {
8875        self.conn
8876            .prepare_cached(
8877                "SELECT id, bootstrap_state, applied FROM _syncular_units ORDER BY rowid",
8878            )
8879            .and_then(|mut stmt| {
8880                stmt.query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))?
8881                    .collect()
8882            })
8883            .ok()
8884    }
8885
8886    /// §7.8: one idle drive's default total deadline, the TS client's value.
8887    pub const SYNC_IDLE_TIMEOUT_MS: u64 = 60_000;
8888
8889    pub fn sync_until_idle(
8890        &mut self,
8891        transport: &mut dyn Transport,
8892        max_rounds: Option<u32>,
8893    ) -> SyncOutcome {
8894        self.sync_until_idle_with_deadline(transport, max_rounds, Some(Self::SYNC_IDLE_TIMEOUT_MS))
8895    }
8896
8897    /// As [`SyncClient::sync_until_idle`], with an explicit total deadline in
8898    /// milliseconds (`None` disables it). The deadline is checked against the
8899    /// client clock between rounds: a drive past it after a completed round
8900    /// ends as `BudgetExhausted`. Each round is bounded by `timeout_ms` on its
8901    /// own, or the transport's tighter whole-round deadline (§1.8); a round
8902    /// that reaches the drive bound fails with `sync.idle_timeout`.
8903    pub fn sync_until_idle_with_deadline(
8904        &mut self,
8905        transport: &mut dyn Transport,
8906        max_rounds: Option<u32>,
8907        timeout_ms: Option<u64>,
8908    ) -> SyncOutcome {
8909        // §7.8: an explicit budget is a strict positive round count. Zero is a
8910        // caller error, not an exhaustion.
8911        if max_rounds == Some(0) {
8912            return SyncOutcome::Failed {
8913                details: None,
8914                error_code: "sync.invalid_request".into(),
8915                message: "maxRounds must be a positive integer".into(),
8916                recommended_action: None,
8917            };
8918        }
8919        if timeout_ms.is_some_and(|ms| ms == 0 || i64::try_from(ms).is_err()) {
8920            return SyncOutcome::Failed {
8921                details: None,
8922                error_code: "sync.invalid_request".into(),
8923                message: "timeoutMs must be a positive integer".into(),
8924                recommended_action: None,
8925            };
8926        }
8927        let started_at_ms = self.clock_now_ms();
8928        let rounds = max_rounds.unwrap_or(20);
8929        let mut aggregate = SyncReport::default();
8930        // §7.2.2: decisions pending at the drive's start are evaluated when it
8931        // reaches idle.
8932        let decisions = match self.prepare_replacements() {
8933            Ok(decisions) => decisions,
8934            Err(message) => {
8935                return SyncOutcome::Failed {
8936                    details: None,
8937                    error_code: "storage.failed".into(),
8938                    message,
8939                    recommended_action: None,
8940                }
8941            }
8942        };
8943        let mut spent = 0;
8944        let mut completed = false;
8945        while spent < rounds {
8946            // §7.8: a deadline that elapsed across completed rounds ends the
8947            // drive like an exhausted round budget; the next drive resumes.
8948            if completed
8949                && timeout_ms.is_some_and(|ms| {
8950                    self.clock_now_ms() >= started_at_ms.saturating_add(ms as i64)
8951                })
8952            {
8953                return SyncOutcome::BudgetExhausted(aggregate);
8954            }
8955            let idle_bound = timeout_ms.map(|ms| (ms, spent));
8956            spent += 1;
8957            let before = self.resume_tokens();
8958            match self.sync_bounded(transport, idle_bound) {
8959                SyncOutcome::Failed {
8960                    error_code,
8961                    message,
8962                    details,
8963                    recommended_action,
8964                } => {
8965                    return SyncOutcome::Failed {
8966                        details,
8967                        error_code,
8968                        message,
8969                        recommended_action,
8970                    };
8971                }
8972                outcome @ SyncOutcome::RealtimeUnavailable { .. } => return outcome,
8973                SyncOutcome::BudgetExhausted(_) => {
8974                    unreachable!("sync() never exhausts a round budget")
8975                }
8976                SyncOutcome::Ok(report) => {
8977                    completed = true;
8978                    if max_rounds.is_none()
8979                        && report.segment_rows_applied > 0
8980                        && !report.bootstrapping.is_empty()
8981                        && before != self.resume_tokens()
8982                    {
8983                        spent = 0;
8984                    }
8985                    // TS parity: a disabled transport or a schema floor ends
8986                    // the loop. The latest round still merges first.
8987                    let stop = !self.transport_enabled() || report.schema_floor.is_some();
8988                    aggregate.merge(&report);
8989                    // §4.5: pull again whenever the response contained
8990                    // commits or segments; resets re-bootstrap; a pending
8991                    // resume token continues paging (§4.7); deferred outbox
8992                    // commits and a raised sync-needed signal push on the next
8993                    // round.
8994                    let more = !stop
8995                        && (!report.bootstrapping.is_empty()
8996                            || report.commits_applied > 0
8997                            || report.segment_rows_applied > 0
8998                            || !report.resets.is_empty()
8999                            || report.deferred_commits > 0
9000                            || self.sync_needed);
9001                    if more {
9002                        continue;
9003                    }
9004                    // §7.2.2: a written decision runs another round, which
9005                    // pushes it.
9006                    match self.settle_replacements(&decisions, &SyncOutcome::Ok(aggregate.clone()))
9007                    {
9008                        Ok(true) if !stop => {}
9009                        Ok(_) => return SyncOutcome::Ok(aggregate),
9010                        Err(message) => {
9011                            return SyncOutcome::Failed {
9012                                details: None,
9013                                error_code: "storage.failed".into(),
9014                                message,
9015                                recommended_action: None,
9016                            }
9017                        }
9018                    }
9019                }
9020            }
9021        }
9022        // §7.8: partial success. The aggregate report is retained and the run
9023        // is explicitly not idle.
9024        SyncOutcome::BudgetExhausted(aggregate)
9025    }
9026
9027    fn process_response(
9028        &mut self,
9029        transport: &mut dyn Transport,
9030        response: Message,
9031        meta: &RequestMeta,
9032    ) -> SyncOutcome {
9033        #[cfg(feature = "bench-internals")]
9034        let _phase = self.benchmark_phases.start(Phase::ResponseApply);
9035        let mut report = SyncReport {
9036            pushed: meta.pushed_ids.len() as u32,
9037            ..SyncReport::default()
9038        };
9039        let mut rejection_details_by_commit: HashMap<String, BTreeMap<i32, RejectionDetails>> =
9040            HashMap::new();
9041        let pushed_ids = meta
9042            .pushed_ids
9043            .iter()
9044            .map(String::as_str)
9045            .collect::<HashSet<_>>();
9046        let mut last_final_push_result_id: Option<String> = None;
9047        for frame in &response.frames {
9048            match frame {
9049                Frame::PushResultDetails {
9050                    client_commit_id,
9051                    entries,
9052                } => {
9053                    let details = rejection_details_by_commit
9054                        .entry(client_commit_id.clone())
9055                        .or_default();
9056                    for entry in entries {
9057                        let parsed = match RejectionDetails::parse(&entry.details.0) {
9058                            Ok(value) => value,
9059                            Err(message) => {
9060                                return SyncOutcome::Failed {
9061                                    details: None,
9062                                    error_code: "sync.invalid_request".to_owned(),
9063                                    message,
9064                                    recommended_action: None,
9065                                };
9066                            }
9067                        };
9068                        details.insert(entry.op_index, parsed);
9069                    }
9070                }
9071                Frame::PushResult {
9072                    client_commit_id,
9073                    status,
9074                    results,
9075                    ..
9076                } if pushed_ids.contains(client_commit_id.as_str())
9077                    && Self::push_result_is_final(*status, results) =>
9078                {
9079                    last_final_push_result_id = Some(client_commit_id.clone());
9080                }
9081                _ => {}
9082            }
9083        }
9084        let mut frames = response.frames.into_iter().peekable();
9085        match frames.next() {
9086            Some(Frame::RespHeader {
9087                required_schema_version,
9088                latest_schema_version,
9089                log_epoch,
9090                reset_required,
9091            }) => {
9092                if let Some(required) = required_schema_version {
9093                    // §1.6 schema-floor response: nothing else is processed;
9094                    // stop syncing and surface the upgrade requirement.
9095                    let floor = SchemaFloor {
9096                        required_schema_version: Some(required),
9097                        latest_schema_version,
9098                    };
9099                    if let Err(message) = self.set_schema_floor(Some(floor.clone())) {
9100                        return SyncOutcome::Failed {
9101                            details: None,
9102                            error_code: "storage.failed".to_owned(),
9103                            message,
9104                            recommended_action: None,
9105                        };
9106                    }
9107                    report.schema_floor = Some(floor);
9108                    return SyncOutcome::Ok(report);
9109                }
9110                if reset_required {
9111                    if frames.next().is_some() {
9112                        return SyncOutcome::Failed {
9113                            details: None,
9114                            error_code: "client.invalid_host_response".to_owned(),
9115                            message: "log-epoch reset response contains body frames".to_owned(),
9116                            recommended_action: None,
9117                        };
9118                    }
9119                    match self.run_log_epoch_reset(&log_epoch) {
9120                        Ok(resets) => {
9121                            report.resets = resets;
9122                            return SyncOutcome::Ok(report);
9123                        }
9124                        Err(message) => {
9125                            return SyncOutcome::Failed {
9126                                details: None,
9127                                error_code: "sync.local_corrupt".to_owned(),
9128                                message,
9129                                recommended_action: None,
9130                            };
9131                        }
9132                    }
9133                }
9134                let stored_epoch = match self.get_meta(LOG_EPOCH_KEY) {
9135                    Ok(stored) => stored,
9136                    Err(message) => {
9137                        return SyncOutcome::Failed {
9138                            details: None,
9139                            error_code: "storage.failed".to_owned(),
9140                            message,
9141                            recommended_action: None,
9142                        };
9143                    }
9144                };
9145                // §2.1 adoption: the epoch is stored before any section applies.
9146                if stored_epoch.is_none() {
9147                    if let Err(message) = self.set_meta(LOG_EPOCH_KEY, &log_epoch) {
9148                        return SyncOutcome::Failed {
9149                            details: None,
9150                            error_code: "storage.failed".to_owned(),
9151                            message,
9152                            recommended_action: None,
9153                        };
9154                    }
9155                } else if stored_epoch.as_deref() != Some(log_epoch.as_str()) {
9156                    return SyncOutcome::Failed {
9157                        details: None,
9158                        error_code: "client.invalid_host_response".to_owned(),
9159                        message: "server changed logEpoch without requiring a reset".to_owned(),
9160                        recommended_action: None,
9161                    };
9162                }
9163            }
9164            _ => {
9165                return SyncOutcome::Failed {
9166                    details: None,
9167                    error_code: "sync.invalid_request".to_owned(),
9168                    message: "response does not start with RESP_HEADER".to_owned(),
9169                    recommended_action: None,
9170                };
9171            }
9172        }
9173
9174        if let Err(message) = self.begin_coalesced_apply() {
9175            return SyncOutcome::Failed {
9176                details: None,
9177                error_code: "storage.failed".into(),
9178                message,
9179                recommended_action: None,
9180            };
9181        }
9182        let mut failure: Option<(String, String, Option<String>, Option<Value>)> = None;
9183        // §4.5 client application: the units whose persisted echo a live
9184        // change must match, fixed at LIVE_START after every SUB_END persisted.
9185        let mut live: Option<(LiveStatus, String, Vec<UnitRecord>)> = None;
9186        let mut live_ended = false;
9187        while let Some(frame) = frames.next() {
9188            match frame {
9189                frame @ Frame::PushResult { .. } => {
9190                    let successful = matches!(
9191                        &frame,
9192                        Frame::PushResult {
9193                            status: PushStatus::Applied | PushStatus::Cached,
9194                            ..
9195                        }
9196                    );
9197                    let mut results = vec![frame];
9198                    if successful {
9199                        while matches!(
9200                            frames.peek(),
9201                            Some(Frame::PushResult {
9202                                status: PushStatus::Applied | PushStatus::Cached,
9203                                ..
9204                            })
9205                        ) {
9206                            if let Some(next) = frames.next() {
9207                                results.push(next);
9208                            }
9209                        }
9210                    }
9211                    if self
9212                        .handle_push_results(
9213                            &results,
9214                            &pushed_ids,
9215                            &rejection_details_by_commit,
9216                            &mut report,
9217                            last_final_push_result_id.as_deref(),
9218                        )
9219                        .is_err()
9220                    {
9221                        failure = Some((
9222                            "client.outcome_persistence_failed".to_owned(),
9223                            "local commit outcome could not be persisted".to_owned(),
9224                            None,
9225                            None,
9226                        ));
9227                        break;
9228                    }
9229                }
9230                Frame::PushResultDetails { .. } => {}
9231                Frame::SubStart {
9232                    id,
9233                    status,
9234                    reason_code,
9235                    bootstrap,
9236                    fenced,
9237                } => {
9238                    // §1.4 rule 5: an ERROR closes the section; its body still
9239                    // applied, its SUB_END values are never persisted.
9240                    let mut body = Vec::new();
9241                    let mut sub_end: Option<(i64, Option<String>)> = None;
9242                    let mut error: Option<Frame> = None;
9243                    for inner in frames.by_ref() {
9244                        match inner {
9245                            Frame::SubEnd {
9246                                next_cursor,
9247                                bootstrap_state,
9248                            } => {
9249                                sub_end = Some((next_cursor, bootstrap_state.map(|r| r.0)));
9250                                break;
9251                            }
9252                            inner @ Frame::Error { .. } => {
9253                                error = Some(inner);
9254                                break;
9255                            }
9256                            Frame::Unknown { .. } => {}
9257                            other => body.push(other),
9258                        }
9259                    }
9260                    let ended = sub_end.is_some();
9261                    let section = SectionStart {
9262                        id,
9263                        status,
9264                        reason_code,
9265                        bootstrap,
9266                        fenced,
9267                    };
9268                    let completed = match self.process_section(
9269                        transport,
9270                        &section,
9271                        body,
9272                        sub_end,
9273                        meta,
9274                        &mut report,
9275                    ) {
9276                        Err(SectionError::Abort(code, message)) => {
9277                            failure = Some((code, message, None, None));
9278                            break;
9279                        }
9280                        Err(SectionError::Transport(error)) => {
9281                            failure = Some((error.code, error.message, None, error.details));
9282                            break;
9283                        }
9284                        Ok(completed) => completed,
9285                        Err(_) => false,
9286                    };
9287                    if ended {
9288                        // The counters stay: `complete` and `failed` report them.
9289                        // §7.7: a completed bootstrap ends with the unit total
9290                        // equal to the rows processed.
9291                        self.progress.update(|p| {
9292                            p.unit_id = None;
9293                            p.table = None;
9294                            p.segment_id = None;
9295                            if completed && p.unit_rows_total.is_some() {
9296                                p.unit_rows_total = p.unit_rows_processed;
9297                            }
9298                        });
9299                    }
9300                    match error {
9301                        Some(Frame::Error {
9302                            code,
9303                            message,
9304                            recommended_action,
9305                            details,
9306                            ..
9307                        }) => {
9308                            failure = Some(Self::error_frame_failure(
9309                                code,
9310                                message,
9311                                recommended_action,
9312                                details,
9313                            ));
9314                            break;
9315                        }
9316                        Some(_) => unreachable!("only an ERROR closes a section early"),
9317                        None if !ended => {
9318                            failure = Some((
9319                                "sync.invalid_request".to_owned(),
9320                                "unit section without SUB_END".to_owned(),
9321                                None,
9322                                None,
9323                            ));
9324                            break;
9325                        }
9326                        None => {}
9327                    }
9328                }
9329                Frame::LiveStart {
9330                    status,
9331                    reason_code,
9332                } => {
9333                    let receivers = match self.stored(load_units(&self.conn)) {
9334                        Ok(units) => units
9335                            .into_iter()
9336                            .filter(|unit| {
9337                                unit.status == UnitStatus::Active
9338                                    && unit.echo.is_some()
9339                                    && !unit.bootstrapping()
9340                            })
9341                            .collect(),
9342                        Err(message) => {
9343                            failure = Some(("storage.failed".to_owned(), message, None, None));
9344                            break;
9345                        }
9346                    };
9347                    live = Some((status, reason_code, receivers));
9348                }
9349                Frame::Commit {
9350                    commit_seq,
9351                    tables,
9352                    changes,
9353                    ..
9354                } => {
9355                    let Some((LiveStatus::Active, _, receivers)) = &live else {
9356                        continue;
9357                    };
9358                    let receivers = receivers.clone();
9359                    match self.apply_live_commit(
9360                        &receivers,
9361                        &tables,
9362                        changes,
9363                        commit_seq,
9364                        AppliedServerCommitSource::Pull,
9365                    ) {
9366                        Ok(applied) => report.commits_applied += u32::from(applied),
9367                        Err(error) => {
9368                            failure = Some((error.code, error.message, None, error.details));
9369                            break;
9370                        }
9371                    }
9372                }
9373                Frame::LiveEnd {
9374                    through_seq,
9375                    echo_digest,
9376                } => {
9377                    let Some((status, reason_code, _)) = live.take() else {
9378                        continue;
9379                    };
9380                    live_ended = true;
9381                    live = Some((status, String::new(), Vec::new()));
9382                    if let Err(message) = self.finish_live_section(
9383                        status,
9384                        &reason_code,
9385                        through_seq,
9386                        echo_digest,
9387                        &meta.registration_digest,
9388                        &mut report,
9389                    ) {
9390                        failure = Some(("storage.failed".to_owned(), message, None, None));
9391                        break;
9392                    }
9393                }
9394                Frame::Lease {
9395                    lease_id,
9396                    expires_at_ms,
9397                } => {
9398                    // §7.3.5: persist the opaque lease; a fresh lease clears
9399                    // any prior lease error (the outage/revocation is over).
9400                    if let Err(message) = self.set_lease_state(Some(LeaseState {
9401                        lease_id: Some(lease_id),
9402                        expires_at_ms: Some(expires_at_ms),
9403                        error_code: None,
9404                    })) {
9405                        failure = Some(("storage.failed".to_owned(), message, None, None));
9406                        break;
9407                    }
9408                }
9409                Frame::Error {
9410                    code,
9411                    message,
9412                    recommended_action,
9413                    details,
9414                    ..
9415                } => {
9416                    // §1.6: the whole request failed; already-completed
9417                    // units keep their applied data and cursors.
9418                    failure = Some(Self::error_frame_failure(
9419                        code,
9420                        message,
9421                        recommended_action,
9422                        details,
9423                    ));
9424                    break;
9425                }
9426                Frame::Unknown { .. } => {}
9427                // §1.6: a building server's keepalive carries nothing to apply.
9428                Frame::Keepalive => {}
9429                _ => {
9430                    failure = Some((
9431                        "sync.invalid_request".to_owned(),
9432                        "unexpected frame in response".to_owned(),
9433                        None,
9434                        None,
9435                    ));
9436                    break;
9437                }
9438            }
9439        }
9440
9441        let applied = match failure {
9442            Some(failure) => Err(failure),
9443            // Local apply boundaries already reconciled their affected overlay.
9444            // Refuse round success if any remaining dirty work fails to
9445            // reconcile. §4.8 E1: the push half may have drained commits that
9446            // pinned rows of a shrunk window unit, so retry deferred evictions.
9447            None => self
9448                .rebuild_overlay_if_dirty()
9449                .and_then(|()| self.drain_pending_evictions())
9450                .map_err(|message| ("storage.failed".to_owned(), message, None, None)),
9451        };
9452        // §1.4 rule 4: frames completed before a failure stay committed; a
9453        // failed COMMIT of the coalesced transaction fails the round.
9454        if let Err(message) = self.finish_coalesced_apply() {
9455            return SyncOutcome::Failed {
9456                details: None,
9457                error_code: "storage.failed".into(),
9458                message,
9459                recommended_action: None,
9460            };
9461        }
9462        if let Err((error_code, message, recommended_action, details)) = applied {
9463            return SyncOutcome::Failed {
9464                details,
9465                error_code,
9466                message,
9467                recommended_action,
9468            };
9469        }
9470        let catching_up = match self.stored(load_units(&self.conn)) {
9471            Ok(units) => units
9472                .into_iter()
9473                .filter(|unit| unit.status == UnitStatus::Active && unit.cursor.is_some())
9474                .map(|unit| unit.id)
9475                .collect::<Vec<_>>(),
9476            Err(message) => {
9477                return SyncOutcome::Failed {
9478                    details: None,
9479                    error_code: "storage.failed".into(),
9480                    message,
9481                    recommended_action: None,
9482                }
9483            }
9484        };
9485        // §7.1: a response without LIVE_END, one answered `unregistered` or
9486        // `reset`, or a unit still catching up leaves sync-needed raised;
9487        // after `unregistered` the next round starts at once.
9488        let status = live.map(|(status, _, _)| status);
9489        if !live_ended || status != Some(LiveStatus::Active) || !catching_up.is_empty() {
9490            self.set_sync_needed(true, status == Some(LiveStatus::Unregistered));
9491        }
9492        self.ack_after_pull(transport);
9493        // §7.4.5: the reset is over once the first post-reset pull round
9494        // leaves no unit catching up: the tables are rebuilt and current.
9495        if self.upgrading && catching_up.is_empty() {
9496            self.set_upgrading(false);
9497        }
9498        report.bootstrapping = catching_up;
9499        SyncOutcome::Ok(report)
9500    }
9501
9502    /// §1.6 / §10.1: the failure an in-band `ERROR` frame reports.
9503    fn error_frame_failure(
9504        code: String,
9505        message: String,
9506        recommended_action: String,
9507        details: Option<RawJson>,
9508    ) -> (String, String, Option<String>, Option<Value>) {
9509        let recommended_action = (!recommended_action.is_empty()).then_some(recommended_action);
9510        let details = details
9511            .and_then(|details| serde_json::from_str::<Value>(&details.0).ok())
9512            .filter(Value::is_object);
9513        (code, message, recommended_action, details)
9514    }
9515
9516    /// §4.5 at `LIVE_END` of a pull: join rule J for an `active` section, the
9517    /// §4.6 replica reset for `reset`, nothing for `unregistered`; then the
9518    /// replica cursor, the echo digest and the digest of the list the server
9519    /// now stores.
9520    fn finish_live_section(
9521        &mut self,
9522        status: LiveStatus,
9523        reason_code: &str,
9524        through_seq: i64,
9525        echo_digest: Option<[u8; 32]>,
9526        registration_digest: &str,
9527        report: &mut SyncReport,
9528    ) -> Result<(), String> {
9529        let rollback = self.capture();
9530        let resets = report.resets.len();
9531        self.begin_observation("syncular_live_end")?;
9532        let finished = (|| {
9533            let mut batch = ChangeAccumulator::default();
9534            let before = self.stored(replica_cursor(&self.conn))?;
9535            let mut cursor = if status == LiveStatus::Unregistered {
9536                before
9537            } else {
9538                -1
9539            };
9540            let joined = before.max(through_seq);
9541            for unit in self.stored(load_units(&self.conn))? {
9542                if unit.status != UnitStatus::Active || unit.bootstrapping() {
9543                    continue;
9544                }
9545                match status {
9546                    // §4.6 replica reset: every live unit bootstraps again;
9547                    // post-bootstrap units keep their cursors.
9548                    LiveStatus::Reset if unit.cursor.is_none() => {
9549                        for (base_key, value) in self.stored(window_refs(&self.conn, &unit.id))? {
9550                            batch.window(&base_key, &value);
9551                        }
9552                        report.resets.push(unit.id.clone());
9553                        self.stored(save_unit(
9554                            &self.conn,
9555                            &UnitRecord {
9556                                reason_code: Some(reason_code.to_owned()),
9557                                cursor: Some(-1),
9558                                seeded: Vec::new(),
9559                                ..unit
9560                            },
9561                        ))?;
9562                    }
9563                    // §4.5 join rule J.
9564                    LiveStatus::Active => {
9565                        let next = unit.cursor.unwrap_or(joined).max(through_seq);
9566                        if next >= joined {
9567                            cursor = joined;
9568                        }
9569                        if unit.cursor.is_some() {
9570                            self.stored(save_unit(
9571                                &self.conn,
9572                                &UnitRecord {
9573                                    cursor: (next < joined).then_some(next),
9574                                    ..unit
9575                                },
9576                            ))?;
9577                        }
9578                    }
9579                    _ => {}
9580                }
9581            }
9582            self.set_meta(REPLICA_CURSOR_KEY, &cursor.to_string())?;
9583            self.set_meta(
9584                ECHO_DIGEST_KEY,
9585                &echo_digest
9586                    .map(|digest| bytes_to_hex(&digest))
9587                    .unwrap_or_default(),
9588            )?;
9589            if status == LiveStatus::Unregistered {
9590                self.delete_meta(REGISTERED_DIGEST_KEY)?;
9591            } else {
9592                self.set_meta(REGISTERED_DIGEST_KEY, registration_digest)?;
9593            }
9594            self.finish_observation("syncular_live_end", batch)
9595        })();
9596        if finished.is_err() {
9597            self.rollback_observation("syncular_live_end");
9598            self.restore(rollback);
9599            report.resets.truncate(resets);
9600        }
9601        finished
9602    }
9603
9604    // -- push results (§6.3, §7.2) ------------------------------------------------
9605
9606    fn handle_push_results(
9607        &mut self,
9608        frames: &[Frame],
9609        pushed_ids: &HashSet<&str>,
9610        rejection_details_by_commit: &HashMap<String, BTreeMap<i32, RejectionDetails>>,
9611        report: &mut SyncReport,
9612        last_final_push_result_id: Option<&str>,
9613    ) -> Result<(), String> {
9614        self.begin_observation("syncular_push_results")?;
9615        let rollback = self.capture();
9616        let conflict_count = self.conflicts.len();
9617        let applied_count = report.applied.len();
9618        let rejected_count = report.rejected.len();
9619        let retryable_count = report.retryable.len();
9620        let mut drained = Vec::new();
9621        let mut batch = ChangeAccumulator::default();
9622        let persisted = (|| {
9623            for frame in frames {
9624                let Frame::PushResult {
9625                    client_commit_id,
9626                    status,
9627                    results,
9628                    commit_seq,
9629                } = frame
9630                else {
9631                    return Err("unexpected acknowledgement frame".to_owned());
9632                };
9633                if !pushed_ids.contains(client_commit_id.as_str()) {
9634                    continue;
9635                }
9636                let Some(index) = self
9637                    .outbox
9638                    .iter()
9639                    .position(|commit| commit.client_commit_id == *client_commit_id)
9640                else {
9641                    continue;
9642                };
9643                let status = *status;
9644                let rejection_details = rejection_details_by_commit.get(client_commit_id);
9645                let operations = self.outbox[index].ops.clone();
9646                match status {
9647                    PushStatus::Applied | PushStatus::Cached => {
9648                        // §7.2: a lost ack replays as `cached` — proceed as if the
9649                        // ack had arrived.
9650                        let sequence =
9651                            commit_seq.ok_or_else(|| "ACK lacks a commit sequence".to_owned())?;
9652                        let affected: HashSet<_> = operations
9653                            .iter()
9654                            .map(|op| (op.table.as_str(), op.row_id.as_str()))
9655                            .collect();
9656                        let prior = self.acknowledged_operations(Some(&affected), None)?;
9657                        for (op_index, operation) in operations.iter().enumerate() {
9658                            let delivered: bool = self.conn.query_row("SELECT EXISTS(SELECT 1 FROM _syncular_row_deliveries WHERE tbl=?1 AND id=?2 AND commit_seq>=?3)", rusqlite::params![operation.table, operation.row_id, sequence], |row| row.get(0))
9659                                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9660                            if delivered {
9661                                continue;
9662                            }
9663                            let table = self
9664                                .schema
9665                                .table(&operation.table)
9666                                .ok_or_else(|| "sync.unknown_table".to_owned())?;
9667                            let mut intended: Map<String, Value> = self
9668                                .stored_row(table, &operation.row_id, true)?
9669                                .map(|(row, _)| {
9670                                    table
9671                                        .columns
9672                                        .iter()
9673                                        .zip(&row)
9674                                        .map(|(column, value)| {
9675                                            (column.name.clone(), column_value_to_json(value))
9676                                        })
9677                                        .collect()
9678                                })
9679                                .unwrap_or_default();
9680                            let mut scope_image = intended.clone();
9681                            for op in prior
9682                                .iter()
9683                                .chain(
9684                                    self.failed_commits
9685                                        .iter()
9686                                        .flat_map(|(commit, _)| &commit.ops),
9687                                )
9688                                .chain(
9689                                    self.outbox
9690                                        .iter()
9691                                        .take(index)
9692                                        .flat_map(|commit| &commit.ops),
9693                                )
9694                                .chain(operations.iter().take(op_index + 1))
9695                            {
9696                                if op.table != operation.table || op.row_id != operation.row_id {
9697                                    continue;
9698                                }
9699                                if let Some(values) = &op.values {
9700                                    intended.extend(values.clone());
9701                                } else {
9702                                    scope_image = intended.clone();
9703                                    intended.clear();
9704                                }
9705                            }
9706                            self.conn.execute("INSERT INTO _syncular_acked_rows(commit_id,idx,tbl,id,commit_seq,op_json,intent_json) VALUES (?1,?2,?3,?4,?5,?6,?7)",
9707                                rusqlite::params![client_commit_id, op_index as i64, operation.table, operation.row_id, sequence,
9708                                    serde_json::to_string(&CommitOperation::from(operation)).map_err(|_| "sync.local_corrupt".to_owned())?,
9709                                    serde_json::to_string(if operation.upsert { &intended } else { &scope_image }).map_err(|_| "sync.local_corrupt".to_owned())?])
9710                                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9711                        }
9712                        let journal_results = results
9713                            .iter()
9714                            .map(|result| {
9715                                let op_index = match result {
9716                                    OpResult::Applied { op_index }
9717                                    | OpResult::Conflict { op_index, .. }
9718                                    | OpResult::Error { op_index, .. } => *op_index,
9719                                };
9720                                CommitOperationOutcome::Applied { op_index }
9721                            })
9722                            .collect::<Vec<_>>();
9723                        let outcome_status = if status == PushStatus::Applied {
9724                            CommitOutcomeStatus::Applied
9725                        } else {
9726                            CommitOutcomeStatus::Cached
9727                        };
9728                        let persisted = self
9729                            .persist_commit_outcome(
9730                                client_commit_id,
9731                                outcome_status,
9732                                &journal_results,
9733                                None,
9734                            )
9735                            .and_then(|()| self.delete_outbox_persisted(client_commit_id));
9736                        persisted?;
9737                        self.sync_needed = true;
9738                        batch.status = true;
9739                        batch.outcomes = true;
9740                    }
9741                    PushStatus::Rejected => {
9742                        if results.iter().any(|result| {
9743                            matches!(
9744                                result,
9745                                OpResult::Error {
9746                                    code,
9747                                    retryable: true,
9748                                    ..
9749                                } if code == "sync.idempotency_cache_miss"
9750                            )
9751                        }) {
9752                            // §6.3/§7.2: a serving failure, not an outcome — keep the
9753                            // exact commit queued for an identical retry.
9754                            report.retryable.push(client_commit_id.to_owned());
9755                            continue;
9756                        }
9757
9758                        let mut journal_results = Vec::with_capacity(results.len());
9759                        let mut conflicts = Vec::new();
9760                        let mut rejections = Vec::new();
9761                        for result in results {
9762                            match result {
9763                                OpResult::Applied { op_index } => {
9764                                    journal_results.push(CommitOperationOutcome::Applied {
9765                                        op_index: *op_index,
9766                                    });
9767                                }
9768                                OpResult::Conflict {
9769                                    op_index,
9770                                    code,
9771                                    message,
9772                                    server_version,
9773                                    server_row,
9774                                    conflict_columns: conflict_columns_bitmap,
9775                                } => {
9776                                    let operation = operations
9777                                        .get(*op_index as usize)
9778                                        .map(CommitOperation::from);
9779                                    let (table, row_id) = operation
9780                                        .as_ref()
9781                                        .map(|op| (op.table.clone(), op.row_id.clone()))
9782                                        .unwrap_or_default();
9783                                    let server_row_json = self
9784                                        .schema
9785                                        .table(&table)
9786                                        .and_then(|t| {
9787                                            decode_row_bytes(t, server_row, &self.encryption)
9788                                                .ok()
9789                                                .map(|(row, _)| (t, row))
9790                                        })
9791                                        .map(|(t, row)| {
9792                                            let mut map = Map::new();
9793                                            for (i, column) in t.columns.iter().enumerate() {
9794                                                map.insert(
9795                                                    column.name.clone(),
9796                                                    column_value_to_json(
9797                                                        row.get(i).unwrap_or(&None),
9798                                                    ),
9799                                                );
9800                                            }
9801                                            map
9802                                        })
9803                                        .unwrap_or_default();
9804                                    // §6.3: decode the presence-layout bitmap
9805                                    // into the marked column names.
9806                                    let conflict_columns = self
9807                                        .schema
9808                                        .table(&table)
9809                                        .map(|t| {
9810                                            t.columns
9811                                                .iter()
9812                                                .enumerate()
9813                                                .filter(|(index, _)| {
9814                                                    conflict_columns_bitmap
9815                                                        .get(index / 8)
9816                                                        .is_some_and(|byte| {
9817                                                            byte & (1 << (index % 8)) != 0
9818                                                        })
9819                                                })
9820                                                .map(|(_, column)| column.name.clone())
9821                                                .collect::<Vec<_>>()
9822                                        })
9823                                        .unwrap_or_default();
9824                                    let conflict = ConflictRecord {
9825                                        client_commit_id: client_commit_id.to_owned(),
9826                                        op_index: *op_index,
9827                                        table,
9828                                        row_id,
9829                                        code: code.clone(),
9830                                        message: message.clone(),
9831                                        server_version: *server_version,
9832                                        server_row: server_row_json,
9833                                        conflict_columns,
9834                                        operation,
9835                                    };
9836                                    journal_results.push(CommitOperationOutcome::Conflict {
9837                                        conflict: conflict.clone(),
9838                                    });
9839                                    conflicts.push(conflict);
9840                                }
9841                                OpResult::Error {
9842                                    op_index,
9843                                    code,
9844                                    message,
9845                                    retryable,
9846                                } => {
9847                                    let rejection = RejectionRecord {
9848                                        client_commit_id: client_commit_id.to_owned(),
9849                                        op_index: *op_index,
9850                                        code: code.clone(),
9851                                        message: message.clone(),
9852                                        retryable: *retryable,
9853                                        details: rejection_details
9854                                            .and_then(|details| details.get(op_index))
9855                                            .cloned(),
9856                                        operation: operations
9857                                            .get(*op_index as usize)
9858                                            .map(CommitOperation::from),
9859                                    };
9860                                    journal_results.push(CommitOperationOutcome::Error {
9861                                        rejection: rejection.clone(),
9862                                    });
9863                                    rejections.push(rejection);
9864                                }
9865                            }
9866                        }
9867                        if self.retain_failed_commits {
9868                            self.retain_failed_commit(client_commit_id, &operations)?;
9869                        }
9870                        let outcome_status = if conflicts.is_empty() {
9871                            CommitOutcomeStatus::Rejected
9872                        } else {
9873                            CommitOutcomeStatus::Conflict
9874                        };
9875                        let persisted = self
9876                            .persist_commit_outcome(
9877                                client_commit_id,
9878                                outcome_status,
9879                                &journal_results,
9880                                Some(&operations),
9881                            )
9882                            .and_then(|()| self.delete_outbox_persisted(client_commit_id));
9883                        persisted?;
9884                        batch.conflicts = !conflicts.is_empty();
9885                        batch.rejections = !rejections.is_empty();
9886                        batch.status = true;
9887                        batch.outcomes = true;
9888                        self.conflicts.extend(conflicts);
9889                        self.rejections.extend(rejections);
9890                    }
9891                }
9892                drained.push(self.outbox.remove(index));
9893                if batch.status {
9894                    for operation in &operations {
9895                        if !self.record_row_scopes(
9896                            &mut batch,
9897                            &operation.table,
9898                            &operation.row_id,
9899                            false,
9900                        ) {
9901                            batch.table(&operation.table);
9902                        }
9903                    }
9904                }
9905                match status {
9906                    PushStatus::Applied | PushStatus::Cached => {
9907                        report.applied.push(client_commit_id.to_owned())
9908                    }
9909                    PushStatus::Rejected => report.rejected.push(client_commit_id.to_owned()),
9910                }
9911            }
9912            if batch.outcomes && frames.iter().any(|frame| matches!(frame, Frame::PushResult { client_commit_id, .. } if Some(client_commit_id.as_str()) == last_final_push_result_id)) {
9913                self.prune_commit_outcomes()?;
9914            }
9915            self.reconcile_rows(&drained)?;
9916            self.rebuild_overlay_if_dirty()?;
9917            self.finish_observation("syncular_push_results", batch)
9918        })();
9919        if let Err(error) = persisted {
9920            self.rollback_observation("syncular_push_results");
9921            self.restore(rollback);
9922            report.applied.truncate(applied_count);
9923            report.rejected.truncate(rejected_count);
9924            report.retryable.truncate(retryable_count);
9925            return Err(error);
9926        }
9927        if !rollback.sync_needed && self.sync_needed {
9928            self.sync_intent_queue.push_back(SyncIntent::Interactive);
9929        }
9930        report.conflicts += (self.conflicts.len() - conflict_count) as u32;
9931        Ok(())
9932    }
9933
9934    fn push_result_is_final(status: PushStatus, results: &[OpResult]) -> bool {
9935        status != PushStatus::Rejected
9936            || !results.iter().any(|result| {
9937                matches!(
9938                    result,
9939                    OpResult::Error {
9940                        code,
9941                        retryable: true,
9942                        ..
9943                    } if code == "sync.idempotency_cache_miss"
9944                )
9945            })
9946    }
9947
9948    // -- unit sections and the live section ------------------------------------------
9949
9950    /// One unit section (§4.4): apply its segments, then persist its
9951    /// `SUB_END` (§4.6 reset, §3.3 revocation and purge, the echo, cursor and
9952    /// resume token). A section of a unit the app released while the round was
9953    /// in flight is skipped; a fail-closed first page fails the unit and skips
9954    /// the rest of its section (§5.6). Returns whether its `SUB_END` completed
9955    /// a bootstrap (§7.7).
9956    fn process_section(
9957        &mut self,
9958        transport: &mut dyn Transport,
9959        start: &SectionStart,
9960        body: Vec<Frame>,
9961        sub_end: Option<(i64, Option<String>)>,
9962        meta: &RequestMeta,
9963        report: &mut SyncReport,
9964    ) -> Result<bool, SectionError> {
9965        let storage = |message| SectionError::Abort("storage.failed".to_owned(), message);
9966        let unit = self
9967            .stored(get_unit(&self.conn, &start.id))
9968            .map_err(storage)?
9969            .filter(|unit| unit.status == UnitStatus::Active);
9970        // §7.7: a section boundary ends the previous payload; the section's
9971        // UNIT_EXTENT starts the unit counters in the same snapshot.
9972        let extent = match (&unit, body.first()) {
9973            (
9974                Some(_),
9975                Some(Frame::UnitExtent {
9976                    rows_total,
9977                    rows_done,
9978                }),
9979            ) => u64::try_from(*rows_total)
9980                .ok()
9981                .zip(u64::try_from(*rows_done).ok()),
9982            _ => None,
9983        };
9984        self.progress.update(|p| {
9985            p.phase = ProgressPhase::Import;
9986            p.unit_id = unit.as_ref().map(|unit| unit.id.clone());
9987            p.unit_rows_total = extent.map(|(total, _)| total);
9988            p.unit_rows_processed = extent.map(|(_, done)| done);
9989            p.table = None;
9990            p.segment_id = None;
9991            p.bytes_received = 0;
9992            p.bytes_total = None;
9993            p.rows_processed = 0;
9994            p.rows_total = None;
9995            p.import_write_ms = None;
9996            p.import_commit_ms = None;
9997        });
9998        let Some(unit) = unit else {
9999            return Ok(false);
10000        };
10001        let echo: Vec<EchoRecord> = start
10002            .fenced
10003            .iter()
10004            .map(|record| EchoRecord {
10005                table: record.table.clone(),
10006                status: record.status.name().to_owned(),
10007                scopes: record.scopes.iter().cloned().collect(),
10008            })
10009            .collect();
10010        // §5.6: rows earlier attempts at this position applied.
10011        let applied_record = self
10012            .stored(applied_rows(&self.conn, &unit.id))
10013            .map_err(storage)?;
10014        // §5.6: a resumed bootstrap keeps the tables it already seeded.
10015        let mut seeded: BTreeSet<String> = if unit.bootstrap_state.is_none() {
10016            BTreeSet::new()
10017        } else {
10018            unit.seeded.iter().cloned().collect()
10019        };
10020        let applied = (|| {
10021            for frame in body {
10022                match frame {
10023                    Frame::SegmentInline { payload } => {
10024                        let segment = decode_rows_segment(&payload).map_err(|e| {
10025                            SectionError::Abort(e.code.as_str().to_owned(), e.detail)
10026                        })?;
10027                        self.progress.update(|p| {
10028                            p.phase = ProgressPhase::Import;
10029                            p.unit_id = Some(unit.id.clone());
10030                            p.table = Some(segment.table.clone());
10031                            p.segment_id = None;
10032                            p.bytes_received = payload.len() as u64;
10033                            p.bytes_total = Some(payload.len() as u64);
10034                            p.rows_processed = 0;
10035                            p.rows_total =
10036                                Some(segment.blocks.iter().map(|block| block.len() as u64).sum());
10037                            p.import_write_ms = None;
10038                            p.import_commit_ms = None;
10039                        });
10040                        let (pair, clear, effective) =
10041                            self.segment_pair(&unit, &echo, &seeded, &segment.table)?;
10042                        report.segment_rows_applied +=
10043                            self.apply_segment(&pair, &segment, clear, &effective, &unit.id)?;
10044                        seeded.insert(segment.table);
10045                    }
10046                    Frame::SegmentRef {
10047                        segment_id,
10048                        byte_length,
10049                        media_type,
10050                        table,
10051                        row_count,
10052                        as_of_commit_seq,
10053                        scope_digest,
10054                        row_cursor,
10055                        next_row_cursor,
10056                        url,
10057                        url_expires_at_ms,
10058                        ..
10059                    } => {
10060                        // §4.2: reject a descriptor whose mediaType was not
10061                        // advertised — never skip or guess.
10062                        let advertised = match media_type {
10063                            MediaType::Rows => {
10064                                meta.accept & ACCEPT_EXTERNAL_ROWS != 0
10065                                    || meta.accept & ACCEPT_INLINE_ROWS != 0
10066                            }
10067                            MediaType::Sqlite => meta.accept & ACCEPT_SQLITE != 0,
10068                        };
10069                        if !advertised {
10070                            return Err(SectionError::Abort(
10071                                "sync.invalid_request".to_owned(),
10072                                format!(
10073                                    "SEGMENT_REF mediaType {} was not advertised in accept (§4.2)",
10074                                    media_type.name()
10075                                ),
10076                            ));
10077                        }
10078                        // §5.6 applied-segment rule: a partly applied image
10079                        // already ran its table's first-page clear.
10080                        let resumed = applied_record.get(&segment_id).copied();
10081                        if resumed.is_some() {
10082                            seeded.insert(table.clone());
10083                        }
10084                        if resumed == Some(row_count) {
10085                            // §7.7: an earlier attempt imported this payload;
10086                            // its rows still count toward the unit.
10087                            self.progress.update(|p| {
10088                                p.phase = ProgressPhase::Import;
10089                                p.unit_id = Some(unit.id.clone());
10090                                p.table = Some(table.clone());
10091                                p.segment_id = Some(segment_id.clone());
10092                                p.bytes_received = 0;
10093                                p.bytes_total = None;
10094                                p.rows_processed = 0;
10095                                p.rows_total = u64::try_from(row_count).ok();
10096                                p.import_write_ms = None;
10097                                p.import_commit_ms = None;
10098                            });
10099                            self.progress.update(|p| {
10100                                p.rows_processed = u64::try_from(row_count).unwrap_or(0)
10101                            });
10102                            continue;
10103                        }
10104                        if meta
10105                            .deferred
10106                            .contains(&(unit.id.clone(), segment_id.clone()))
10107                        {
10108                            return Err(SectionError::Deferred);
10109                        }
10110                        self.progress.update(|p| {
10111                            p.phase = ProgressPhase::Download;
10112                            p.unit_id = Some(unit.id.clone());
10113                            p.table = Some(table.clone());
10114                            p.segment_id = Some(segment_id.clone());
10115                            p.bytes_received = 0;
10116                            p.bytes_total = u64::try_from(byte_length).ok();
10117                            p.rows_processed = 0;
10118                            p.rows_total = u64::try_from(row_count).ok();
10119                            p.import_write_ms = None;
10120                            p.import_commit_ms = None;
10121                        });
10122                        let observer = self.progress.clone();
10123                        let mut on_progress = |bytes| observer.update(|p| p.bytes_received = bytes);
10124                        let bytes = if let Some(url) = url {
10125                            // §5.4: a url-carrying descriptor MUST be fetched
10126                            // from that URL; failure invalidates the whole
10127                            // descriptor (no fall-through to §5.5 — re-pull
10128                            // recovers, §1.4 rule 5).
10129                            if meta.accept & ACCEPT_SIGNED_URLS == 0 {
10130                                return Err(SectionError::Abort(
10131                                    "sync.invalid_request".to_owned(),
10132                                    "SEGMENT_REF carries a url but accept bit 3 was not advertised (§5.4)"
10133                                        .to_owned(),
10134                                ));
10135                            }
10136                            // §5.4: MUST NOT start a fetch at/past expiry.
10137                            if url_expires_at_ms.is_some_and(|exp| {
10138                                exp <= transport
10139                                    .segment_fetch_started_at(&url)
10140                                    .unwrap_or_else(|| self.clock_now_ms())
10141                            }) {
10142                                return Err(SectionError::Abort(
10143                                    "sync.segment_expired".to_owned(),
10144                                    format!(
10145                                        "signed URL for segment {segment_id} expired before fetch — re-pull mints fresh descriptors (§5.4)"
10146                                    ),
10147                                ));
10148                            }
10149                            transport
10150                                .fetch_url(
10151                                    &url,
10152                                    u64::try_from(byte_length).unwrap_or(0),
10153                                    &mut on_progress,
10154                                )
10155                                .map_err(|e| self.segment_fetch_failed(e))?
10156                        } else {
10157                            // §5.5: the owning unit's scope map, each value a one-element list.
10158                            let requested_scopes_json =
10159                                canonical_scope_json(&unit_scope_lists(&unit.scopes));
10160                            transport
10161                                .download_segment(
10162                                    &SegmentRequest {
10163                                        segment_id: segment_id.clone(),
10164                                        table: table.clone(),
10165                                        requested_scopes_json,
10166                                        byte_length: u64::try_from(byte_length).unwrap_or(0),
10167                                    },
10168                                    &mut on_progress,
10169                                )
10170                                .map_err(|e| {
10171                                    // §5.5: access changed since the response;
10172                                    // the next round's SUB_START settles the unit.
10173                                    if e.code == "sync.forbidden" {
10174                                        SectionError::Refused
10175                                    } else {
10176                                        self.segment_fetch_failed(e)
10177                                    }
10178                                })?
10179                        };
10180                        // §5.4: the descriptor's byteLength caps the body before
10181                        // §5.1 verifies the content address; the client discards
10182                        // the segment and re-pulls on either failure.
10183                        if bytes.len() as i64 > byte_length {
10184                            self.schedule_background_retry();
10185                            return Err(SectionError::Abort(
10186                                "sync.invalid_request".to_owned(),
10187                                "segment exceeds its byteLength".to_owned(),
10188                            ));
10189                        }
10190                        on_progress(bytes.len() as u64);
10191                        let digest = Sha256::digest(&bytes);
10192                        let expected = segment_id
10193                            .strip_prefix("sha256:")
10194                            .unwrap_or(segment_id.as_str());
10195                        if bytes_to_hex(&digest) != expected {
10196                            // §5.1: the client discards the segment and re-pulls.
10197                            self.schedule_background_retry();
10198                            return Err(SectionError::Abort(
10199                                "sync.invalid_request".to_owned(),
10200                                "segment bytes do not match the content address (§5.1)".to_owned(),
10201                            ));
10202                        }
10203                        self.progress.update(|p| p.phase = ProgressPhase::Import);
10204                        if media_type == MediaType::Sqlite {
10205                            // §5.3: a range image keeps its position in the resume
10206                            // token; a descriptor row cursor is invalid.
10207                            if row_cursor.is_some() || next_row_cursor.is_some() {
10208                                return Err(SectionError::Abort(
10209                                    "sync.invalid_request".to_owned(),
10210                                    "sqlite segments are whole-table: rowCursor/nextRowCursor must be absent (§5.3)"
10211                                        .to_owned(),
10212                                ));
10213                            }
10214                            let (pair, clear, effective) =
10215                                self.segment_pair(&unit, &echo, &seeded, &table)?;
10216                            report.segment_rows_applied += self.apply_sqlite_segment(
10217                                &pair,
10218                                &bytes,
10219                                clear,
10220                                &effective,
10221                                &unit.id,
10222                                &segment_id,
10223                                resumed.unwrap_or(0),
10224                                row_count,
10225                                as_of_commit_seq,
10226                                &scope_digest,
10227                            )?;
10228                            seeded.insert(table);
10229                        } else {
10230                            let segment = decode_rows_segment(&bytes).map_err(|e| {
10231                                SectionError::Abort(e.code.as_str().to_owned(), e.detail)
10232                            })?;
10233                            let (pair, clear, effective) =
10234                                self.segment_pair(&unit, &echo, &seeded, &segment.table)?;
10235                            report.segment_rows_applied +=
10236                                self.apply_segment(&pair, &segment, clear, &effective, &unit.id)?;
10237                            seeded.insert(segment.table);
10238                            // An image records its rows with each chunk.
10239                            self.stored(record_applied_rows(
10240                                &self.conn,
10241                                &unit.id,
10242                                &segment_id,
10243                                row_count,
10244                            ))
10245                            .map_err(storage)?;
10246                        }
10247                    }
10248                    Frame::UnitExtent { .. } => {}
10249                    _ => {
10250                        return Err(SectionError::Abort(
10251                            "sync.invalid_request".to_owned(),
10252                            "unexpected frame inside a unit section".to_owned(),
10253                        ));
10254                    }
10255                }
10256            }
10257            Ok(())
10258        })();
10259        match applied {
10260            Ok(()) => {}
10261            // The unit keeps catching up with its cursor unpersisted.
10262            Err(SectionError::Refused | SectionError::Deferred) => return Ok(false),
10263            Err(SectionError::FailClosed) => {
10264                // §5.6: the failure is unit-local; the rest of the response
10265                // still applies and the failed unit leaves the list.
10266                self.begin_observation("syncular_section_failure")
10267                    .map_err(storage)?;
10268                let failed = (|| {
10269                    let mut batch = ChangeAccumulator::default();
10270                    for (base_key, value) in self.stored(window_refs(&self.conn, &unit.id))? {
10271                        batch.window(&base_key, &value);
10272                    }
10273                    batch.status = true;
10274                    self.stored(save_unit(
10275                        &self.conn,
10276                        &UnitRecord {
10277                            status: UnitStatus::Failed,
10278                            reason_code: Some("sync.scope_revoked".to_owned()),
10279                            restriction: None,
10280                            ..unit.clone()
10281                        },
10282                    ))?;
10283                    self.finish_observation("syncular_section_failure", batch)
10284                })();
10285                if let Err(message) = failed {
10286                    self.rollback_observation("syncular_section_failure");
10287                    return Err(storage(message));
10288                }
10289                report.failed.push(unit.id);
10290                return Ok(false);
10291            }
10292            Err(error) => return Err(error),
10293        }
10294        let Some((next_cursor, bootstrap_state)) = sub_end else {
10295            return Ok(false);
10296        };
10297        let completes = start.bootstrap && bootstrap_state.is_none();
10298        let rollback = self.capture();
10299        let counts = (
10300            report.resets.len(),
10301            report.revoked.len(),
10302            report.failed.len(),
10303        );
10304        self.begin_observation("syncular_sub_end")
10305            .map_err(storage)?;
10306        let finished = self
10307            .finish_section(start, echo, next_cursor, bootstrap_state, seeded, report)
10308            .and_then(|batch| {
10309                self.rebuild_overlay_if_dirty()?;
10310                self.finish_observation("syncular_sub_end", batch)
10311            });
10312        if let Err(message) = finished {
10313            self.rollback_observation("syncular_sub_end");
10314            self.restore(rollback);
10315            report.resets.truncate(counts.0);
10316            report.revoked.truncate(counts.1);
10317            report.failed.truncate(counts.2);
10318            return Err(storage(message));
10319        }
10320        Ok(completes)
10321    }
10322
10323    /// §5.6: the table, first-page flag and effective scopes of one segment
10324    /// of `unit`'s section. A first page outside the unit's restriction ends it.
10325    #[allow(clippy::type_complexity)]
10326    fn segment_pair(
10327        &self,
10328        unit: &UnitRecord,
10329        echo: &[EchoRecord],
10330        seeded: &BTreeSet<String>,
10331        name: &str,
10332    ) -> Result<(TableSchema, bool, Vec<(String, Vec<String>)>), SectionError> {
10333        let table = self.schema.table(name).cloned().ok_or_else(|| {
10334            SectionError::Abort(
10335                "sync.schema_mismatch".to_owned(),
10336                format!("segment table {name:?} is not in the client schema"),
10337            )
10338        })?;
10339        let effective = covers(&table, &unit.scopes)
10340            .then(|| pair_scopes(&unit.scopes, name, Some(echo)))
10341            .flatten()
10342            .ok_or_else(|| {
10343                SectionError::Abort(
10344                    "sync.invalid_request".to_owned(),
10345                    "a segment names no active pair of its unit".to_owned(),
10346                )
10347            })?;
10348        let first_page = !seeded.contains(name);
10349        // §5.6: a first page of a table outside the restriction is a whole
10350        // bootstrap from a server that predates RESTRICT; the restriction ends
10351        // before the clear runs.
10352        if first_page
10353            && unit
10354                .restriction
10355                .as_ref()
10356                .is_some_and(|restriction| !restriction.tables.iter().any(|t| t == name))
10357        {
10358            self.conn
10359                .execute(
10360                    "UPDATE _syncular_units SET restriction=NULL WHERE id=?1",
10361                    [&unit.id],
10362                )
10363                .map_err(|error| {
10364                    SectionError::Abort(
10365                        "storage.failed".to_owned(),
10366                        Self::sqlite_failure(&self.storage_failure, error),
10367                    )
10368                })?;
10369        }
10370        Ok((table, first_page, effective))
10371    }
10372
10373    /// §4.4 `SUB_END` of an active, revoked or reset section, inside the
10374    /// caller's observation: a reset discards cursor, resume token and
10375    /// first-page record and keeps the echo; a revoked pair purges by its last
10376    /// echo (§3.3); otherwise the echo replaces the persisted one, a revoked
10377    /// fenced record that turns active again re-bootstraps the unit, and a
10378    /// bootstrap section persists its pin, resume token and seeded tables.
10379    fn finish_section(
10380        &mut self,
10381        start: &SectionStart,
10382        echo: Vec<EchoRecord>,
10383        next_cursor: i64,
10384        bootstrap_state: Option<String>,
10385        seeded: BTreeSet<String>,
10386        report: &mut SyncReport,
10387    ) -> Result<ChangeAccumulator, String> {
10388        #[cfg(feature = "bench-internals")]
10389        let _phase = self.benchmark_phases.start(Phase::CursorPersist);
10390        let mut batch = ChangeAccumulator::default();
10391        // §4.4: SUB_END writes only into a unit that is still registered.
10392        let Some(unit) = self
10393            .stored(get_unit(&self.conn, &start.id))?
10394            .filter(|unit| unit.status == UnitStatus::Active)
10395        else {
10396            return Ok(batch);
10397        };
10398        for (base_key, value) in self.stored(window_refs(&self.conn, &unit.id))? {
10399            batch.window(&base_key, &value);
10400        }
10401        if start.status == SubStatus::Reset {
10402            // §4.6: staleness, never a purge; the rows stay until the
10403            // bootstrap replaces them.
10404            report.resets.push(unit.id.clone());
10405            self.stored(save_unit(
10406                &self.conn,
10407                &UnitRecord {
10408                    reason_code: Some(start.reason_code.clone()),
10409                    cursor: Some(-1),
10410                    bootstrap_state: None,
10411                    seeded: Vec::new(),
10412                    restriction: None,
10413                    ..unit
10414                },
10415            ))?;
10416            return Ok(batch);
10417        }
10418        let revoked = start.status == SubStatus::Revoked;
10419        let mut units = None;
10420        let mut failed = false;
10421        let mut rearmed = false;
10422        for table in self.schema.tables.clone() {
10423            if !covers(&table, &unit.scopes) {
10424                continue;
10425            }
10426            let last = unit
10427                .echo
10428                .as_deref()
10429                .and_then(|last| pair_scopes(&unit.scopes, &table.name, Some(last)));
10430            let next = (!revoked)
10431                .then(|| pair_scopes(&unit.scopes, &table.name, Some(&echo)))
10432                .flatten();
10433            // §3.3: a revoked fenced record that turns active again re-bootstraps.
10434            rearmed |= unit.echo.is_some() && last.is_none() && next.is_some();
10435            // §3.3: purge a pair the response revokes, keyed on the echo the
10436            // client persisted while the pair was active.
10437            if let (Some(last), None) = (&last, &next) {
10438                let units = match units {
10439                    Some(ref units) => units,
10440                    None => units.insert(self.stored(load_units(&self.conn))?),
10441                };
10442                let owners = unit_owners(units, &table, &unit.id);
10443                failed |= !self.purge_pair(&mut batch, &table, last, &owners)?;
10444            }
10445        }
10446        if revoked || failed {
10447            if revoked {
10448                report.revoked.push(unit.id.clone());
10449            }
10450            if failed {
10451                report.failed.push(unit.id.clone());
10452            }
10453            batch.status = true;
10454            self.stored(save_unit(
10455                &self.conn,
10456                &UnitRecord {
10457                    status: if failed {
10458                        UnitStatus::Failed
10459                    } else {
10460                        UnitStatus::Revoked
10461                    },
10462                    reason_code: Some(if failed {
10463                        "sync.scope_revoked".to_owned()
10464                    } else {
10465                        start.reason_code.clone()
10466                    }),
10467                    cursor: Some(-1),
10468                    bootstrap_state: None,
10469                    seeded: Vec::new(),
10470                    echo: None,
10471                    restriction: None,
10472                    ..unit
10473                },
10474            ))?;
10475            return Ok(batch);
10476        }
10477        let completed = start.bootstrap && !rearmed && bootstrap_state.is_none();
10478        let (cursor, bootstrap_state, seeded) = if rearmed {
10479            (Some(-1), None, Vec::new())
10480        } else if start.bootstrap {
10481            let seeded = if bootstrap_state.is_none() {
10482                Vec::new()
10483            } else {
10484                seeded.into_iter().collect()
10485            };
10486            (Some(next_cursor), bootstrap_state, seeded)
10487        } else {
10488            (
10489                unit.cursor,
10490                unit.bootstrap_state.clone(),
10491                unit.seeded.clone(),
10492            )
10493        };
10494        self.stored(save_unit(
10495            &self.conn,
10496            &UnitRecord {
10497                reason_code: None,
10498                echo: Some(echo.clone()),
10499                cursor,
10500                bootstrap_state,
10501                seeded,
10502                // §7.4.3: the restriction ends when the unit re-bootstraps or
10503                // completes its bootstrap.
10504                restriction: unit.restriction.clone().filter(|_| !rearmed && !completed),
10505                ..unit.clone()
10506            },
10507        ))?;
10508        if !completed {
10509            return Ok(batch);
10510        }
10511        // §7.1: a completed bootstrap covers acknowledged intent of its pairs
10512        // at or below its pin. The intent or the bootstrapped row proves the
10513        // pair: a patch omits its scope columns, and the base row it merged
10514        // over may be gone (a §7.4.4 reset before the ACK).
10515        for table in self.schema.tables.clone() {
10516            let Some(effective) = covers(&table, &unit.scopes)
10517                .then(|| pair_scopes(&unit.scopes, &table.name, Some(&echo)))
10518                .flatten()
10519            else {
10520                continue;
10521            };
10522            let mut statement = self
10523                .conn
10524                .prepare("SELECT id,intent_json FROM _syncular_acked_rows WHERE tbl=?1 AND commit_seq<=?2")
10525                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
10526            let rows = statement
10527                .query_map(rusqlite::params![table.name, next_cursor], |row| {
10528                    Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
10529                })
10530                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
10531            let mut covered = HashSet::new();
10532            for row in rows {
10533                let (id, intent) =
10534                    row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
10535                let values: Map<String, Value> =
10536                    serde_json::from_str(&intent).map_err(|_| "sync.local_corrupt".to_owned())?;
10537                let base = self.stored_row(&table, &id, true)?.map(|(row, _)| {
10538                    table
10539                        .columns
10540                        .iter()
10541                        .zip(&row)
10542                        .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
10543                        .collect()
10544                });
10545                if [Some(values), base]
10546                    .iter()
10547                    .flatten()
10548                    .any(|values| owns_local_row(&table, &effective, &[], values))
10549                {
10550                    covered.insert(id);
10551                }
10552            }
10553            drop(statement);
10554            for id in covered {
10555                self.conn
10556                    .execute(
10557                        "DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2 AND commit_seq<=?3",
10558                        rusqlite::params![table.name, id, next_cursor],
10559                    )
10560                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
10561                self.overlay_dirty.table(&table.name);
10562                batch.table(&table.name);
10563            }
10564        }
10565        Ok(batch)
10566    }
10567
10568    /// §3.3: purge one revoked pair inside the caller's observation: delete
10569    /// the rows matching its last effective scopes that no owner holds, drop
10570    /// the retained intent and every pending commit that provably writes into
10571    /// them, and delete now-unauthorized blob bodies. `Ok(false)` when the
10572    /// table has no local scope-column mapping (fail closed: nothing deleted).
10573    fn purge_pair(
10574        &mut self,
10575        batch: &mut ChangeAccumulator,
10576        table: &TableSchema,
10577        last: &[(String, Vec<String>)],
10578        owners: &[Vec<(String, Vec<String>)>],
10579    ) -> Result<bool, String> {
10580        if local_scope_predicate(table, last, owners).is_err() {
10581            return Ok(false);
10582        }
10583        for commit in self.drop_failed_commits(|name, values| {
10584            name == table.name && owns_local_row(table, last, owners, values)
10585        })? {
10586            batch.conflicts = true;
10587            batch.rejections = true;
10588            batch.outcomes = true;
10589            for operation in commit.ops {
10590                batch.table(&operation.table);
10591            }
10592        }
10593        if !self.purge_scope_rows(&table.name, last, owners)? {
10594            return Err("storage.failed: the revoked scope purge failed".to_owned());
10595        }
10596        self.record_scope_map(batch, &table.name, last);
10597        if self.drop_doomed_outbox(table, last, owners)? {
10598            batch.status = true;
10599            batch.rejections = true;
10600            batch.outcomes = true;
10601        }
10602        // §5.9.7 B2: revocation deletes now-unauthorized blob bodies.
10603        self.delete_unreferenced_cached_blobs()?;
10604        Ok(true)
10605    }
10606
10607    /// §4.5 client application: apply one live-section or delta `COMMIT`
10608    /// with only the changes that match the persisted echo of a receiving
10609    /// unit; every applied change then goes through the version guard.
10610    /// Returns whether any change of the frame applied.
10611    fn apply_live_commit(
10612        &mut self,
10613        receivers: &[UnitRecord],
10614        tables: &[String],
10615        changes: Vec<ssp2::model::Change>,
10616        commit_seq: i64,
10617        source: AppliedServerCommitSource,
10618    ) -> Result<bool, TransportError> {
10619        let changes: Vec<ssp2::model::Change> = changes
10620            .into_iter()
10621            .filter(|change| {
10622                // An unknown table fails loud in the apply below.
10623                let Some(table) = tables
10624                    .get(change.table_index as usize)
10625                    .and_then(|name| self.schema.table(name))
10626                else {
10627                    return true;
10628                };
10629                receivers.iter().any(|unit| {
10630                    covers(table, &unit.scopes)
10631                        && pair_scopes(&unit.scopes, &table.name, unit.echo.as_deref()).is_some_and(
10632                            |scopes| {
10633                                scopes.iter().all(|(variable, values)| {
10634                                    change
10635                                        .scopes
10636                                        .iter()
10637                                        .find(|(key, _)| key == variable)
10638                                        .is_some_and(|(_, value)| values.contains(value))
10639                                })
10640                            },
10641                        )
10642                })
10643            })
10644            .collect();
10645        if changes.is_empty() {
10646            return Ok(false);
10647        }
10648        let affected_tables = changes
10649            .iter()
10650            .filter_map(|change| tables.get(change.table_index as usize))
10651            .cloned()
10652            .collect::<BTreeSet<_>>();
10653        let previous_batches = self.change_queue.len();
10654        self.apply_commit_frame(tables, &changes, commit_seq)?;
10655        if self.change_queue.len() > previous_batches {
10656            if let Some(batch) = self.change_queue.back_mut() {
10657                batch
10658                    .applied_commits
10659                    .extend(
10660                        affected_tables
10661                            .into_iter()
10662                            .map(|table| AppliedServerCommit {
10663                                table,
10664                                commit_seq,
10665                                source,
10666                            }),
10667                    );
10668            }
10669        }
10670        Ok(true)
10671    }
10672
10673    // Both pull and realtime frames use the same durable observation boundary.
10674    fn apply_commit_frame(
10675        &mut self,
10676        tables: &[String],
10677        changes: &[ssp2::model::Change],
10678        sequence: i64,
10679    ) -> Result<(), TransportError> {
10680        #[cfg(feature = "bench-internals")]
10681        let _phase = self.benchmark_phases.start(Phase::CommitApply);
10682        let rollback = self.capture();
10683        self.coalesce_unit(changes.len())
10684            .and_then(|()| self.begin_observation("syncular_commit"))
10685            .map_err(|message| TransportError::new("storage.failed", message))?;
10686        let applied = (|| {
10687            let mut batch = ChangeAccumulator::default();
10688            self.record_commit_changes(&mut batch, tables, changes);
10689            // §7.1 chained base: a change names the version an acknowledged own
10690            // commit wrote; pending upserts chained on it move to that version.
10691            for change in changes {
10692                let Some(table) = tables.get(change.table_index as usize) else {
10693                    continue;
10694                };
10695                let acknowledged = self
10696                    .conn
10697                    .prepare_cached("SELECT op_json FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2 AND commit_seq=?3")
10698                    .and_then(|mut statement| {
10699                        statement
10700                            .query_map(rusqlite::params![table, change.row_id, sequence], |row| row.get::<_, String>(0))?
10701                            .collect::<Result<Vec<_>, _>>()
10702                    })
10703                    .map_err(|error| TransportError::new("storage.failed", Self::sqlite_failure(&self.storage_failure, error)))?
10704                    .iter()
10705                    .map(|op| serde_json::from_str::<CommitOperation>(op).map(OutboxOp::from))
10706                    .collect::<Result<Vec<_>, _>>()
10707                    .map_err(|_| TransportError::new("storage.failed", "sync.local_corrupt: invalid acknowledged operation".to_owned()))?;
10708                for index in 0..self.outbox.len() {
10709                    if !self.outbox[index]
10710                        .ops
10711                        .iter()
10712                        .any(|op| acknowledged.iter().any(|earlier| chains_on(earlier, op)))
10713                    {
10714                        continue;
10715                    }
10716                    let commit = Arc::make_mut(&mut self.outbox[index]);
10717                    for op in &mut commit.ops {
10718                        if acknowledged.iter().any(|earlier| chains_on(earlier, op)) {
10719                            op.base_version = Some(change.row_version);
10720                        }
10721                    }
10722                    let ops = serde_json::to_string(
10723                        &commit
10724                            .ops
10725                            .iter()
10726                            .map(CommitOperation::from)
10727                            .collect::<Vec<_>>(),
10728                    )
10729                    .map_err(|_| {
10730                        TransportError::new(
10731                            "storage.failed",
10732                            "sync.local_corrupt: unencodable outbox".to_owned(),
10733                        )
10734                    })?;
10735                    let id = commit.client_commit_id.clone();
10736                    self.conn
10737                        .execute(
10738                            "UPDATE _syncular_outbox SET ops_json=?1 WHERE commit_id=?2",
10739                            rusqlite::params![ops, id],
10740                        )
10741                        .map_err(|error| {
10742                            TransportError::new(
10743                                "storage.failed",
10744                                Self::sqlite_failure(&self.storage_failure, error),
10745                            )
10746                        })?;
10747                }
10748            }
10749            {
10750                let storage_failed = |error| {
10751                    TransportError::new(
10752                        "storage.failed",
10753                        Self::sqlite_failure(&self.storage_failure, error),
10754                    )
10755                };
10756                let pinned: HashSet<(&str, &str)> = self
10757                    .outbox
10758                    .iter()
10759                    .flat_map(|commit| &commit.ops)
10760                    .map(|op| (op.table.as_str(), op.row_id.as_str()))
10761                    .collect();
10762                for change in changes {
10763                    let table = tables.get(change.table_index as usize).ok_or_else(|| {
10764                        TransportError::new(
10765                            "sync.invalid_request",
10766                            "change tableIndex out of range".to_owned(),
10767                        )
10768                    })?;
10769                    if pinned.contains(&(table.as_str(), change.row_id.as_str())) {
10770                        self.conn.prepare_cached("INSERT INTO _syncular_row_deliveries(tbl,id,commit_seq) VALUES (?1,?2,?3) ON CONFLICT(tbl,id) DO UPDATE SET commit_seq=MAX(commit_seq,excluded.commit_seq)")
10771                            .and_then(|mut statement| statement.execute(rusqlite::params![table, change.row_id, sequence]))
10772                            .map_err(storage_failed)?;
10773                    }
10774                    self.conn.prepare_cached("DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2 AND commit_seq<=?3")
10775                        .and_then(|mut statement| statement.execute(rusqlite::params![table, change.row_id, sequence]))
10776                        .map_err(storage_failed)?;
10777                }
10778            }
10779            self.apply_commit_changes(tables, changes)?;
10780            self.rebuild_overlay_if_dirty()
10781                .map_err(|message| TransportError::new("storage.failed", message))?;
10782            self.finish_observation("syncular_commit", batch)
10783                .map_err(|message| TransportError::new("storage.failed", message))
10784        })();
10785        if applied.is_err() {
10786            self.rollback_observation("syncular_commit");
10787            self.restore(rollback);
10788        }
10789        applied
10790    }
10791
10792    fn apply_commit_changes(
10793        &mut self,
10794        tables: &[String],
10795        changes: &[ssp2::model::Change],
10796    ) -> Result<(), TransportError> {
10797        // A clean overlay can reconcile changed primary keys independently
10798        // when their tables have no secondary unique constraints. Such a
10799        // constraint can change whether another row's write succeeds, so a
10800        // table with one and any local intent takes the table rebuild.
10801        let changed_tables: BTreeSet<&str> = changes
10802            .iter()
10803            .filter_map(|change| tables.get(change.table_index as usize))
10804            .map(String::as_str)
10805            .collect();
10806        let mut mirror_visible = !self.overlay_dirty.get();
10807        for name in &changed_tables {
10808            if mirror_visible
10809                && self
10810                    .schema
10811                    .table(name)
10812                    .is_some_and(|table| table.indexes.iter().any(|index| index.unique))
10813            {
10814                let (pending, protected) = self
10815                    .table_intent(name)
10816                    .map_err(|message| TransportError::new("storage.failed", message))?;
10817                mirror_visible = !pending && !protected;
10818            }
10819        }
10820        for change in changes {
10821            let table_name = tables.get(change.table_index as usize).ok_or_else(|| {
10822                TransportError::new(
10823                    "sync.invalid_request",
10824                    "change tableIndex out of range".to_owned(),
10825                )
10826            })?;
10827            let table = self.schema.table(table_name).ok_or_else(|| {
10828                TransportError::new(
10829                    "sync.schema_mismatch",
10830                    format!("change targets unknown table {table_name:?}"),
10831                )
10832            })?;
10833            let landed = match change.op {
10834                Op::Upsert => {
10835                    let payload = change.row.as_ref().ok_or_else(|| {
10836                        TransportError::new(
10837                            "sync.invalid_request",
10838                            "upsert change without row payload".to_owned(),
10839                        )
10840                    })?;
10841                    // §5.11: decrypt encrypted columns on apply, or seal the row.
10842                    let (row, sealed) = {
10843                        #[cfg(feature = "bench-internals")]
10844                        let _phase = self.benchmark_phases.start(Phase::RowDecode);
10845                        // §5.11: a decrypt failure keeps its own identity.
10846                        decode_row_bytes(table, payload, &self.encryption)?
10847                    };
10848                    let table_name = table.name.clone();
10849                    // §4.5 version guard: only a newer upsert lands.
10850                    let landed = self
10851                        .write_base_row(&table_name, &row, change.row_version, sealed.as_ref())
10852                        .map_err(|m| TransportError::new("sync.invalid_request", m))?;
10853                    if mirror_visible && landed {
10854                        self.write_row(
10855                            &visible_table(&table_name),
10856                            &table_name,
10857                            &row,
10858                            change.row_version,
10859                            sealed.as_ref(),
10860                        )
10861                        .map_err(|m| TransportError::new("sync.invalid_request", m))?;
10862                    }
10863                    landed
10864                }
10865                // §4.5 version guard: a delete at or below the local version
10866                // is skipped, a delete of an absent row is a no-op.
10867                Op::Delete => self
10868                    .delete_base_row(table_name, &change.row_id, change.row_version)
10869                    .map_err(|m| TransportError::new("sync.invalid_request", m))?,
10870            };
10871            // The change dropped the row's acknowledged intent either way
10872            // (§7.1), so the overlay of a skipped change rebuilds from the
10873            // unchanged base.
10874            self.overlay_dirty.table(table_name);
10875            if mirror_visible && !(landed && change.op == Op::Upsert) {
10876                let predicate = row_id_predicate(table);
10877                let visible = visible_table(table_name);
10878                let mut statements = vec![format!("DELETE FROM {visible} WHERE {predicate}")];
10879                if !landed {
10880                    statements.push(format!(
10881                        "INSERT INTO {visible} SELECT * FROM {} WHERE {predicate}",
10882                        base_table(table_name)
10883                    ));
10884                }
10885                for statement in &statements {
10886                    self.conn
10887                        .prepare_cached(statement)
10888                        .and_then(|mut statement| {
10889                            statement.execute(rusqlite::params![change.row_id])
10890                        })
10891                        .map_err(|error| {
10892                            TransportError::new("sync.invalid_request", error.to_string())
10893                        })?;
10894                }
10895            }
10896        }
10897        if mirror_visible {
10898            let changed_rows: HashSet<(&str, &str)> = changes
10899                .iter()
10900                .map(|change| {
10901                    (
10902                        tables[change.table_index as usize].as_str(),
10903                        change.row_id.as_str(),
10904                    )
10905                })
10906                .collect();
10907            self.replay_overlay(&changed_tables, Some(&changed_rows))
10908                .map_err(|message| TransportError::new("storage.failed", message))?;
10909            self.overlay_dirty.set(false);
10910        }
10911        Ok(())
10912    }
10913
10914    /// §5.6 segment application: validate against the generated schema,
10915    /// clear the pair's scope on the table's first page (fail closed
10916    /// without a mapping), then upsert each row with its segment-carried
10917    /// server version under the §4.5 version guard (§5.2).
10918    fn apply_segment(
10919        &mut self,
10920        table: &TableSchema,
10921        segment: &RowsSegment,
10922        first_page: bool,
10923        effective: &[(String, Vec<String>)],
10924        unit_id: &str,
10925    ) -> Result<u32, SectionError> {
10926        // §5.2: the column table validates against the generated schema —
10927        // order, names, types, nullability; mismatch is fatal. §5.11: the
10928        // server sends the WIRE types (bytes for an encrypted column), so
10929        // validate against wire_columns.
10930        let column = if segment.columns.len() != table.wire_columns.len() {
10931            Some(Value::from("count"))
10932        } else {
10933            segment
10934                .columns
10935                .iter()
10936                .zip(table.wire_columns.iter())
10937                .position(|(a, b)| a.name != b.name || a.ty != b.ty || a.nullable != b.nullable)
10938                .map(Value::from)
10939        };
10940        if segment.table != table.name
10941            || segment.schema_version != self.schema.version
10942            || column.is_some()
10943        {
10944            let mut details = Map::new();
10945            details.insert("table".to_owned(), Value::from(table.name.as_str()));
10946            if let Some(column) = column {
10947                details.insert("column".to_owned(), column);
10948            }
10949            return Err(SectionError::Transport(TransportError {
10950                code: "sync.schema_mismatch".to_owned(),
10951                message: "rows segment columns do not match the generated schema (§5.2)".to_owned(),
10952                details: Some(Value::Object(details)),
10953            }));
10954        }
10955        let mut applied = 0u32;
10956        let (mut write, mut commit) = (std::time::Duration::ZERO, std::time::Duration::ZERO);
10957        let empty = Vec::new();
10958        let blocks = if segment.blocks.is_empty() {
10959            std::slice::from_ref(&empty)
10960        } else {
10961            &segment.blocks
10962        };
10963        for (index, block) in blocks.iter().enumerate() {
10964            let clear = first_page && index == 0;
10965            if !clear && block.is_empty() {
10966                continue;
10967            }
10968            let rollback = self.capture();
10969            let (pending, protected) = self
10970                .table_intent(&table.name)
10971                .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10972            let reconcile_pending =
10973                !protected && pending && table.indexes.iter().any(|index| index.unique);
10974            let mirror_visible = !protected && !reconcile_pending;
10975            self.coalesce_unit(block.len())
10976                .and_then(|()| self.begin_observation("syncular_segment_block"))
10977                .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10978            let started = std::time::Instant::now();
10979            let outcome = (|| {
10980                self.rebuild_overlay_if_dirty()
10981                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10982                let mut changed_keys = Vec::new();
10983                let mut batch = ChangeAccumulator::default();
10984                if clear && self.clear_scope(table, effective, unit_id)? || !block.is_empty() {
10985                    batch.table(&table.name);
10986                }
10987                for (row_index, row) in block.iter().enumerate() {
10988                    let decrypted;
10989                    let mut sealed = None;
10990                    let values = if table.has_encrypted_columns() {
10991                        let mut values = row.values.clone();
10992                        sealed = crate::values::decrypt_segment_row(
10993                            table,
10994                            &mut values,
10995                            &self.encryption,
10996                        )
10997                        .map_err(SectionError::Transport)?;
10998                        decrypted = values;
10999                        &decrypted
11000                    } else {
11001                        &row.values
11002                    };
11003                    if !self
11004                        .write_base_row(&table.name, values, row.server_version, sealed.as_ref())
11005                        .map_err(|message| {
11006                            SectionError::Abort("sync.invalid_request".into(), message)
11007                        })?
11008                    {
11009                        continue;
11010                    }
11011                    if reconcile_pending {
11012                        changed_keys.push(
11013                            owned_sql_value(RowParam::Cell(&values[table.pk_index])).map_err(
11014                                |message| SectionError::Abort("storage.failed".into(), message),
11015                            )?,
11016                        );
11017                    }
11018                    if mirror_visible {
11019                        self.write_row(
11020                            &visible_table(&table.name),
11021                            &table.name,
11022                            values,
11023                            row.server_version,
11024                            sealed.as_ref(),
11025                        )
11026                        .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
11027                    }
11028                    let processed = u64::from(applied) + row_index as u64 + 1;
11029                    if processed.is_multiple_of(1024) {
11030                        self.progress.update(|p| p.rows_processed = processed);
11031                    }
11032                }
11033                self.reconcile_import(table, &mut changed_keys, protected, reconcile_pending)
11034                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
11035                let written = std::time::Instant::now();
11036                self.finish_observation("syncular_segment_block", batch)
11037                    .map(|()| written)
11038                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))
11039            })();
11040            let written = match outcome {
11041                Ok(written) => written,
11042                Err(error) => {
11043                    self.rollback_observation("syncular_segment_block");
11044                    self.restore(rollback);
11045                    return Err(error);
11046                }
11047            };
11048            // §7.7: a payload reports its import cost.
11049            write += written - started;
11050            commit += written.elapsed();
11051            applied += block.len() as u32;
11052            self.progress.update(|p| {
11053                p.rows_processed = u64::from(applied);
11054                p.import_write_ms = u64::try_from(write.as_millis()).ok();
11055                p.import_commit_ms = u64::try_from(commit.as_millis()).ok();
11056            });
11057            std::thread::yield_now();
11058        }
11059        self.progress
11060            .update(|p| p.rows_processed = u64::from(applied));
11061        Ok(applied)
11062    }
11063
11064    /// §5.3 sqlite-image application: validate the in-file metadata
11065    /// against the descriptor, validate column names/order against the
11066    /// generated schema, run the §5.6 first-page clear when fresh, then
11067    /// replace-or-upsert every image row with its `_syncular_version`.
11068    /// Mechanics: the image lands in a temp file read through a second
11069    /// rusqlite connection, equivalent to ATTACH + INSERT…SELECT.
11070    /// Each image chunk commits its writes and revision before readers continue.
11071    #[allow(clippy::too_many_arguments)]
11072    fn apply_sqlite_segment(
11073        &mut self,
11074        table: &TableSchema,
11075        bytes: &[u8],
11076        first_page: bool,
11077        effective: &[(String, Vec<String>)],
11078        unit_id: &str,
11079        segment_id: &str,
11080        resumed: i64,
11081        row_count: i64,
11082        as_of_commit_seq: i64,
11083        scope_digest: &str,
11084    ) -> Result<u32, SectionError> {
11085        let invalid = |detail: &str| {
11086            SectionError::Abort(
11087                "sync.invalid_request".to_owned(),
11088                format!("sqlite segment rejected: {detail} (§5.3)"),
11089            )
11090        };
11091        // §5.11: an image bypasses the decrypt seam, so a server cannot serve
11092        // one for a table with encrypted columns.
11093        if table.has_encrypted_columns() {
11094            return Err(invalid("table has encrypted columns"));
11095        }
11096
11097        // The image lands beside the replica so it shares the replica's
11098        // directory permissions and volume; an in-memory replica has no
11099        // directory and uses the system temp dir. The file is owner-only.
11100        let directory = self
11101            .previous_version_replica_path()
11102            .and_then(|replica| std::path::Path::new(&replica).parent().map(Into::into))
11103            .unwrap_or_else(std::env::temp_dir);
11104        let path = directory.join(format!(".syncular-image-{}.tmp", uuid::Uuid::new_v4()));
11105        let outcome = (|| {
11106            let mut options = std::fs::OpenOptions::new();
11107            options.write(true).create_new(true);
11108            #[cfg(unix)]
11109            std::os::unix::fs::OpenOptionsExt::mode(&mut options, 0o600);
11110            std::io::Write::write_all(
11111                &mut options
11112                    .open(&path)
11113                    .map_err(|_| invalid("image temp file write failed"))?,
11114                bytes,
11115            )
11116            .map_err(|_| invalid("image temp file write failed"))?;
11117            let img = rusqlite::Connection::open_with_flags(
11118                &path,
11119                rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY,
11120            )
11121            .map_err(|_| invalid("bytes do not open as a SQLite database"))?;
11122            self.apply_sqlite_image(
11123                &img,
11124                table,
11125                first_page,
11126                effective,
11127                unit_id,
11128                segment_id,
11129                resumed,
11130                row_count,
11131                as_of_commit_seq,
11132                scope_digest,
11133            )
11134        })();
11135        let _ = std::fs::remove_file(&path);
11136        outcome
11137    }
11138
11139    #[allow(clippy::too_many_arguments)]
11140    fn apply_sqlite_image(
11141        &mut self,
11142        img: &rusqlite::Connection,
11143        table: &crate::schema::TableSchema,
11144        first_fresh_page: bool,
11145        effective: &[(String, Vec<String>)],
11146        unit_id: &str,
11147        segment_id: &str,
11148        resumed: i64,
11149        row_count: i64,
11150        as_of_commit_seq: i64,
11151        scope_digest: &str,
11152    ) -> Result<u32, SectionError> {
11153        let invalid = |detail: String| {
11154            SectionError::Abort(
11155                "sync.invalid_request".to_owned(),
11156                format!("sqlite segment rejected: {detail} (§5.3)"),
11157            )
11158        };
11159
11160        // 1. Metadata vs descriptor + client state (§5.3 rule 2; exactly
11161        //    one row).
11162        type MetaRow = (i64, String, i64, i64, String, i64, i64);
11163        let meta: MetaRow = img
11164            .query_row(
11165                "SELECT format, \"table\", \"schemaVersion\", \"asOfCommitSeq\",
11166                        \"scopeDigest\", \"rowCount\",
11167                        (SELECT count(*) FROM _syncular_segment)
11168                 FROM _syncular_segment",
11169                [],
11170                |row| {
11171                    Ok((
11172                        row.get(0)?,
11173                        row.get(1)?,
11174                        row.get(2)?,
11175                        row.get(3)?,
11176                        row.get(4)?,
11177                        row.get(5)?,
11178                        row.get(6)?,
11179                    ))
11180                },
11181            )
11182            .map_err(|_| invalid("missing or unreadable _syncular_segment metadata".to_owned()))?;
11183        let (format, meta_table, schema_version, pin, digest, meta_rows, meta_count) = meta;
11184        if meta_count != 1 {
11185            return Err(invalid(format!(
11186                "_syncular_segment must contain exactly one row, found {meta_count}"
11187            )));
11188        }
11189        if format != 1 {
11190            return Err(invalid(format!("format {format}")));
11191        }
11192        if meta_table != table.name {
11193            return Err(invalid(format!("image table {meta_table:?}")));
11194        }
11195        if schema_version != i64::from(self.schema.version) {
11196            return Err(invalid(format!("schemaVersion {schema_version}")));
11197        }
11198        if pin != as_of_commit_seq {
11199            return Err(invalid(format!("asOfCommitSeq {pin}")));
11200        }
11201        if digest != scope_digest {
11202            return Err(invalid("scopeDigest mismatch".to_owned()));
11203        }
11204        if meta_rows != row_count {
11205            return Err(invalid(format!("rowCount {meta_rows}")));
11206        }
11207
11208        // 2. Column names and order vs the generated schema (§5.3 rule 3).
11209        let mut names: Vec<String> = Vec::new();
11210        let mut primary_keys: Vec<String> = Vec::new();
11211        {
11212            let mut stmt = img
11213                .prepare(&format!("PRAGMA table_info({})", quote_ident(&table.name)))
11214                .map_err(|_| invalid("image data table missing".to_owned()))?;
11215            let mut rows = stmt
11216                .query([])
11217                .map_err(|_| invalid("image data table unreadable".to_owned()))?;
11218            while let Some(row) = rows
11219                .next()
11220                .map_err(|_| invalid("image data table unreadable".to_owned()))?
11221            {
11222                if row
11223                    .get::<_, i64>(5)
11224                    .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?
11225                    > 0
11226                {
11227                    primary_keys.push(row.get(1).map_err(|error| {
11228                        invalid(Self::sqlite_failure(&self.storage_failure, error))
11229                    })?);
11230                }
11231                names.push(
11232                    row.get::<_, String>(1)
11233                        .map_err(|_| invalid("image data table unreadable".to_owned()))?,
11234                );
11235            }
11236        }
11237        let mut expected: Vec<&str> = table.columns.iter().map(|c| c.name.as_str()).collect();
11238        expected.push("_syncular_version");
11239        if names.len() != expected.len() || names.iter().zip(expected.iter()).any(|(a, b)| a != b) {
11240            return Err(SectionError::Transport(TransportError {
11241                code: "sync.schema_mismatch".to_owned(),
11242                message: "sqlite segment columns do not match the generated schema (§5.3)"
11243                    .to_owned(),
11244                details: Some(serde_json::json!({ "table": table.name })),
11245            }));
11246        }
11247
11248        if primary_keys != vec![table.primary_key.clone()] {
11249            return Err(invalid(
11250                "image primary key does not match generated schema".into(),
11251            ));
11252        }
11253        let actual_count: i64 = img
11254            .query_row(
11255                &format!("SELECT count(*) FROM {}", quote_ident(&table.name)),
11256                [],
11257                |row| row.get(0),
11258            )
11259            .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
11260        if actual_count != row_count {
11261            return Err(invalid("image row count does not match descriptor".into()));
11262        }
11263        let insert = self.insert_row_sql(&base_table(&table.name), table);
11264        let visible_insert = self.insert_row_sql(&visible_table(&table.name), table);
11265        let column_list = names
11266            .iter()
11267            .map(|n| quote_ident(n))
11268            .collect::<Vec<_>>()
11269            .join(", ");
11270        // §5.6: an earlier attempt at this position committed the first
11271        // `resumed` rows in primary-key order; the content address pins them.
11272        let mut stmt = img
11273            .prepare(&format!(
11274                "SELECT {column_list} FROM {} ORDER BY {} LIMIT -1 OFFSET ?1",
11275                quote_ident(&table.name),
11276                quote_ident(&table.primary_key)
11277            ))
11278            .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
11279        let mut rows = stmt
11280            .query([resumed])
11281            .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
11282        let resumed = u32::try_from(resumed)
11283            .ok()
11284            .filter(|rows| i64::from(*rows) <= actual_count)
11285            .ok_or_else(|| invalid("applied-segment record exceeds the image".into()))?;
11286        let mut applied = resumed;
11287        let mut limit = if first_fresh_page {
11288            1024
11289        } else {
11290            self.image_chunk_rows
11291                .get(&table.name)
11292                .copied()
11293                .unwrap_or(1024)
11294        };
11295        let (mut write, mut commit) = (std::time::Duration::ZERO, std::time::Duration::ZERO);
11296        loop {
11297            let clear = applied == 0 && first_fresh_page;
11298            let rollback = self.capture();
11299            let (pending, protected) = self
11300                .table_intent(&table.name)
11301                .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
11302            let reconcile_pending =
11303                !protected && pending && table.indexes.iter().any(|index| index.unique);
11304            let mirror_visible = !protected && !reconcile_pending;
11305            // §5.3: a table's first chunk holds 1,024 rows; a later range image
11306            // starts at the size its previous range reached. A later chunk holds at
11307            // least 1,024 rows, at most twice the rows of the one before, at
11308            // most 65,536, and at most the rows the one before applied per
11309            // 250 ms from its start to its COMMIT; a chunk that replays local
11310            // writes holds at most 1,024.
11311            let chunk = if pending || protected {
11312                limit.min(1024)
11313            } else {
11314                limit
11315            };
11316            self.coalesce_unit(chunk as usize)
11317                .and_then(|()| self.begin_observation("syncular_image_chunk"))
11318                .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
11319            let started = std::time::Instant::now();
11320            let result = (|| {
11321                self.rebuild_overlay_if_dirty()
11322                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
11323                let mut changed_keys = Vec::new();
11324                let mut batch = ChangeAccumulator::default();
11325                if clear && self.clear_scope(table, effective, unit_id)? {
11326                    batch.table(&table.name);
11327                }
11328                let mut processed = 0u32;
11329                {
11330                    let mut ins = self.conn.prepare_cached(&insert).map_err(|error| {
11331                        invalid(Self::sqlite_failure(&self.storage_failure, error))
11332                    })?;
11333                    let mut mirror = if mirror_visible {
11334                        Some(self.conn.prepare_cached(&visible_insert).map_err(|error| {
11335                            invalid(Self::sqlite_failure(&self.storage_failure, error))
11336                        })?)
11337                    } else {
11338                        None
11339                    };
11340                    for _ in 0..chunk {
11341                        let Some(row) = rows.next().map_err(|error| {
11342                            invalid(Self::sqlite_failure(&self.storage_failure, error))
11343                        })?
11344                        else {
11345                            break;
11346                        };
11347                        for (i, column) in table.columns.iter().enumerate() {
11348                            let cell = row.get_ref(i).map_err(|error| {
11349                                invalid(Self::sqlite_failure(&self.storage_failure, error))
11350                            })?;
11351                            let param = image_cell_param(column, cell).map_err(invalid)?;
11352                            ins.raw_bind_parameter(i + 1, &param).map_err(|error| {
11353                                invalid(Self::sqlite_failure(&self.storage_failure, error))
11354                            })?;
11355                            if let Some(mirror) = &mut mirror {
11356                                mirror.raw_bind_parameter(i + 1, &param).map_err(|error| {
11357                                    invalid(Self::sqlite_failure(&self.storage_failure, error))
11358                                })?;
11359                            }
11360                        }
11361                        let version_index = table.columns.len();
11362                        let version: i64 = row.get(version_index).map_err(|error| {
11363                            invalid(Self::sqlite_failure(&self.storage_failure, error))
11364                        })?;
11365                        if version < 1 {
11366                            return Err(invalid("image row version must be positive".into()));
11367                        }
11368                        ins.raw_bind_parameter(version_index + 1, version)
11369                            .map_err(|error| {
11370                                invalid(Self::sqlite_failure(&self.storage_failure, error))
11371                            })?;
11372                        // §4.5 version guard: the base upsert decides.
11373                        let landed = ins.raw_execute().map_err(|error| {
11374                            invalid(Self::sqlite_failure(&self.storage_failure, error))
11375                        })? > 0;
11376                        processed += 1;
11377                        if processed.is_multiple_of(1024) && processed < chunk {
11378                            self.progress
11379                                .update(|p| p.rows_processed = u64::from(applied + processed));
11380                        }
11381                        if !landed {
11382                            continue;
11383                        }
11384                        if let Some(mirror) = &mut mirror {
11385                            mirror
11386                                .raw_bind_parameter(version_index + 1, version)
11387                                .map_err(|error| {
11388                                    invalid(Self::sqlite_failure(&self.storage_failure, error))
11389                                })?;
11390                            mirror.raw_execute().map_err(|error| {
11391                                invalid(Self::sqlite_failure(&self.storage_failure, error))
11392                            })?;
11393                        }
11394                        if reconcile_pending {
11395                            changed_keys.push(row.get::<_, SqlValue>(table.pk_index).map_err(
11396                                |error| invalid(Self::sqlite_failure(&self.storage_failure, error)),
11397                            )?);
11398                        }
11399                    }
11400                }
11401                if processed > 0 {
11402                    batch.table(&table.name);
11403                    self.overlay_dirty.table(&table.name);
11404                }
11405                self.stored(record_applied_rows(
11406                    &self.conn,
11407                    unit_id,
11408                    segment_id,
11409                    i64::from(applied + processed),
11410                ))
11411                .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
11412                self.reconcile_import(table, &mut changed_keys, protected, reconcile_pending)
11413                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
11414                let written = std::time::Instant::now();
11415                self.finish_observation("syncular_image_chunk", batch)
11416                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
11417                Ok((processed, written))
11418            })();
11419            let (processed, written) = match result {
11420                Ok(done) => done,
11421                Err(error) => {
11422                    self.rollback_observation("syncular_image_chunk");
11423                    self.restore(rollback);
11424                    return Err(error);
11425                }
11426            };
11427            // The chunk commits before the yield, also inside a coalesced apply.
11428            let coalesced = self.coalesced.is_some();
11429            if coalesced {
11430                self.commit_coalesced_apply()
11431                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
11432            }
11433            write += written - started;
11434            commit += written.elapsed();
11435            applied += processed;
11436            let last = processed < chunk || i64::from(applied) == actual_count;
11437            // A file replica's WAL keeps its high-water size until truncated.
11438            if last && self.conn.is_autocommit() {
11439                self.conn
11440                    .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))
11441                    .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
11442            }
11443            if coalesced {
11444                self.begin_coalesced_apply()
11445                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
11446            }
11447            self.progress.update(|p| {
11448                p.rows_processed = u64::from(applied);
11449                p.import_write_ms = u64::try_from(write.as_millis()).ok();
11450                p.import_commit_ms = u64::try_from(commit.as_millis()).ok();
11451            });
11452            let duration = started.elapsed().as_secs_f64() * 1000.0;
11453            #[cfg(test)]
11454            let duration = self.image_chunk_ms.get().unwrap_or(duration);
11455            std::thread::yield_now();
11456            if last {
11457                break;
11458            }
11459            // A zero duration saturates the cast and leaves the doubling.
11460            limit = (processed * 2)
11461                .min(65_536)
11462                .min((f64::from(processed) * 250.0 / duration) as u32)
11463                .max(1024);
11464            self.image_chunk_rows.insert(table.name.clone(), limit);
11465        }
11466        if i64::from(applied) != actual_count {
11467            return Err(invalid(format!(
11468                "image yielded {applied} rows, expected {actual_count}"
11469            )));
11470        }
11471        Ok(applied - resumed)
11472    }
11473
11474    /// §7.1 keyed reconciliation of the rows `drained` commits touch: restore
11475    /// them from the base and replay their remaining intent. A table with a
11476    /// secondary unique index restores and replays every row with intent,
11477    /// since one row's value can admit or defer another row's write.
11478    fn reconcile_rows(&self, drained: &[Arc<OutboxCommit>]) -> Result<(), String> {
11479        let names: BTreeSet<&str> = drained
11480            .iter()
11481            .flat_map(|commit| &commit.ops)
11482            .map(|op| op.table.as_str())
11483            .collect();
11484        for name in names {
11485            let Some(table) = self.schema.table(name) else {
11486                continue;
11487            };
11488            let ops = drained.iter().flat_map(|commit| &commit.ops);
11489            let tables = BTreeSet::from([name]);
11490            let rows: HashSet<_> = ops
11491                .clone()
11492                .map(|op| (op.table.as_str(), op.row_id.as_str()))
11493                .collect();
11494            let unique = table.indexes.iter().any(|index| index.unique);
11495            let acknowledged = if unique {
11496                self.acknowledged_operations(None, Some(&tables))?
11497            } else {
11498                Vec::new()
11499            };
11500            let peers = acknowledged
11501                .iter()
11502                .chain(
11503                    self.failed_commits
11504                        .iter()
11505                        .flat_map(|(commit, _)| &commit.ops),
11506                )
11507                .chain(self.outbox.iter().flat_map(|commit| &commit.ops))
11508                .filter(|_| unique);
11509            self.restore_base_rows(table, &mut Vec::new(), peers.chain(ops.clone()))?;
11510            self.replay_overlay(&tables, (!unique).then_some(&rows))?;
11511        }
11512        Ok(())
11513    }
11514
11515    /// Whether `table` has pending outbox intent, and whether it has protected
11516    /// (acknowledged or retained failed) intent.
11517    fn table_intent(&self, table: &str) -> Result<(bool, bool), String> {
11518        let touches = |op: &OutboxOp| op.table == table;
11519        Ok((
11520            self.outbox
11521                .iter()
11522                .flat_map(|commit| &commit.ops)
11523                .any(touches),
11524            self.failed_commits
11525                .iter()
11526                .flat_map(|(commit, _)| &commit.ops)
11527                .any(touches)
11528                || self
11529                    .conn
11530                    .query_row(
11531                        "SELECT EXISTS(SELECT 1 FROM _syncular_acked_rows WHERE tbl=?1)",
11532                        [table],
11533                        |row| row.get(0),
11534                    )
11535                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
11536        ))
11537    }
11538
11539    /// §7.1 reconciliation after an import block or chunk. Keyed replay covers
11540    /// only pending intent, so a table with protected intent rebuilds; a
11541    /// `restore` restores the imported `keys` and every pending row first.
11542    fn reconcile_import(
11543        &mut self,
11544        table: &TableSchema,
11545        keys: &mut Vec<SqlValue>,
11546        protected: bool,
11547        restore: bool,
11548    ) -> Result<(), String> {
11549        if protected {
11550            return self.rebuild_overlay_if_dirty();
11551        }
11552        let pending = self.outbox.iter().flat_map(|commit| &commit.ops);
11553        if restore {
11554            self.restore_base_rows(table, keys, pending.clone())?;
11555        }
11556        self.apply_outbox_ops(pending.filter(|op| op.table == table.name))?;
11557        self.overlay_dirty.set(false);
11558        Ok(())
11559    }
11560
11561    /// Restore the visible rows of `keys` and of the rows `ops` touch in
11562    /// `table` from the base before the caller replays their intent. The
11563    /// caller owns the transaction and publishes its revision after replay.
11564    fn restore_base_rows<'a>(
11565        &self,
11566        table: &TableSchema,
11567        keys: &mut Vec<SqlValue>,
11568        ops: impl IntoIterator<Item = &'a OutboxOp>,
11569    ) -> Result<(), String> {
11570        let visible = visible_table(&table.name);
11571        let base = base_table(&table.name);
11572        let pk = quote_ident(&table.primary_key);
11573        let predicate = row_id_predicate(table);
11574        let mut lookup = self.conn.prepare_cached(&format!(
11575            "SELECT {pk} FROM {base} WHERE {predicate} UNION SELECT {pk} FROM {visible} WHERE {predicate}"
11576        )).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11577        let mut deleted = HashSet::new();
11578        for op in ops.into_iter().filter(|op| op.table == table.name) {
11579            if op.upsert {
11580                if let Some(values) = &op.values {
11581                    let key = json_to_column_value(
11582                        &table.columns[table.pk_index],
11583                        values.get(&table.primary_key),
11584                    )?;
11585                    keys.push(owned_sql_value(RowParam::Cell(&key))?);
11586                }
11587            } else if deleted.insert(&op.row_id) {
11588                keys.extend(
11589                    lookup
11590                        .query_map([&op.row_id], |row| row.get::<_, SqlValue>(0))
11591                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
11592                        .collect::<Result<Vec<_>, _>>()
11593                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
11594                );
11595            }
11596        }
11597        // Delete the complete affected set before restoring any base value, so
11598        // old optimistic unique values cannot collide with the new server rows.
11599        for chunk in keys.chunks(400) {
11600            let holes = vec!["?"; chunk.len()].join(",");
11601            self.conn
11602                .prepare_cached(&format!("DELETE FROM {visible} WHERE {pk} IN ({holes})"))
11603                .and_then(|mut statement| statement.execute(rusqlite::params_from_iter(chunk)))
11604                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11605        }
11606        let updates = table
11607            .columns
11608            .iter()
11609            .map(|column| column.name.as_str())
11610            .chain(std::iter::once("_syncular_version"))
11611            .chain(table.sealed_columns().iter().copied())
11612            .filter(|name| *name != table.primary_key)
11613            .map(|name| {
11614                let name = quote_ident(name);
11615                format!("{name}=excluded.{name}")
11616            })
11617            .collect::<Vec<_>>()
11618            .join(",");
11619        for chunk in keys.chunks(400) {
11620            let holes = vec!["?"; chunk.len()].join(",");
11621            self.conn.prepare_cached(&format!("INSERT INTO {visible} SELECT * FROM {base} WHERE {pk} IN ({holes}) ON CONFLICT ({pk}) DO UPDATE SET {updates}"))
11622                .and_then(|mut statement| statement.execute(rusqlite::params_from_iter(chunk)))
11623                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11624        }
11625        Ok(())
11626    }
11627
11628    // -- application-authorized local purge ---------------------------------------
11629
11630    fn compile_local_data_purge(
11631        &self,
11632        input: &LocalDataPurgeInput,
11633    ) -> Result<(Vec<CompiledLocalDataPurgeTarget>, String), String> {
11634        let invalid = |message: String| format!("sync.invalid_request: {message}");
11635        if input.purge_id.is_empty()
11636            || input.purge_id.len() > 128
11637            || !is_local_operation_code_like(&input.purge_id)
11638        {
11639            return Err(invalid(
11640                "local purge purgeId must be a 1–128 character code-like identifier".to_owned(),
11641            ));
11642        }
11643        if input.targets.is_empty() || input.targets.len() > MAX_LOCAL_PURGE_TARGETS {
11644            return Err(invalid(format!(
11645                "local purge needs between 1 and {MAX_LOCAL_PURGE_TARGETS} targets"
11646            )));
11647        }
11648
11649        let mut deduplicated: BTreeMap<
11650            String,
11651            (CompiledLocalDataPurgeTarget, LocalDataPurgeTarget),
11652        > = BTreeMap::new();
11653        for target in &input.targets {
11654            let table = self.schema.table(&target.table).ok_or_else(|| {
11655                invalid(format!(
11656                    "local purge names unknown table {:?}",
11657                    target.table
11658                ))
11659            })?;
11660            if target.selectors.is_empty() || target.selectors.len() > MAX_LOCAL_PURGE_SELECTORS {
11661                return Err(invalid(format!(
11662                    "local purge target {:?} needs between 1 and {MAX_LOCAL_PURGE_SELECTORS} selectors",
11663                    target.table
11664                )));
11665            }
11666            let mut selectors = Vec::with_capacity(target.selectors.len());
11667            let mut canonical_selectors = BTreeMap::new();
11668            for (column_name, raw_values) in &target.selectors {
11669                let Some((column_index, column)) = table
11670                    .columns
11671                    .iter()
11672                    .enumerate()
11673                    .find(|(_, column)| column.name == *column_name)
11674                else {
11675                    return Err(invalid(format!(
11676                        "local purge target {:?} names unknown column {:?}",
11677                        target.table, column_name
11678                    )));
11679                };
11680                let encrypted = table
11681                    .encrypted_columns
11682                    .iter()
11683                    .any(|candidate| candidate.index == column_index);
11684                if column.ty != ColumnType::String || encrypted {
11685                    return Err(invalid(format!(
11686                        "local purge selector {:?}.{:?} must be a plaintext string column",
11687                        target.table, column_name
11688                    )));
11689                }
11690                if raw_values.is_empty() || raw_values.len() > MAX_LOCAL_PURGE_VALUES {
11691                    return Err(invalid(format!(
11692                        "local purge selector {:?}.{:?} needs between 1 and {MAX_LOCAL_PURGE_VALUES} values",
11693                        target.table, column_name
11694                    )));
11695                }
11696                let mut values = raw_values.clone();
11697                values.sort();
11698                values.dedup();
11699                if values.iter().any(|value| {
11700                    value.is_empty()
11701                        || value.len() > MAX_LOCAL_PURGE_VALUE_LENGTH
11702                        || !is_local_operation_code_like(value)
11703                }) {
11704                    return Err(invalid(format!(
11705                        "local purge selector values must be 1–{MAX_LOCAL_PURGE_VALUE_LENGTH} character code-like identifiers"
11706                    )));
11707                }
11708                selectors.push((column_name.clone(), values.clone()));
11709                canonical_selectors.insert(column_name.clone(), values);
11710            }
11711            let canonical = LocalDataPurgeTarget {
11712                table: target.table.clone(),
11713                selectors: canonical_selectors,
11714            };
11715            let key = serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
11716            deduplicated.insert(
11717                key,
11718                (
11719                    CompiledLocalDataPurgeTarget {
11720                        table: target.table.clone(),
11721                        selectors,
11722                    },
11723                    canonical,
11724                ),
11725            );
11726        }
11727        let targets = deduplicated
11728            .values()
11729            .map(|(compiled, _)| compiled.clone())
11730            .collect::<Vec<_>>();
11731        let canonical = deduplicated
11732            .values()
11733            .map(|(_, target)| target.clone())
11734            .collect::<Vec<_>>();
11735        let canonical_plan =
11736            serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
11737        Ok((targets, canonical_plan))
11738    }
11739
11740    fn local_purge_base_row_ids(
11741        &self,
11742        targets: &[CompiledLocalDataPurgeTarget],
11743    ) -> Result<BTreeMap<String, BTreeSet<String>>, String> {
11744        let mut by_table: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
11745        for target in targets {
11746            let table = self
11747                .schema
11748                .table(&target.table)
11749                .ok_or_else(|| format!("sync.invalid_request: unknown table {:?}", target.table))?;
11750            let mut clauses = Vec::with_capacity(target.selectors.len());
11751            let mut params = Vec::new();
11752            for (column, values) in &target.selectors {
11753                clauses.push(format!(
11754                    "{} IN ({})",
11755                    quote_ident(column),
11756                    values.iter().map(|_| "?").collect::<Vec<_>>().join(", ")
11757                ));
11758                params.extend(values.iter().cloned().map(SqlValue::Text));
11759            }
11760            let sql = format!(
11761                "SELECT CAST({} AS TEXT) FROM {} WHERE {}",
11762                quote_ident(&table.primary_key),
11763                base_table(&target.table),
11764                clauses.join(" AND ")
11765            );
11766            let mut statement = self
11767                .conn
11768                .prepare(&sql)
11769                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11770            let rows = statement
11771                .query_map(rusqlite::params_from_iter(params), |row| {
11772                    let value = sql_ref_to_json(&table.columns[table.pk_index], row.get_ref(0)?);
11773                    render_row_id_json(Some(&value))
11774                        .map_err(|message| rusqlite::Error::ToSqlConversionFailure(message.into()))
11775                })
11776                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11777            let ids = by_table.entry(target.table.clone()).or_default();
11778            for row in rows {
11779                ids.insert(
11780                    row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
11781                );
11782            }
11783        }
11784        Ok(by_table)
11785    }
11786
11787    fn local_purge_values_match(
11788        target: &CompiledLocalDataPurgeTarget,
11789        values: &Map<String, Value>,
11790    ) -> bool {
11791        target.selectors.iter().all(|(column, allowed)| {
11792            values
11793                .get(column)
11794                .and_then(Value::as_str)
11795                .is_some_and(|value| allowed.iter().any(|candidate| candidate == value))
11796        })
11797    }
11798
11799    /// Apply one host-authorized local security purge. The host MUST gate the
11800    /// corresponding subscriptions first; this primitive owns local SQLite
11801    /// cleanup and never grants/revokes server authority by itself.
11802    pub fn purge_local_data(
11803        &mut self,
11804        input: &LocalDataPurgeInput,
11805    ) -> Result<LocalDataPurgeResult, String> {
11806        let (targets, canonical_plan) = self.compile_local_data_purge(input)?;
11807        let meta_key = format!("localPurge:{}", input.purge_id);
11808        let applied_plan = self
11809            .conn
11810            .query_row(
11811                "SELECT value FROM _syncular_meta WHERE key = ?1",
11812                rusqlite::params![meta_key],
11813                |row| row.get::<_, String>(0),
11814            )
11815            .optional()
11816            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11817        if let Some(applied_plan) = applied_plan {
11818            if applied_plan != canonical_plan {
11819                return Err(format!(
11820                    "sync.invalid_request: local purge id {:?} was already used with a different plan",
11821                    input.purge_id
11822                ));
11823            }
11824            // A retry after a failed rewrite finishes it.
11825            self.erase_purged_pages()?;
11826            return Ok(LocalDataPurgeResult {
11827                already_applied: true,
11828                purged_rows: 0,
11829                dropped_commits: 0,
11830            });
11831        }
11832
11833        // RFC 0005 D7: the container file cannot be a purge target (its name is
11834        // not in the running schema), so purge drops it as a fixed step, before
11835        // any mirror row is touched and unconditionally.
11836        let _ = self.drop_previous_version();
11837
11838        let rollback = self.capture();
11839        self.begin_observation("syncular_local_purge")?;
11840        let applied = (|| -> Result<(ChangeAccumulator, LocalDataPurgeResult), String> {
11841            let row_ids = self.local_purge_base_row_ids(&targets)?;
11842            let retained_dropped = self.drop_failed_commits(|table, values| {
11843                targets.iter().any(|target| {
11844                    target.table == table && Self::local_purge_values_match(target, values)
11845                })
11846            })?;
11847            let doomed = self
11848                .outbox
11849                .iter()
11850                .filter(|commit| {
11851                    commit.ops.iter().any(|operation| {
11852                        let matching_targets = targets
11853                            .iter()
11854                            .filter(|target| target.table == operation.table)
11855                            .collect::<Vec<_>>();
11856                        if matching_targets.is_empty() {
11857                            return false;
11858                        }
11859                        if row_ids
11860                            .get(&operation.table)
11861                            .is_some_and(|ids| ids.contains(&operation.row_id))
11862                        {
11863                            return true;
11864                        }
11865                        operation.values.as_ref().is_some_and(|values| {
11866                            matching_targets
11867                                .iter()
11868                                .any(|target| Self::local_purge_values_match(target, values))
11869                        })
11870                    })
11871                })
11872                .cloned()
11873                .collect::<Vec<_>>();
11874            let doomed_ids = doomed
11875                .iter()
11876                .map(|commit| commit.client_commit_id.clone())
11877                .collect::<BTreeSet<_>>();
11878            let mut batch = ChangeAccumulator::default();
11879            for commit in &retained_dropped {
11880                batch.conflicts = true;
11881                batch.rejections = true;
11882                batch.outcomes = true;
11883                for operation in &commit.ops {
11884                    batch.table(&operation.table);
11885                }
11886            }
11887            let mut rejections = Vec::new();
11888            // §5.11 purge erasure: the journal keeps a dropped commit's identity and
11889            // codes, never its operations, so no purged value survives in it.
11890            for commit in &doomed {
11891                for operation in &commit.ops {
11892                    batch.table(&operation.table);
11893                    self.overlay_dirty.table(&operation.table);
11894                }
11895                let results = (0..commit.ops.len())
11896                    .map(|op_index| {
11897                        let rejection = RejectionRecord {
11898                            client_commit_id: commit.client_commit_id.clone(),
11899                            op_index: op_index as i32,
11900                            code: "client.local_data_purged".to_owned(),
11901                            message: "the commit was dropped by an application-authorized local data purge".to_owned(),
11902                            retryable: false,
11903                            details: None,
11904                            operation: None,
11905                        };
11906                        rejections.push(rejection.clone());
11907                        CommitOperationOutcome::Error { rejection }
11908                    })
11909                    .collect::<Vec<_>>();
11910                self.persist_commit_outcome(
11911                    &commit.client_commit_id,
11912                    CommitOutcomeStatus::Rejected,
11913                    &results,
11914                    None,
11915                )?;
11916                self.delete_outbox_persisted(&commit.client_commit_id)?;
11917            }
11918            if !doomed.is_empty() {
11919                self.outbox
11920                    .retain(|commit| !doomed_ids.contains(&commit.client_commit_id));
11921                self.rejections.extend(rejections);
11922                self.prune_commit_outcomes()?;
11923                batch.status = true;
11924                batch.rejections = true;
11925                batch.outcomes = true;
11926            }
11927
11928            let mut purged_rows = 0usize;
11929            for (table, ids) in &row_ids {
11930                if ids.is_empty() {
11931                    continue;
11932                }
11933                batch.table(table);
11934                self.overlay_dirty.table(table);
11935                for row_id in ids {
11936                    self.delete_base_row(table, row_id, i64::MAX)?;
11937                    purged_rows += 1;
11938                }
11939            }
11940            // §5.11 purge erasure: a journaled outcome with an operation or
11941            // conflict row the purge reaches keeps its identity, status, ids
11942            // and codes only.
11943            let touches = |table: &str, row_id: &str, values: Option<&Map<String, Value>>| {
11944                targets.iter().any(|target| {
11945                    target.table == table
11946                        && (row_ids.get(table).is_some_and(|ids| ids.contains(row_id))
11947                            || values.is_some_and(|values| {
11948                                Self::local_purge_values_match(target, values)
11949                            }))
11950                })
11951            };
11952            let entries = self
11953                .conn
11954                .prepare("SELECT client_commit_id, results_json, operations_json FROM _syncular_commit_outcomes")
11955                .and_then(|mut statement| {
11956                    statement
11957                        .query_map([], |row| {
11958                            Ok((
11959                                row.get::<_, String>(0)?,
11960                                row.get::<_, String>(1)?,
11961                                row.get::<_, Option<String>>(2)?,
11962                            ))
11963                        })?
11964                        .collect::<Result<Vec<_>, _>>()
11965                })
11966                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11967            for (client_commit_id, results_json, operations_json) in entries {
11968                let corrupt = |_| "sync.local_corrupt: persisted commit outcome is invalid".to_owned();
11969                let mut results: Vec<CommitOperationOutcome> =
11970                    serde_json::from_str(&results_json).map_err(corrupt)?;
11971                let operations: Vec<CommitOperation> = operations_json
11972                    .map(|json| serde_json::from_str(&json))
11973                    .transpose()
11974                    .map_err(corrupt)?
11975                    .unwrap_or_default();
11976                let named = operations
11977                    .iter()
11978                    .chain(results.iter().filter_map(|result| match result {
11979                        CommitOperationOutcome::Conflict { conflict } => conflict.operation.as_ref(),
11980                        CommitOperationOutcome::Error { rejection } => rejection.operation.as_ref(),
11981                        CommitOperationOutcome::Applied { .. } => None,
11982                    }))
11983                    .any(|operation| touches(&operation.table, &operation.row_id, operation.values.as_ref()))
11984                    || results.iter().any(|result| {
11985                        matches!(result, CommitOperationOutcome::Conflict { conflict }
11986                            if touches(&conflict.table, &conflict.row_id, Some(&conflict.server_row)))
11987                    });
11988                if !named {
11989                    continue;
11990                }
11991                for result in &mut results {
11992                    match result {
11993                        CommitOperationOutcome::Conflict { conflict } => {
11994                            conflict.server_row = Map::new();
11995                            conflict.conflict_columns.clear();
11996                            conflict.operation = None;
11997                        }
11998                        CommitOperationOutcome::Error { rejection } => {
11999                            rejection.details = None;
12000                            rejection.operation = None;
12001                        }
12002                        CommitOperationOutcome::Applied { .. } => {}
12003                    }
12004                }
12005                let results_json = serde_json::to_string(&results).map_err(|error| error.to_string())?;
12006                self.conn
12007                    .execute(
12008                        "UPDATE _syncular_commit_outcomes SET results_json = ?1, operations_json = NULL WHERE client_commit_id = ?2",
12009                        rusqlite::params![results_json, client_commit_id],
12010                    )
12011                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12012                batch.outcomes = true;
12013            }
12014            self.rebuild_overlay_if_dirty()?;
12015            self.delete_unreferenced_cached_blobs()?;
12016            self.conn
12017                .execute(
12018                    "INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
12019                    rusqlite::params![meta_key, canonical_plan],
12020                )
12021                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12022            Ok((
12023                batch,
12024                LocalDataPurgeResult {
12025                    already_applied: false,
12026                    purged_rows,
12027                    dropped_commits: doomed.len(),
12028                },
12029            ))
12030        })()
12031        .and_then(|(batch, result)| {
12032            self.finish_observation("syncular_local_purge", batch)
12033                .map(|()| result)
12034        });
12035        let result = match applied {
12036            Ok(result) => result,
12037            Err(error) => {
12038                self.rollback_observation("syncular_local_purge");
12039                self.restore(rollback);
12040                return Err(error);
12041            }
12042        };
12043        self.cancel_sync_round();
12044        self.erase_purged_pages()?;
12045        Ok(result)
12046    }
12047
12048    /// §5.11 purge erasure: SQLite leaves a deleted value in the freed pages of the
12049    /// database file, and the WAL keeps the frames written before the purge,
12050    /// until either is rewritten. VACUUM rebuilds the file from the remaining
12051    /// rows, and the checkpoint truncates the WAL.
12052    fn erase_purged_pages(&self) -> Result<(), String> {
12053        let erased = self
12054            .conn
12055            .execute_batch("PRAGMA secure_delete = ON; VACUUM;")
12056            .and_then(|()| {
12057                self.conn
12058                    .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))
12059            });
12060        let reset = self.conn.execute_batch("PRAGMA secure_delete = OFF");
12061        erased
12062            .and(reset)
12063            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
12064    }
12065
12066    /// Application-authorized recovery of the replicated projection. Unlike a
12067    /// local security purge, this retains the entire outbox, device identity,
12068    /// lease, outcomes, unit registrations, and protected bookkeeping.
12069    /// The projection reset, unit rewind, optimistic replay, and
12070    /// idempotency marker share one savepoint for interruption safety.
12071    pub fn rebootstrap_local_data(
12072        &mut self,
12073        input: &LocalDataRebootstrapInput,
12074    ) -> Result<LocalDataRebootstrapResult, String> {
12075        if input.rebootstrap_id.is_empty()
12076            || input.rebootstrap_id.len() > 128
12077            || !is_local_operation_code_like(&input.rebootstrap_id)
12078        {
12079            return Err(
12080                "sync.invalid_request: local rebootstrap rebootstrapId must be a 1–128 character code-like identifier"
12081                    .to_owned(),
12082            );
12083        }
12084        let meta_key = format!("localRebootstrap:{}", input.rebootstrap_id);
12085        if let Some(persisted) = meta_get(&self.conn, &meta_key).map_err(|_| {
12086            "sync.local_corrupt: persisted local rebootstrap receipt is unreadable".to_owned()
12087        })? {
12088            let (retained_commits, reset_subscriptions) =
12089                decode_local_rebootstrap_receipt(&persisted)?;
12090            return Ok(LocalDataRebootstrapResult {
12091                already_applied: true,
12092                retained_commits,
12093                reset_subscriptions,
12094            });
12095        }
12096        if self.stopped || self.schema_floor.is_some() {
12097            return Err(
12098                "sync.invalid_request: local rebootstrap cannot bypass an active schema-floor stop; update the application first"
12099                    .to_owned(),
12100            );
12101        }
12102
12103        let retained_commits = self.outbox.len();
12104        let reset_subscriptions = self.stored(load_units(&self.conn))?.len();
12105        let receipt = encode_local_rebootstrap_receipt(retained_commits, reset_subscriptions)?;
12106        let rollback = self.capture();
12107        self.begin_observation("syncular_local_rebootstrap")?;
12108        let mut batch = ChangeAccumulator::default();
12109        let applied = (|| -> Result<(), String> {
12110            self.run_schema_reset_observed(&mut batch, false)?;
12111            self.conn
12112                .execute(
12113                    "INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
12114                    rusqlite::params![meta_key, receipt],
12115                )
12116                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12117            self.sync_needed = true;
12118            self.sync_intent_queue.push_back(SyncIntent::Interactive);
12119            batch.status = true;
12120            Ok(())
12121        })()
12122        .and_then(|()| self.finish_observation("syncular_local_rebootstrap", batch));
12123        if let Err(error) = applied {
12124            self.rollback_observation("syncular_local_rebootstrap");
12125            self.restore(rollback);
12126            return Err(error);
12127        }
12128        self.cancel_sync_round();
12129
12130        Ok(LocalDataRebootstrapResult {
12131            already_applied: false,
12132            retained_commits,
12133            reset_subscriptions,
12134        })
12135    }
12136
12137    // -- scope purge + doomed outbox (§3.3) ----------------------------------------
12138
12139    /// Delete base and visible rows owned by the effective scopes
12140    /// ([`local_scope_predicate`]); `Ok(false)` = no local scope-column
12141    /// mapping for a key (the fail-closed case). A SQLite failure is an error.
12142    fn purge_scope_rows(
12143        &mut self,
12144        table_name: &str,
12145        effective: &[(String, Vec<String>)],
12146        owners: &[Vec<(String, Vec<String>)>],
12147    ) -> Result<bool, String> {
12148        let Some(table) = self.schema.table(table_name) else {
12149            return Ok(false);
12150        };
12151        let Ok(predicate) = local_scope_predicate(table, effective, owners) else {
12152            return Ok(false);
12153        };
12154        let Some((predicate, params)) = predicate else {
12155            return Ok(true);
12156        };
12157        self.overlay_dirty.table(table_name);
12158        for source in [base_table(table_name), visible_table(table_name)] {
12159            self.conn
12160                .execute(
12161                    &format!("DELETE FROM {source} WHERE {predicate}"),
12162                    rusqlite::params_from_iter(&params),
12163                )
12164                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12165        }
12166        Ok(true)
12167    }
12168
12169    /// §3.3: drop pending commits whose upserts provably land in the
12170    /// revoked effective scopes and no other owner covers (row ownership)
12171    /// — whole-commit, never per-operation.
12172    fn drop_doomed_outbox(
12173        &mut self,
12174        table: &TableSchema,
12175        effective: &[(String, Vec<String>)],
12176        owners: &[Vec<(String, Vec<String>)>],
12177    ) -> Result<bool, String> {
12178        let doomed: Vec<_> =
12179            self.outbox
12180                .iter()
12181                .filter(|commit| {
12182                    commit.ops.iter().any(|op| {
12183                        op.upsert
12184                            && op.table == table.name
12185                            && op.values.as_ref().is_some_and(|values| {
12186                                owns_local_row(table, effective, owners, values)
12187                            })
12188                    })
12189                })
12190                .cloned()
12191                .collect();
12192        if doomed.is_empty() {
12193            return Ok(false);
12194        }
12195        let mut rejections = Vec::new();
12196        for commit in &doomed {
12197            let results = commit
12198                .ops
12199                .iter()
12200                .enumerate()
12201                .map(|(op_index, operation)| {
12202                    self.overlay_dirty.table(&operation.table);
12203                    let rejection = RejectionRecord {
12204                        client_commit_id: commit.client_commit_id.clone(),
12205                        op_index: op_index as i32,
12206                        code: "sync.scope_revoked".to_owned(),
12207                        message: "the commit was dropped because its effective scope was revoked"
12208                            .to_owned(),
12209                        retryable: false,
12210                        details: None,
12211                        operation: Some(CommitOperation::from(operation)),
12212                    };
12213                    rejections.push(rejection.clone());
12214                    CommitOperationOutcome::Error { rejection }
12215                })
12216                .collect::<Vec<_>>();
12217            self.persist_commit_outcome(
12218                &commit.client_commit_id,
12219                CommitOutcomeStatus::Rejected,
12220                &results,
12221                Some(&commit.ops),
12222            )?;
12223            self.delete_outbox_persisted(&commit.client_commit_id)?;
12224        }
12225        self.prune_commit_outcomes()?;
12226        let doomed_ids = doomed
12227            .iter()
12228            .map(|commit| commit.client_commit_id.as_str())
12229            .collect::<BTreeSet<_>>();
12230        self.outbox
12231            .retain(|commit| !doomed_ids.contains(commit.client_commit_id.as_str()));
12232        self.rejections.extend(rejections);
12233        Ok(true)
12234    }
12235
12236    // -- blobs (§5.9) ----------------------------------------------------------------
12237
12238    /// §5.9.7: hash bytes into the content address, cache them, queue the
12239    /// upload (flushed before the next push, B4). Returns the canonical
12240    /// BlobRef JSON `{blobId, byteLength, mediaType?}` for a `blob_ref`
12241    /// column value.
12242    pub fn upload_blob(
12243        &mut self,
12244        bytes: &[u8],
12245        media_type: Option<String>,
12246        name: Option<String>,
12247    ) -> Result<Value, String> {
12248        let blob_id = blob_id_for(bytes);
12249        let now = self.clock_now_ms();
12250        let tx = self
12251            .conn
12252            .transaction()
12253            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12254        tx.execute(
12255            "INSERT INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES (?,?,?,?,?)
12256             ON CONFLICT(blob_id) DO NOTHING",
12257            rusqlite::params![blob_id, bytes, bytes.len() as i64, media_type, now],
12258        )
12259        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12260        tx.execute(
12261            "INSERT OR IGNORE INTO _syncular_blob_uploads(blob_id, media_type, created_at_ms) VALUES (?,?,?)",
12262            rusqlite::params![blob_id, media_type, now],
12263        )
12264        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12265        tx.commit()
12266            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12267        self.enforce_blob_cache_cap();
12268        let mut obj = Map::new();
12269        obj.insert("blobId".to_owned(), Value::from(blob_id));
12270        obj.insert("byteLength".to_owned(), Value::from(bytes.len() as i64));
12271        if let Some(mt) = media_type {
12272            obj.insert("mediaType".to_owned(), Value::from(mt));
12273        }
12274        if let Some(n) = name {
12275            obj.insert("name".to_owned(), Value::from(n));
12276        }
12277        Ok(Value::Object(obj))
12278    }
12279
12280    /// §5.9.7: resolve an owned blob body without driver encoding. A
12281    /// content-addressed cache hit serves with no fetch (B1); a miss downloads
12282    /// (§5.9.5), verifies the address, caches the bytes, and returns them.
12283    pub fn fetch_blob_bytes(
12284        &mut self,
12285        transport: &mut dyn Transport,
12286        blob_id_or_ref: &str,
12287    ) -> Result<FetchedBlob, (String, String)> {
12288        let simple = |m: String| ("client.failed".to_owned(), m);
12289        // A BlobRef declares the size that caps the download (§5.9.5); a bare
12290        // blobId declares none.
12291        let (blob_id, byte_length) = if blob_id_or_ref.starts_with("sha256:") {
12292            (blob_id_or_ref.to_owned(), None)
12293        } else {
12294            let value: Value = serde_json::from_str(blob_id_or_ref)
12295                .map_err(|_| simple("blob ref is not JSON".to_owned()))?;
12296            (
12297                value
12298                    .get("blobId")
12299                    .and_then(Value::as_str)
12300                    .ok_or_else(|| simple("blob ref has no blobId".to_owned()))?
12301                    .to_owned(),
12302                value.get("byteLength").and_then(Value::as_u64),
12303            )
12304        };
12305        if let Some(cached) = self.get_cached_blob_bytes(&blob_id).map_err(simple)? {
12306            return Ok(cached);
12307        }
12308        if !self.transport_enabled {
12309            return Err(("sync.offline".into(), "transport gate is closed".into()));
12310        }
12311        // §5.9.5: propagate the server's blob.* code (blob.forbidden /
12312        // blob.not_found) verbatim so the harness can assert on it. The
12313        // authorized endpoint serves bytes inline OR (always-issue, presign
12314        // configured) a signed url the client fetches directly — no host auth,
12315        // no fall-through: failure => re-request (the caller's next fetch_blob_bytes).
12316        #[cfg(feature = "bench-internals")]
12317        let download_phase = self.benchmark_phases.start(Phase::BlobDownload);
12318        let bytes = match transport
12319            .blob_download(&blob_id, byte_length)
12320            .map_err(|e| (e.code, e.message))?
12321        {
12322            BlobDownload::Bytes(bytes) => bytes,
12323            BlobDownload::Url {
12324                url,
12325                url_expires_at_ms,
12326            } => {
12327                // §5.9.5: MUST NOT start a fetch at/past expiry.
12328                if url_expires_at_ms.is_some_and(|exp| exp <= self.clock_now_ms()) {
12329                    return Err((
12330                        "sync.segment_expired".to_owned(),
12331                        format!(
12332                            "blob url for {blob_id} expired before fetch — re-request mints a fresh url (§5.9.5)"
12333                        ),
12334                    ));
12335                }
12336                transport
12337                    .fetch_blob_url(&url, byte_length)
12338                    .map_err(|e| (e.code, e.message))?
12339            }
12340        };
12341        #[cfg(feature = "bench-internals")]
12342        drop(download_phase);
12343        #[cfg(feature = "bench-internals")]
12344        let validate_phase = self.benchmark_phases.start(Phase::BlobValidate);
12345        // §5.9.5 inherits §5.1: the declared size caps the body before the
12346        // content address is verified.
12347        if byte_length.is_some_and(|cap| bytes.len() as u64 > cap) {
12348            return Err((
12349                "sync.invalid_request".to_owned(),
12350                "blob exceeds its byteLength".to_owned(),
12351            ));
12352        }
12353        if blob_id_for(&bytes) != blob_id {
12354            return Err(simple(format!(
12355                "blob content address mismatch for {blob_id}"
12356            )));
12357        }
12358        #[cfg(feature = "bench-internals")]
12359        drop(validate_phase);
12360        #[cfg(feature = "bench-internals")]
12361        let insert_phase = self.benchmark_phases.start(Phase::BlobCacheInsert);
12362        let now = self.clock_now_ms();
12363        self.conn
12364            .execute(
12365                "INSERT OR IGNORE INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES (?,?,?,NULL,?)",
12366                rusqlite::params![blob_id, bytes, bytes.len() as i64, now],
12367            )
12368            .map_err(|e| simple(e.to_string()))?;
12369        self.conn
12370            .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))
12371            .map_err(|e| simple(e.to_string()))?;
12372        self.enforce_blob_cache_cap();
12373        #[cfg(feature = "bench-internals")]
12374        drop(insert_phase);
12375        self.get_cached_blob_bytes(&blob_id)
12376            .map_err(simple)?
12377            .ok_or_else(|| simple("blob cache write failed".to_owned()))
12378    }
12379
12380    fn get_cached_blob_bytes(&self, blob_id: &str) -> Result<Option<FetchedBlob>, String> {
12381        #[cfg(feature = "bench-internals")]
12382        let _phase = self.benchmark_phases.start(Phase::BlobCacheRead);
12383        let mut stmt = self
12384            .conn
12385            .prepare("SELECT bytes, byte_length, media_type FROM _syncular_blobs WHERE blob_id = ?")
12386            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12387        let mut rows = stmt
12388            .query(rusqlite::params![blob_id])
12389            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12390        if let Some(row) = rows
12391            .next()
12392            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
12393        {
12394            let bytes: Vec<u8> = row
12395                .get(0)
12396                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12397            let byte_length: i64 = row
12398                .get(1)
12399                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12400            let media_type: Option<String> = row
12401                .get(2)
12402                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12403            return Ok(Some(FetchedBlob {
12404                blob_id: blob_id.to_owned(),
12405                byte_length,
12406                bytes,
12407                media_type,
12408            }));
12409        }
12410        Ok(None)
12411    }
12412
12413    /// §5.9.7 B4: upload every queued blob before push.
12414    fn pending_blob_uploads(&self) -> Result<Vec<crate::round::PendingUpload>, TransportError> {
12415        let pending: Vec<(String, Option<String>)> = {
12416            let mut stmt = self
12417                .conn
12418                .prepare(
12419                    "SELECT blob_id, media_type FROM (
12420                      SELECT blob_id, media_type, created_at_ms FROM _syncular_blob_uploads
12421                      UNION ALL
12422                      SELECT DISTINCT r.blob_id, b.media_type, 9223372036854775807 AS created_at_ms
12423                      FROM _syncular_blob_commit_refs r LEFT JOIN _syncular_blobs b ON b.blob_id = r.blob_id
12424                      WHERE NOT EXISTS (SELECT 1 FROM _syncular_blob_uploads u WHERE u.blob_id = r.blob_id)
12425                    ) ORDER BY created_at_ms, blob_id",
12426                )
12427                .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
12428            let rows = stmt
12429                .query_map([], |row| {
12430                    Ok((row.get::<_, String>(0)?, row.get::<_, Option<String>>(1)?))
12431                })
12432                .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
12433            rows.collect::<Result<_, _>>()
12434                .map_err(|error| match error {
12435                    rusqlite::Error::InvalidColumnType(..)
12436                    | rusqlite::Error::FromSqlConversionFailure(..) => TransportError::new(
12437                        "sync.local_corrupt",
12438                        "Pending blob upload metadata is invalid",
12439                    ),
12440                    _ => TransportError::new("client.failed", error.to_string()),
12441                })?
12442        };
12443        let mut uploads = Vec::new();
12444        for (blob_id, media_type) in pending {
12445            let bytes = (|| {
12446                let stored: Option<(SqlValue, SqlValue)> = self
12447                    .conn
12448                    .query_row(
12449                        "SELECT bytes, byte_length FROM _syncular_blobs WHERE blob_id = ?",
12450                        rusqlite::params![blob_id],
12451                        |row| Ok((row.get(0)?, row.get(1)?)),
12452                    )
12453                    .optional()
12454                    .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
12455                let Some((SqlValue::Blob(bytes), SqlValue::Integer(byte_length))) = stored else {
12456                    return Err(TransportError::new(
12457                        "sync.local_corrupt",
12458                        "Pending blob upload body is missing or corrupt",
12459                    ));
12460                };
12461                if byte_length != bytes.len() as i64 || blob_id_for(&bytes) != blob_id {
12462                    return Err(TransportError::new(
12463                        "sync.local_corrupt",
12464                        "Pending blob upload body is missing or corrupt",
12465                    ));
12466                }
12467                Ok(bytes)
12468            })();
12469            let failed = bytes.is_err();
12470            uploads.push(crate::round::PendingUpload {
12471                id: blob_id,
12472                bytes,
12473                media_type,
12474            });
12475            if failed {
12476                break;
12477            }
12478        }
12479        Ok(uploads)
12480    }
12481
12482    fn visible_blob_ids_sql(&self) -> String {
12483        let mut selects = Vec::new();
12484        for table in &self.schema.tables {
12485            for column in table
12486                .columns
12487                .iter()
12488                .filter(|column| column.ty == ColumnType::BlobRef)
12489            {
12490                let identifier = quote_ident(&column.name);
12491                selects.push(format!(
12492                    "SELECT json_extract({identifier}, '$.blobId') AS blob_id FROM {} \
12493                     WHERE typeof({identifier}) = 'text' AND json_valid({identifier}) \
12494                     AND json_type({identifier}, '$.blobId') = 'text'",
12495                    quote_ident(&table.name),
12496                ));
12497            }
12498        }
12499        if selects.is_empty() {
12500            "SELECT NULL AS blob_id WHERE 0".to_owned()
12501        } else {
12502            selects.join(" UNION ")
12503        }
12504    }
12505
12506    /// §5.9.7 B1 size cap. Visible rows, pending commits, and
12507    /// staged uploads pin bodies directly; no derived refcount is stored.
12508    fn enforce_blob_cache_cap(&self) {
12509        let Some(max_bytes) = self.limits.blob_cache_max_bytes else {
12510            return;
12511        };
12512        let mut total: i64 = self
12513            .conn
12514            .query_row(
12515                "SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs",
12516                [],
12517                |row| row.get(0),
12518            )
12519            .unwrap_or(0);
12520        if total <= max_bytes {
12521            return;
12522        }
12523        let query = format!(
12524            "SELECT blob_id, byte_length FROM _syncular_blobs
12525             WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
12526               AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
12527               AND blob_id NOT IN ({})
12528             ORDER BY created_at_ms ASC, blob_id ASC",
12529            self.visible_blob_ids_sql(),
12530        );
12531        let candidates: Vec<(String, i64)> = {
12532            let Ok(mut stmt) = self.conn.prepare(&query) else {
12533                return;
12534            };
12535            let Ok(rows) = stmt.query_map([], |row| {
12536                Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?))
12537            }) else {
12538                return;
12539            };
12540            rows.filter_map(Result::ok).collect()
12541        };
12542        for (blob_id, byte_length) in candidates {
12543            if total <= max_bytes {
12544                break;
12545            }
12546            let _ = self.conn.execute(
12547                "DELETE FROM _syncular_blobs WHERE blob_id = ?",
12548                rusqlite::params![blob_id],
12549            );
12550            total -= byte_length;
12551        }
12552    }
12553
12554    fn delete_unreferenced_cached_blobs(&mut self) -> Result<(), String> {
12555        #[cfg(feature = "bench-internals")]
12556        let _phase = self.benchmark_phases.start(Phase::BlobRetention);
12557        if !self.schema_has_blobs() {
12558            return Ok(());
12559        }
12560        self.rebuild_overlay_if_dirty()?;
12561        self.conn
12562            .execute(
12563                &format!(
12564                    "DELETE FROM _syncular_blobs
12565                 WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
12566                   AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
12567                   AND blob_id NOT IN ({})",
12568                    self.visible_blob_ids_sql(),
12569                ),
12570                [],
12571            )
12572            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12573        Ok(())
12574    }
12575
12576    // -- local row storage ----------------------------------------------------------
12577
12578    /// The cached per-table primary-key upsert SQL for `full_table` (see
12579    /// the `insert_sql` field: built once, reused per row).
12580    fn insert_row_sql(&self, full_table: &str, table: &crate::schema::TableSchema) -> String {
12581        if let Some(sql) = self.insert_sql.borrow().get(full_table) {
12582            return sql.clone();
12583        }
12584        let mut columns: Vec<String> = table.columns.iter().map(|c| quote_ident(&c.name)).collect();
12585        columns.push(quote_ident("_syncular_version"));
12586        columns.extend(table.sealed_columns().iter().map(|name| quote_ident(name)));
12587        let placeholders: Vec<&str> = columns.iter().map(|_| "?").collect();
12588        let primary_key = quote_ident(&table.primary_key);
12589        let updates = columns
12590            .iter()
12591            .filter(|column| **column != primary_key)
12592            .map(|column| format!("{column}=excluded.{column}"))
12593            .collect::<Vec<_>>()
12594            .join(", ");
12595        // §4.5 version guard: server state lands in the base table only above
12596        // the stored version; the visible overlay takes every write.
12597        let guard = if full_table == base_table(&table.name) {
12598            format!(" WHERE excluded._syncular_version > {full_table}._syncular_version")
12599        } else {
12600            String::new()
12601        };
12602        let sql = format!(
12603            "INSERT INTO {full_table} ({}) VALUES ({}) ON CONFLICT ({primary_key}) DO UPDATE SET {updates}{guard}",
12604            columns.join(", "),
12605            placeholders.join(", ")
12606        );
12607        self.insert_sql
12608            .borrow_mut()
12609            .insert(full_table.to_owned(), sql.clone());
12610        sql
12611    }
12612
12613    /// §4.5: write a server row under the version guard; returns whether it
12614    /// landed. `sealed` is its §5.11 sealed state, `None` for an unsealed row.
12615    fn write_base_row(
12616        &self,
12617        table_name: &str,
12618        row: &Row,
12619        version: i64,
12620        sealed: Option<&SealedRow>,
12621    ) -> Result<bool, String> {
12622        let landed = self.write_row(&base_table(table_name), table_name, row, version, sealed)?;
12623        if landed {
12624            self.overlay_dirty.table(table_name);
12625        }
12626        Ok(landed)
12627    }
12628
12629    fn write_row(
12630        &self,
12631        full_table: &str,
12632        table_name: &str,
12633        row: &Row,
12634        version: i64,
12635        sealed: Option<&SealedRow>,
12636    ) -> Result<bool, String> {
12637        #[cfg(feature = "bench-internals")]
12638        let _phase = self.benchmark_phases.start(Phase::RowWrite);
12639        let table = self
12640            .schema
12641            .table(table_name)
12642            .ok_or_else(|| format!("unknown table {table_name:?}"))?;
12643        let sql = self.insert_row_sql(full_table, table);
12644        let mut stmt = self
12645            .conn
12646            .prepare_cached(&sql)
12647            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12648        let params = row
12649            .iter()
12650            .map(RowParam::Cell)
12651            .chain(std::iter::once(RowParam::Version(version)))
12652            .chain(
12653                [
12654                    sealed.map(|sealed| sealed.key_id.as_str()),
12655                    sealed.map(|sealed| sealed.envelopes.as_str()),
12656                ]
12657                .into_iter()
12658                .take(table.sealed_columns().len())
12659                .map(RowParam::Text),
12660            );
12661        stmt.execute(rusqlite::params_from_iter(params))
12662            .map(|changed| changed > 0)
12663            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
12664    }
12665
12666    /// §4.5: delete a server row whose stored version lies below `version`;
12667    /// returns whether a row left.
12668    fn delete_base_row(
12669        &self,
12670        table_name: &str,
12671        row_id: &str,
12672        version: i64,
12673    ) -> Result<bool, String> {
12674        let table = self
12675            .schema
12676            .table(table_name)
12677            .ok_or_else(|| format!("unknown table {table_name:?}"))?;
12678        let sql = format!(
12679            "DELETE FROM {} WHERE {} AND _syncular_version < ?2",
12680            base_table(table_name),
12681            row_id_predicate(table)
12682        );
12683        let mut stmt = self
12684            .conn
12685            .prepare_cached(&sql)
12686            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12687        let landed = stmt
12688            .execute(rusqlite::params![row_id, version])
12689            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
12690            > 0;
12691        if landed {
12692            self.overlay_dirty.table(table_name);
12693        }
12694        Ok(landed)
12695    }
12696
12697    fn has_acknowledged_rows(&self) -> Result<bool, String> {
12698        self.conn
12699            .query_row(
12700                "SELECT EXISTS(SELECT 1 FROM _syncular_acked_rows)",
12701                [],
12702                |row| row.get(0),
12703            )
12704            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
12705    }
12706
12707    fn acknowledged_operations(
12708        &self,
12709        rows: Option<&HashSet<(&str, &str)>>,
12710        tables: Option<&BTreeSet<&str>>,
12711    ) -> Result<Vec<OutboxOp>, String> {
12712        let keys = rows.map(|rows| serde_json::to_string(&rows).expect("row keys"));
12713        let mut statement = self
12714            .conn
12715            .prepare_cached(if keys.is_some() {
12716                ACKED_ROWS_BY_ROW
12717            } else {
12718                "SELECT op_json,intent_json FROM _syncular_acked_rows WHERE ?1 IS NULL AND (?2 IS NULL OR tbl IN(SELECT value FROM json_each(?2))) ORDER BY commit_seq,idx"
12719            })
12720            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12721        let rows = statement
12722            .query_map(
12723                rusqlite::params![
12724                    keys,
12725                    tables.map(|tables| serde_json::to_string(tables).expect("table keys"))
12726                ],
12727                |row| {
12728                    #[cfg(test)]
12729                    self.acknowledged_replay_count
12730                        .set(self.acknowledged_replay_count.get() + 1);
12731                    Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
12732                },
12733            )
12734            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12735        rows.map(|row| {
12736            let (op, intent) =
12737                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12738            let op: CommitOperation =
12739                serde_json::from_str(&op).map_err(|_| "sync.local_corrupt".to_owned())?;
12740            let table = self
12741                .schema
12742                .table(&op.table)
12743                .ok_or_else(|| "sync.unknown_table".to_owned())?;
12744            let values = if op.op == "upsert" && self.visible_row(table, &op.row_id)?.is_none() {
12745                Some(serde_json::from_str(&intent).map_err(|_| "sync.local_corrupt".to_owned())?)
12746            } else {
12747                op.values.clone()
12748            };
12749            Ok(OutboxOp {
12750                values,
12751                ..op.into()
12752            })
12753        })
12754        .collect()
12755    }
12756
12757    fn drop_acknowledged_rows(
12758        &self,
12759        matches: impl Fn(&str, &Map<String, Value>) -> bool,
12760    ) -> Result<Vec<OutboxCommit>, String> {
12761        let mut statement = self
12762            .conn
12763            .prepare("SELECT commit_id,tbl,intent_json,id FROM _syncular_acked_rows")
12764            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12765        let rows = statement
12766            .query_map([], |row| {
12767                Ok((
12768                    row.get::<_, String>(0)?,
12769                    row.get::<_, String>(1)?,
12770                    row.get::<_, String>(2)?,
12771                    row.get::<_, String>(3)?,
12772                ))
12773            })
12774            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12775        let mut doomed = HashSet::new();
12776        for row in rows {
12777            let (id, table, intent, row_id) =
12778                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12779            let values =
12780                serde_json::from_str(&intent).map_err(|_| "sync.local_corrupt".to_owned())?;
12781            let base_matches = self
12782                .schema
12783                .table(&table)
12784                .map(|schema| self.stored_row(schema, &row_id, true))
12785                .transpose()?
12786                .flatten()
12787                .is_some_and(|(base, _)| {
12788                    let schema = self.schema.table(&table).expect("acknowledged table");
12789                    let values = schema
12790                        .columns
12791                        .iter()
12792                        .zip(base.iter())
12793                        .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
12794                        .collect();
12795                    matches(&table, &values)
12796                });
12797            if matches(&table, &values) || base_matches {
12798                doomed.insert(id);
12799            }
12800        }
12801        let mut dropped = Vec::new();
12802        for id in doomed {
12803            let mut statement = self
12804                .conn
12805                .prepare("SELECT op_json FROM _syncular_acked_rows WHERE commit_id=?1 ORDER BY idx")
12806                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12807            let operations = statement
12808                .query_map([&id], |row| row.get::<_, String>(0))
12809                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12810            let mut ops = Vec::new();
12811            for operation in operations {
12812                let json = operation
12813                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12814                let op: CommitOperation =
12815                    serde_json::from_str(&json).map_err(|_| "sync.local_corrupt".to_owned())?;
12816                self.overlay_dirty.table(&op.table);
12817                ops.push(OutboxOp {
12818                    table: op.table,
12819                    row_id: op.row_id,
12820                    upsert: op.op == "upsert",
12821                    base_version: op.base_version,
12822                    values: op.values,
12823                    insert: op.insert,
12824                });
12825            }
12826            dropped.push(OutboxCommit {
12827                client_commit_id: id.clone(),
12828                ops,
12829            });
12830            self.conn
12831                .execute("DELETE FROM _syncular_acked_rows WHERE commit_id=?1", [id])
12832                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12833        }
12834        Ok(dropped)
12835    }
12836
12837    /// [`Self::rebuild_overlay`], skipped when neither the base tables nor
12838    /// the outbox changed since the last rebuild (the no-op sync round).
12839    fn rebuild_overlay_if_dirty(&mut self) -> Result<(), String> {
12840        if self.overlay_dirty.get() {
12841            self.rebuild_overlay()?;
12842        }
12843        Ok(())
12844    }
12845
12846    /// §7.1: local reads see outbox state applied optimistically — rebuild
12847    /// every visible table as (base server state) + (pending outbox replay
12848    /// on top). Optimistic rows carry version `-1`.
12849    fn rebuild_overlay(&mut self) -> Result<(), String> {
12850        #[cfg(feature = "bench-internals")]
12851        let _phase = self.benchmark_phases.start(Phase::OverlayRebuild);
12852        #[cfg(test)]
12853        self.overlay_rebuild_count
12854            .set(self.overlay_rebuild_count.get() + 1);
12855        let dirty = self.overlay_dirty.snapshot();
12856        let tables: Vec<_> = self
12857            .schema
12858            .tables
12859            .iter()
12860            .filter(|table| {
12861                dirty
12862                    .as_ref()
12863                    .is_none_or(|names| names.is_empty() || names.contains(&table.name))
12864            })
12865            .cloned()
12866            .collect();
12867        let names: BTreeSet<_> = tables.iter().map(|table| table.name.as_str()).collect();
12868        self.begin_observation("syncular_overlay")?;
12869        let rebuilt = (|| {
12870            for table in &tables {
12871                for index in &table.fts_indexes {
12872                    self.drop_fts_triggers(index)?;
12873                }
12874                let visible = visible_table(&table.name);
12875                let base = base_table(&table.name);
12876                self.conn
12877                    .execute_batch(&format!(
12878                        "DELETE FROM {visible}; INSERT INTO {visible} SELECT * FROM {base}"
12879                    ))
12880                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12881            }
12882            self.replay_overlay(&names, None)?;
12883            for table in &tables {
12884                for index in &table.fts_indexes {
12885                    self.rebuild_fts_projection(table, index)?;
12886                    self.create_fts_triggers(table, index)?;
12887                }
12888            }
12889            self.conn
12890                .execute_batch("RELEASE syncular_overlay")
12891                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12892            Ok(())
12893        })();
12894        if rebuilt.is_err() {
12895            self.conn
12896                .execute_batch("ROLLBACK TO syncular_overlay; RELEASE syncular_overlay")
12897                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12898        } else {
12899            self.overlay_dirty.set(false);
12900        }
12901        rebuilt
12902    }
12903
12904    /// §7.1 replay of acknowledged, retained failed and pending intent, in
12905    /// that order, for the operations of `tables` on `rows` (every row for
12906    /// `None`) over their current visible state.
12907    fn replay_overlay(
12908        &self,
12909        tables: &BTreeSet<&str>,
12910        rows: Option<&HashSet<(&str, &str)>>,
12911    ) -> Result<(), String> {
12912        let replays = |op: &&OutboxOp| {
12913            tables.contains(op.table.as_str())
12914                && rows.is_none_or(|rows| rows.contains(&(op.table.as_str(), op.row_id.as_str())))
12915        };
12916        self.apply_outbox_ops(&self.acknowledged_operations(rows, Some(tables))?)?;
12917        for (commit, initial) in self.failed_commits.iter() {
12918            for operation in commit.ops.iter().filter(replays) {
12919                let Some(table) = self.schema.table(&operation.table) else {
12920                    continue;
12921                };
12922                if operation.upsert
12923                    && self.visible_row(table, &operation.row_id)?.is_none()
12924                    && operation.presented_values(table).is_some_and(|values| {
12925                        table
12926                            .columns
12927                            .iter()
12928                            .all(|column| values.contains_key(&column.name))
12929                    })
12930                {
12931                    let key = serde_json::to_string(&(&operation.table, &operation.row_id))
12932                        .expect("row key serialization");
12933                    if let Some(Value::Object(values)) = initial.get(&key) {
12934                        self.apply_outbox_ops(&[OutboxOp {
12935                            values: Some(values.clone()),
12936                            ..operation.clone()
12937                        }])?;
12938                    }
12939                }
12940            }
12941            self.apply_outbox_ops(commit.ops.iter().filter(replays))?;
12942        }
12943        self.apply_outbox_ops(
12944            self.outbox
12945                .iter()
12946                .flat_map(|commit| &commit.ops)
12947                .filter(replays),
12948        )
12949    }
12950
12951    /// §7.1 replay: every operation is applied over the current visible
12952    /// state. An operation that no longer applies (a server row now holds its
12953    /// unique value) stays pending and invisible until a later replay admits
12954    /// it or the server answers its push.
12955    fn apply_outbox_ops<'a>(
12956        &self,
12957        ops: impl IntoIterator<Item = &'a OutboxOp>,
12958    ) -> Result<(), String> {
12959        #[cfg(feature = "bench-internals")]
12960        let mut phase = self.benchmark_phases.start(Phase::PendingReplay);
12961        for op in ops {
12962            #[cfg(feature = "bench-internals")]
12963            if let Some(phase) = &mut phase {
12964                phase.unit();
12965            }
12966            if op.upsert {
12967                if let Some((table, values)) = self
12968                    .schema
12969                    .table(&op.table)
12970                    .filter(|table| table.indexes.iter().any(|index| index.unique))
12971                    .and_then(|table| Some((table, op.presented_values(table)?)))
12972                {
12973                    let mut intended: Map<String, Value> = self
12974                        .visible_row(table, &op.row_id)?
12975                        .map(|(row, _)| {
12976                            table
12977                                .columns
12978                                .iter()
12979                                .zip(&row)
12980                                .map(|(column, value)| {
12981                                    (column.name.clone(), column_value_to_json(value))
12982                                })
12983                                .collect()
12984                        })
12985                        .unwrap_or_default();
12986                    intended.extend(values.into_owned());
12987                    if !self.unique_conflicts(table, &intended, false)?.is_empty() {
12988                        continue;
12989                    }
12990                }
12991            }
12992            self.apply_outbox_op(op)?;
12993        }
12994        Ok(())
12995    }
12996
12997    fn apply_outbox_op(&self, op: &OutboxOp) -> Result<(), String> {
12998        let Some(table) = self.schema.table(&op.table) else {
12999            return Ok(());
13000        };
13001        let visible = visible_table(&table.name);
13002        if !op.upsert {
13003            let sql = format!("DELETE FROM {visible} WHERE {}", row_id_predicate(table));
13004            self.conn
13005                .prepare_cached(&sql)
13006                .and_then(|mut statement| statement.execute(rusqlite::params![op.row_id]))
13007                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13008            return Ok(());
13009        }
13010        // §7.1: an insert-intent operation that lacks a non-nullable column
13011        // cannot encode; it stays invisible until the send-time prepass drops
13012        // it as sync.outbox_incompatible (§7.4.4).
13013        let Some(values) = op.presented_values(table) else {
13014            return Ok(());
13015        };
13016        // §7.1: the overlay applies the operation's PRESENT columns
13017        // over the current local row. A partial operation over an
13018        // absent local row leaves it absent — the server answers it
13019        // with sync.row_deleted or sync.row_missing (§6.2). An
13020        // insert-intent operation presents every column.
13021        let full = table
13022            .columns
13023            .iter()
13024            .all(|column| values.contains_key(&column.name));
13025        let local = self.visible_row(table, &op.row_id)?;
13026        if local.is_none() && !full {
13027            return Ok(());
13028        }
13029        let mut row: Row = Vec::with_capacity(table.columns.len());
13030        for (index, column) in table.columns.iter().enumerate() {
13031            let incoming = match values.get(&column.name) {
13032                Some(raw) => json_to_column_value(
13033                    if raw.get("$bytes").is_some()
13034                        && table
13035                            .encrypted_columns
13036                            .iter()
13037                            .any(|encrypted| encrypted.index == index)
13038                    {
13039                        &table.wire_columns[index]
13040                    } else {
13041                        column
13042                    },
13043                    Some(raw),
13044                ),
13045                None => Ok(local
13046                    .as_ref()
13047                    .and_then(|(values, _)| values.get(index).cloned())
13048                    .flatten()),
13049            };
13050            row.push(incoming.map_err(|_| {
13051                "sync.local_corrupt: persisted optimistic row value is invalid".to_owned()
13052            })?);
13053        }
13054        let version = local.as_ref().map_or(-1, |(_, version)| *version);
13055        self.write_row(&visible, &table.name, &row, version, None)
13056            .map(|_| ())
13057            .map_err(|error| {
13058                if error.starts_with("UNIQUE constraint failed:") {
13059                    "sync.constraint_violation: local write violates a unique constraint".to_owned()
13060                } else {
13061                    error
13062                }
13063            })
13064    }
13065
13066    /// Whether the row is present in the overlay. Existence reads only the
13067    /// key: locked encrypted values stay opaque, and decoding the complete row
13068    /// would reject a valid plain write.
13069    fn visible_row_exists(&self, table: &TableSchema, row_id: &str) -> Result<bool, String> {
13070        self.conn
13071            .query_row(
13072                &format!(
13073                    "SELECT EXISTS(SELECT 1 FROM {} WHERE {})",
13074                    visible_table(&table.name),
13075                    row_id_predicate(table)
13076                ),
13077                rusqlite::params![row_id],
13078                |row| row.get(0),
13079            )
13080            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
13081    }
13082
13083    /// §5.11: a sealed row is read-only; a write could otherwise send NULL
13084    /// over ciphertext this device cannot see.
13085    fn refuse_sealed_row(&self, table: &TableSchema, row_id: &str) -> Result<(), ClientError> {
13086        if !table.has_encrypted_columns() {
13087            return Ok(());
13088        }
13089        let key_id: Option<String> = self
13090            .conn
13091            .query_row(
13092                &format!(
13093                    "SELECT \"_sync_sealed_key\" FROM {} WHERE {}",
13094                    visible_table(&table.name),
13095                    row_id_predicate(table)
13096                ),
13097                rusqlite::params![row_id],
13098                |row| row.get(0),
13099            )
13100            .optional()
13101            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
13102            .flatten();
13103        match key_id {
13104            None => Ok(()),
13105            Some(key_id) => Err(ClientError {
13106                code: "client.encryption_key_missing".to_owned(),
13107                message: "the row is sealed under a key this device does not hold".to_owned(),
13108                details: Some(json!({ "table": table.name, "keyId": key_id })),
13109                retryable: false,
13110            }),
13111        }
13112    }
13113
13114    /// §5.11 reveal: decrypt every sealed base row whose envelope keys the
13115    /// keyring now holds into its columns and clear both reserved columns,
13116    /// one change batch per table. `client.decrypt_failed` fails the caller.
13117    fn reveal_sealed_rows(&mut self) -> Result<(), TransportError> {
13118        let storage = |message: String| TransportError::new("storage.failed", message);
13119        let tables: Vec<TableSchema> = self
13120            .schema
13121            .tables
13122            .iter()
13123            .filter(|table| table.has_encrypted_columns())
13124            .cloned()
13125            .collect();
13126        for table in &tables {
13127            let base = base_table(&table.name);
13128            let pk = quote_ident(&table.primary_key);
13129            let sealed: Vec<(SqlValue, i64, String)> = self
13130                .conn
13131                .prepare_cached(&format!(
13132                    "SELECT {pk}, \"_syncular_version\", \"_sync_sealed\" FROM {base} WHERE \"_sync_sealed_key\" IS NOT NULL"
13133                ))
13134                .and_then(|mut statement| {
13135                    statement
13136                        .query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))?
13137                        .collect()
13138                })
13139                .map_err(|error| storage(Self::sqlite_failure(&self.storage_failure, error)))?;
13140            let mut revealed = Vec::new();
13141            for (key, version, envelopes) in sealed {
13142                if let Some(row) =
13143                    crate::values::reveal_sealed_row(table, &envelopes, &self.encryption)?
13144                {
13145                    revealed.push((key, version, row));
13146                }
13147            }
13148            if revealed.is_empty() {
13149                continue;
13150            }
13151            self.begin_observation("syncular_reveal").map_err(storage)?;
13152            let outcome = (|| {
13153                let assignments = table
13154                    .encrypted_columns
13155                    .iter()
13156                    .map(|enc| format!("{}=?", quote_ident(&table.columns[enc.index].name)))
13157                    .collect::<Vec<_>>()
13158                    .join(",");
13159                let mut statement = self
13160                    .conn
13161                    .prepare_cached(&format!(
13162                        "UPDATE {base} SET {assignments},\"_sync_sealed_key\"=NULL,\"_sync_sealed\"=NULL WHERE {pk}=? AND \"_syncular_version\"=?"
13163                    ))
13164                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13165                for (key, version, row) in &revealed {
13166                    let mut params = table
13167                        .encrypted_columns
13168                        .iter()
13169                        .map(|enc| owned_sql_value(RowParam::Cell(&row[enc.index])))
13170                        .collect::<Result<Vec<_>, _>>()?;
13171                    params.push(key.clone());
13172                    params.push(SqlValue::Integer(*version));
13173                    statement
13174                        .execute(rusqlite::params_from_iter(params))
13175                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13176                }
13177                drop(statement);
13178                self.overlay_dirty.table(&table.name);
13179                self.rebuild_overlay()?;
13180                let mut batch = ChangeAccumulator::default();
13181                batch.table(&table.name);
13182                self.finish_observation("syncular_reveal", batch)
13183            })();
13184            if outcome.is_err() {
13185                self.rollback_observation("syncular_reveal");
13186            }
13187            outcome.map_err(storage)?;
13188        }
13189        Ok(())
13190    }
13191
13192    /// The visible (optimistic) row's values plus version, or `None` when the
13193    /// row is absent from the overlay.
13194    fn visible_row(&self, table: &TableSchema, row_id: &str) -> Result<Option<(Row, i64)>, String> {
13195        self.stored_row(table, row_id, false)
13196    }
13197
13198    fn stored_row(
13199        &self,
13200        table: &TableSchema,
13201        row_id: &str,
13202        base: bool,
13203    ) -> Result<Option<(Row, i64)>, String> {
13204        let sql = format!(
13205            "SELECT * FROM {} WHERE {}",
13206            if base {
13207                base_table(&table.name)
13208            } else {
13209                visible_table(&table.name)
13210            },
13211            row_id_predicate(table)
13212        );
13213        let mut stmt = self
13214            .conn
13215            .prepare_cached(&sql)
13216            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13217        let mut rows = stmt
13218            .query(rusqlite::params![row_id])
13219            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13220        let Some(row) = rows
13221            .next()
13222            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
13223        else {
13224            return Ok(None);
13225        };
13226        let mut values = Vec::with_capacity(table.columns.len());
13227        for (index, column) in table.columns.iter().enumerate() {
13228            let raw = row
13229                .get_ref(index)
13230                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13231            if let rusqlite::types::ValueRef::Blob(bytes) = raw {
13232                if table
13233                    .encrypted_columns
13234                    .iter()
13235                    .any(|encrypted| encrypted.index == index)
13236                {
13237                    values.push(Some(ColumnValue::Bytes(bytes.to_vec())));
13238                    continue;
13239                }
13240            }
13241            let json = sql_ref_to_json(column, raw);
13242            values.push(json_to_column_value(column, Some(&json)).map_err(|_| {
13243                "sync.local_corrupt: persisted local row value is invalid".to_owned()
13244            })?);
13245        }
13246        let version: i64 = row
13247            .get(table.columns.len())
13248            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
13249        Ok(Some((values, version)))
13250    }
13251
13252    // -- realtime (§8) ---------------------------------------------------------------
13253
13254    /// §8.8: the configured realtime policy.
13255    #[must_use]
13256    pub fn realtime_policy(&self) -> RealtimePolicy {
13257        self.realtime_policy
13258    }
13259
13260    /// §8.8: the explicit realtime availability state.
13261    #[must_use]
13262    pub fn realtime_state(&self) -> RealtimeState {
13263        self.realtime_state
13264    }
13265
13266    /// §8.8: set the realtime policy. Refused while the socket is connected:
13267    /// a policy change never silently re-routes an ownership state.
13268    pub fn set_realtime_policy(&mut self, policy: RealtimePolicy) -> Result<(), String> {
13269        if self.realtime_state == RealtimeState::Connected && policy != self.realtime_policy {
13270            return Err(
13271                "sync.invalid_request: disconnect realtime before changing the realtime policy"
13272                    .to_owned(),
13273            );
13274        }
13275        self.realtime_policy = policy;
13276        match policy {
13277            RealtimePolicy::Off => self.set_realtime_state(RealtimeState::Disabled, None),
13278            RealtimePolicy::Required | RealtimePolicy::Optional
13279                if self.realtime_state == RealtimeState::Disabled =>
13280            {
13281                self.set_realtime_state(RealtimeState::Disconnected, None);
13282            }
13283            RealtimePolicy::Required | RealtimePolicy::Optional => {}
13284        }
13285        Ok(())
13286    }
13287
13288    /// Record an availability transition, keeping the reason code bounded and
13289    /// code-like (§7.6) and clearing it where it cannot apply.
13290    fn set_realtime_state(&mut self, state: RealtimeState, reason_code: Option<&str>) {
13291        self.realtime_state = state;
13292        self.realtime_reason_code = reason_code.map(Self::diagnostic_code);
13293        if matches!(state, RealtimeState::Connected | RealtimeState::Disabled) {
13294            self.realtime_reason_code = None;
13295            self.realtime_retry_delay_ms = None;
13296        }
13297    }
13298
13299    #[must_use]
13300    fn realtime_connected(&self) -> bool {
13301        self.realtime_state == RealtimeState::Connected
13302    }
13303
13304    pub fn connect_realtime(&mut self, transport: &mut dyn Transport) -> Result<(), String> {
13305        if !self.transport_enabled {
13306            return Err("sync.offline: transport gate is closed".into());
13307        }
13308        if self.realtime_policy == RealtimePolicy::Off {
13309            return Err(
13310                "sync.invalid_request: realtime is off; realtime connect is refused".to_owned(),
13311            );
13312        }
13313        if self.realtime_connected() {
13314            return Ok(());
13315        }
13316        self.realtime_state = RealtimeState::Connecting;
13317        match transport.realtime_connect(&self.client_id) {
13318            Ok(()) => {
13319                self.set_realtime_state(RealtimeState::Connected, None);
13320                Ok(())
13321            }
13322            Err(error) => {
13323                self.set_realtime_state(RealtimeState::Refused, Some(&error.code));
13324                Err(format!("{}: {}", error.code, error.message))
13325            }
13326        }
13327    }
13328
13329    pub fn disconnect_realtime(&mut self, transport: &mut dyn Transport) {
13330        self.cancel_sync_round();
13331        // A deliberately disconnected or policy-disabled client has nothing
13332        // to release; a `lost` socket still does (the host owns the handle).
13333        if self.realtime_state == RealtimeState::Disconnected
13334            || self.realtime_state == RealtimeState::Disabled
13335        {
13336            return;
13337        }
13338        let _ = transport.realtime_close();
13339        self.set_realtime_state(
13340            if self.realtime_policy == RealtimePolicy::Off {
13341                RealtimeState::Disabled
13342            } else {
13343                RealtimeState::Disconnected
13344            },
13345            None,
13346        );
13347        self.presence.clear(); // §8.6.1: presence is per-connection
13348    }
13349
13350    /// §8.8: the host observed the socket end without disconnecting it (a
13351    /// peer close, an I/O error, a failed send; the native transport reports
13352    /// these as `Inbound::Disconnected`). Releases the transport, moves a
13353    /// `Connected` state to `Lost` with the transport's code as the reason, and schedules
13354    /// one background retry intent so the host reconnects and catches up.
13355    pub fn realtime_lost(&mut self, transport: &mut dyn Transport, error: &TransportError) {
13356        if self.realtime_state != RealtimeState::Connected {
13357            return;
13358        }
13359        self.cancel_sync_round();
13360        let _ = transport.realtime_close();
13361        self.set_realtime_state(RealtimeState::Lost, Some(&error.code));
13362        self.realtime_retry_delay_ms = Some(self.schedule_background_retry());
13363        self.presence.clear();
13364    }
13365
13366    /// §8.6.2: publish (or clear, `doc: None`) this client's presence
13367    /// document for `scope_key`. Requires a live socket; the document is
13368    /// ephemeral (lost on disconnect). Authorization is the connection's
13369    /// registration (§8.6.3) — an unheld key is rejected loudly by the
13370    /// server with `presence.forbidden`.
13371    pub fn set_presence(
13372        &mut self,
13373        transport: &mut dyn Transport,
13374        scope_key: &str,
13375        doc: Option<&Value>,
13376    ) -> Result<(), String> {
13377        if !self.transport_enabled {
13378            return Err("sync.offline: transport gate is closed".into());
13379        }
13380        if !self.realtime_connected() {
13381            return Err("setPresence requires a connected realtime socket (§8.6)".to_string());
13382        }
13383        let text = encode_presence_publish(scope_key, doc);
13384        transport
13385            .realtime_send(&text)
13386            .map_err(|e| format!("{}: {}", e.code, e.message))
13387    }
13388
13389    /// §8.6: the peers currently present on a scope key (ephemeral).
13390    pub fn presence(&self, scope_key: &str) -> Vec<PresencePeer> {
13391        self.presence
13392            .get(scope_key)
13393            .map(|peers| peers.values().cloned().collect())
13394            .unwrap_or_default()
13395    }
13396
13397    /// §8.6 apply an inbound presence fanout to the local map.
13398    fn apply_presence(
13399        &mut self,
13400        scope_key: String,
13401        kind: Option<PresenceKind>,
13402        actor_id: Option<String>,
13403        client_id: Option<String>,
13404        doc: Option<Value>,
13405        error: Option<String>,
13406    ) {
13407        // A refused publish (§8.6.2) changes no peer; the native host surfaces
13408        // it as a `presence` event carrying the server's `error` code.
13409        if error.is_some() {
13410            return;
13411        }
13412        let (Some(kind), Some(actor_id), Some(client_id)) = (kind, actor_id, client_id) else {
13413            return;
13414        };
13415        let peer_key = format!("{actor_id} {client_id}");
13416        match kind {
13417            PresenceKind::Leave => {
13418                if let Some(peers) = self.presence.get_mut(&scope_key) {
13419                    peers.remove(&peer_key);
13420                    if peers.is_empty() {
13421                        self.presence.remove(&scope_key);
13422                    }
13423                }
13424            }
13425            _ => {
13426                let doc = match doc {
13427                    Some(Value::Object(_)) => doc.unwrap(),
13428                    _ => return,
13429                };
13430                self.presence.entry(scope_key).or_default().insert(
13431                    peer_key,
13432                    PresencePeer {
13433                        actor_id,
13434                        client_id,
13435                        doc,
13436                    },
13437                );
13438            }
13439        }
13440    }
13441
13442    /// Inbound JSON control message (§8.1). Unknown events are tolerated.
13443    pub fn on_realtime_text(&mut self, text: &str) {
13444        if self.stopped || !self.transport_enabled || self.security_preflight {
13445            return;
13446        }
13447        match parse_control(text) {
13448            Ok(ControlMessage::Hello { requires_sync, .. }) => {
13449                if requires_sync {
13450                    // §8.1: pull before trusting the socket for continuity.
13451                    self.set_sync_needed(true, true);
13452                }
13453            }
13454            Ok(ControlMessage::Presence {
13455                scope_key,
13456                kind,
13457                actor_id,
13458                client_id,
13459                doc,
13460                error,
13461                ..
13462            }) => {
13463                self.apply_presence(scope_key, kind, actor_id, client_id, doc, error);
13464            }
13465            Ok(ControlMessage::Wake { .. }) => {
13466                // §8.3: any wake-up means "run a pull soon", never data.
13467                self.set_sync_needed(true, true);
13468            }
13469            Ok(ControlMessage::Scopes { .. }) => {
13470                // §8.7: surfaced to the app; no sync-needed, no round.
13471                self.scopes_changed += 1;
13472            }
13473            _ => {}
13474        }
13475    }
13476
13477    /// Inbound binary delta: a complete SSP2 response (§8.2), applied like
13478    /// a pull response per section; an unapplied delta is a wake-up.
13479    /// Mailbox hosts send durable acknowledgements on their I/O executor.
13480    pub fn on_realtime_binary_queued(&mut self, bytes: &[u8]) -> Vec<String> {
13481        let mut transport = crate::round::DownloadResults::default();
13482        self.on_realtime_binary(&mut transport, bytes);
13483        transport.controls
13484    }
13485
13486    /// §8.2: a delta applies exactly like a live section and sets the replica
13487    /// cursor to its `throughSeq`; one at or below the replica cursor is a
13488    /// duplicate. A delta the client does not apply (no live unit, a status
13489    /// other than `active`, a decode or apply failure) is a drop and raises
13490    /// sync-needed.
13491    pub fn on_realtime_binary(&mut self, transport: &mut dyn Transport, bytes: &[u8]) {
13492        if self.stopped || !self.transport_enabled || self.security_preflight {
13493            return;
13494        }
13495        #[cfg(feature = "bench-internals")]
13496        let decode_phase = self.benchmark_phases.start(Phase::ResponseDecode);
13497        let message = match decode_message(bytes) {
13498            Ok(m) if m.msg_kind == MsgKind::Response => m,
13499            _ => {
13500                self.set_sync_needed(true, true);
13501                return;
13502            }
13503        };
13504        #[cfg(feature = "bench-internals")]
13505        drop(decode_phase);
13506        // §8.2: a delta's header means what a pull response's header means. A
13507        // schema floor (§1.6) is recorded, stops syncing and applies nothing.
13508        if let Some(Frame::RespHeader {
13509            required_schema_version: Some(required),
13510            latest_schema_version,
13511            ..
13512        }) = message.frames.first()
13513        {
13514            let floor = SchemaFloor {
13515                required_schema_version: Some(*required),
13516                latest_schema_version: *latest_schema_version,
13517            };
13518            if self.set_schema_floor(Some(floor)).is_err() {
13519                self.set_sync_needed(true, true);
13520            }
13521            return;
13522        }
13523        let through = message.frames.iter().find_map(|frame| match frame {
13524            Frame::LiveEnd { through_seq, .. } => Some(*through_seq),
13525            _ => None,
13526        });
13527        let epoch = self.get_meta(LOG_EPOCH_KEY).ok().flatten();
13528        let mut dropped = !matches!(message.frames.first(), Some(Frame::RespHeader {
13529                reset_required: false,
13530                log_epoch,
13531                ..
13532            }) if Some(log_epoch) == epoch.as_ref());
13533        let mut applied: Option<i64> = None;
13534        dropped |= self.begin_coalesced_apply().is_err();
13535        let mut receivers: Option<Vec<UnitRecord>> = None;
13536        for frame in message.frames {
13537            if dropped {
13538                break;
13539            }
13540            match frame {
13541                Frame::LiveStart { status, .. } => {
13542                    let Ok(cursor) = self.stored(replica_cursor(&self.conn)) else {
13543                        dropped = true;
13544                        break;
13545                    };
13546                    if status != LiveStatus::Active || cursor < 0 {
13547                        dropped = true;
13548                    } else if !through.is_some_and(|through| through <= cursor) {
13549                        // A catching-up unit receives no delta (§8.2).
13550                        receivers = self.stored(load_units(&self.conn)).ok().map(|units| {
13551                            units
13552                                .into_iter()
13553                                .filter(|unit| {
13554                                    unit.status == UnitStatus::Active
13555                                        && unit.echo.is_some()
13556                                        && unit.cursor.is_none()
13557                                })
13558                                .collect()
13559                        });
13560                        dropped = receivers.is_none();
13561                    }
13562                }
13563                Frame::Commit {
13564                    commit_seq,
13565                    tables,
13566                    changes,
13567                    ..
13568                } => {
13569                    let Some(receivers) = receivers.clone() else {
13570                        continue;
13571                    };
13572                    dropped = self
13573                        .apply_live_commit(
13574                            &receivers,
13575                            &tables,
13576                            changes,
13577                            commit_seq,
13578                            AppliedServerCommitSource::Delta,
13579                        )
13580                        .is_err();
13581                }
13582                Frame::LiveEnd { through_seq, .. } if receivers.is_some() => {
13583                    dropped = self
13584                        .set_meta(REPLICA_CURSOR_KEY, &through_seq.to_string())
13585                        .is_err();
13586                    applied = Some(through_seq);
13587                }
13588                Frame::Error { .. } => dropped = true,
13589                _ => {}
13590            }
13591        }
13592        // §1.4 rule 4: the ack follows the COMMIT. A failed COMMIT restored
13593        // the replica cursor, so the delta counts as a drop.
13594        if self.finish_coalesced_apply().is_err() || dropped {
13595            applied = None;
13596            dropped = true;
13597        }
13598        if let Some(cursor) = applied {
13599            // §8.2 ack point: the delta's throughSeq.
13600            let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
13601            let _ = transport.realtime_send(&ack);
13602        }
13603        if dropped {
13604            self.set_sync_needed(true, true);
13605        }
13606    }
13607
13608    /// §8.2 ack point after a pull round on a live connection: the replica
13609    /// cursor while it is at least `0`.
13610    fn ack_after_pull(&mut self, transport: &mut dyn Transport) {
13611        if !self.realtime_connected() {
13612            return;
13613        }
13614        if let Ok(cursor @ 0..) = self.stored(replica_cursor(&self.conn)) {
13615            let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
13616            let _ = transport.realtime_send(&ack);
13617        }
13618    }
13619}
13620
13621#[cfg(test)]
13622mod previous_version_wiring_tests;
13623
13624#[cfg(test)]
13625mod identity_tests;
13626
13627#[cfg(test)]
13628mod insert_intent_tests;
13629
13630#[cfg(test)]
13631mod robustness_tests;
13632
13633#[cfg(test)]
13634mod range_image_tests;