Skip to main content

syncular_client/
client.rs

1//! The Syncular v2 Rust client core (SPEC.md client-behavior contract):
2//! rusqlite local storage, §3.2/§3.3 effective-scope persistence + purge,
3//! §4 pull/cursor/bootstrap (§4.7 resume, §5.6 segment application), §6
4//! push with outbox order, §7 optimistic apply / rollback / replay-on-top,
5//! §2.3 clientCommitId idempotency, §8 realtime client rules, §10 errors.
6//!
7//! Built from `SPEC.md` and the committed `ssp2` codec alone — no
8//! reference to the v1 Rust tree or the v2 TypeScript client.
9
10use crate::{ProgressObserver, ProgressPhase, ProgressState};
11use ssp2::encode_message;
12use std::borrow::Cow;
13#[cfg(test)]
14use std::cell::Cell;
15use std::cell::RefCell;
16use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
17use std::sync::Arc;
18
19use rusqlite::types::{ToSqlOutput, Value as SqlValue, ValueRef};
20use rusqlite::{Connection, OpenFlags, OptionalExtension};
21use serde::{Deserialize, Serialize};
22use serde_json::{json, Map, Value};
23use sha2::{Digest, Sha256};
24use ssp2::decode::WIRE_VERSION;
25use ssp2::model::{
26    FencedRecord, Frame, LiveStatus, MediaType, Message, MsgKind, Op, OpResult, PushStatus,
27    SubStatus,
28};
29use ssp2::primitives::RawJson;
30use ssp2::registration::canonical_scope_json;
31use ssp2::segment::{decode_rows_segment, Column, ColumnType, ColumnValue, Row, RowsSegment};
32use ssp2::{decode_message, encode_presence_publish, parse_control, ControlMessage, PresenceKind};
33
34use crate::api::{
35    AppliedServerCommit, AppliedServerCommitSource, ClientChangeBatch, ClientDiagnosticsHost,
36    ClientDiagnosticsLease, ClientDiagnosticsReplica, ClientDiagnosticsSchema,
37    ClientDiagnosticsSnapshot, ClientDiagnosticsStorage, ClientError, ClientLimits, CommandEffects,
38    CommitDelivery, CommitOperation, CommitOperationOutcome, CommitOutcome, CommitOutcomeQuery,
39    CommitOutcomeResolution, CommitOutcomeStatus, ConflictRecord, CoverageSnapshot,
40    CurrentBaseProof, CurrentBaseProofReason, DiagnosticLastChange, DiagnosticLastRound,
41    DiagnosticQueryFailure, DiagnosticRoundCounters, DiagnosticUnits, EchoRecord, FetchedBlob,
42    LeaseState, LocalDataPurgeInput, LocalDataPurgeResult, LocalDataPurgeTarget,
43    LocalDataRebootstrapInput, LocalDataRebootstrapResult, Mutation, PresencePeer,
44    PreviousVersionStatus, QueryOwner, QueryReadFailure, QueryRow, QuerySnapshot, QueryValue,
45    RejectionDetails, RejectionRecord, ResolveCommitOutcomeInput, RetainedCommitRow,
46    RetainedUniqueConflict, RowState, SchemaFloor, SnapshotRead, SyncIntent, SyncOutcome,
47    SyncReport, SyncStatusSnapshot, TableChange, UnitCatchup, UnitRecord, UnitScopes, UnitStatus,
48    WindowBase, WindowChange, WindowCoverage, WindowState, WindowUnitRef,
49    CLIENT_DIAGNOSTICS_VERSION, MAX_DIAGNOSTIC_QUERY_FAILURES,
50};
51use crate::api::{RealtimePolicy, RealtimeState, REALTIME_UNAVAILABLE_CODE};
52#[cfg(feature = "bench-internals")]
53use crate::bench::{Phase, Recorder};
54use crate::previous_version::{
55    capture_previous_version_from_replica, first_incompatibility,
56    reconcile_previous_version_at_boot, set_local_schema_descriptor,
57    sweep_previous_version_container, PendingCommitAudit, PreviousVersionContextConfig,
58    PreviousVersionLifecycle, PreviousVersionReadSpec, PreviousVersionSnapshot,
59};
60use crate::schema::{parse_schema_json, ClientSchema, FtsIndexSchema, TableSchema};
61use crate::transport::{BlobDownload, SegmentRequest, Transport, TransportError};
62use crate::values::{
63    bytes_to_hex, column_value_to_json, decode_row_bytes, encode_sparse_row_json, full_row_values,
64    is_canonical_bytes_envelope, json_to_column_value, json_to_scope_map, normalize_values_casing,
65    render_row_id_json, scope_map_to_json, SealedRow,
66};
67
68/// §4.2 default: rows and SQLite images, both applied in committed chunks.
69/// Bit 3 (signed URLs, §5.4) is
70/// added per transport capability at request-build time.
71const DEFAULT_ACCEPT: u8 = 0b0111;
72const ACCEPT_INLINE_ROWS: u8 = 1 << 0;
73const ACCEPT_EXTERNAL_ROWS: u8 = 1 << 1;
74const ACCEPT_SQLITE: u8 = 1 << 2;
75const ACCEPT_SIGNED_URLS: u8 = 1 << 3;
76const MAX_DIAGNOSTIC_DOMAINS: usize = 256;
77
78/// §10.1 `recommendedAction` carried by a server HTTP error body.
79fn recommended_action_of(details: &Option<Value>) -> Option<String> {
80    details
81        .as_ref()?
82        .get("recommendedAction")?
83        .as_str()
84        .map(str::to_owned)
85}
86
87/// §7.4.1 persisted local schema-version marker (`_syncular_meta` key).
88pub(crate) const LOCAL_SCHEMA_VERSION_KEY: &str = "localSchemaVersion";
89const LOCAL_REVISION_KEY: &str = "localRevision";
90/// §4.5 replica cursor; absent means `-1`.
91const REPLICA_CURSOR_KEY: &str = "replicaCursor";
92/// Hex of the last pull `LIVE_END.echoDigest` (§4.2); absent means empty.
93const ECHO_DIGEST_KEY: &str = "echoDigest";
94/// Hex of the registration digest of the list the server stores (§4.1).
95const REGISTERED_DIGEST_KEY: &str = "registrationDigest";
96/// §7.1: the identity of the outbox sequence.
97const OUTBOX_ID_KEY: &str = "outboxId";
98const CLIENT_ID_KEY: &str = "clientId";
99const LEASE_STATE_KEY: &str = "leaseState";
100const SCHEMA_FLOOR_KEY: &str = "schemaFloor";
101pub(crate) const LOG_EPOCH_KEY: &str = "logEpoch";
102/// Persisted fail-closed quarantine marker: present while a security preflight
103/// is pending and `activateSecurity` has yet to run. Storing it in the replica
104/// keeps the gate with the data it protects, so a rebuilt host handle reopening
105/// the same file returns in preflight even though its in-memory state is fresh.
106const SECURITY_PREFLIGHT_PENDING_KEY: &str = "securityPreflightPending";
107const LOCAL_REBOOTSTRAP_RECEIPT_VERSION: u8 = 2;
108const MAX_JS_SAFE_INTEGER: u64 = 9_007_199_254_740_991;
109/// §7.4.4 client-local code: a pending outbox commit cannot re-encode under
110/// the new schema after a bump. Never a wire code (§10.3).
111const OUTBOX_INCOMPATIBLE_CODE: &str = "sync.outbox_incompatible";
112/// §5.11 client-local code: an encode at the push seam resolved no usable key
113/// id or named an unknown key. Never a wire code (§10.3).
114const ENCRYPT_FAILED_CODE: &str = "client.encrypt_failed";
115const INVALID_REQUEST_CODE: &str = "sync.invalid_request:";
116/// §7.4.4: static text for a persisted commit the current codec refuses.
117const LEGACY_VALUE_REFUSAL_MESSAGE: &str =
118    "the persisted commit carries values the current codec refuses";
119/// §7.4.4: bounded reason on every intrinsic value drop.
120const INVALID_STORED_VALUES_REASON: &str = "invalid_stored_values";
121
122#[derive(Debug, Serialize, Deserialize)]
123#[serde(rename_all = "camelCase", deny_unknown_fields)]
124struct PersistedLocalDataRebootstrapReceipt {
125    version: u8,
126    retained_commits: u64,
127    reset_subscriptions: u64,
128}
129
130fn invalid_local_rebootstrap_receipt() -> String {
131    "sync.local_corrupt: persisted local rebootstrap receipt is invalid".to_owned()
132}
133
134fn encode_local_rebootstrap_receipt(
135    retained_commits: usize,
136    reset_subscriptions: usize,
137) -> Result<String, String> {
138    let retained_commits =
139        u64::try_from(retained_commits).map_err(|_| invalid_local_rebootstrap_receipt())?;
140    let reset_subscriptions =
141        u64::try_from(reset_subscriptions).map_err(|_| invalid_local_rebootstrap_receipt())?;
142    if retained_commits > MAX_JS_SAFE_INTEGER || reset_subscriptions > MAX_JS_SAFE_INTEGER {
143        return Err(invalid_local_rebootstrap_receipt());
144    }
145    serde_json::to_string(&PersistedLocalDataRebootstrapReceipt {
146        version: LOCAL_REBOOTSTRAP_RECEIPT_VERSION,
147        retained_commits,
148        reset_subscriptions,
149    })
150    .map_err(|_| invalid_local_rebootstrap_receipt())
151}
152
153fn decode_local_rebootstrap_receipt(value: &str) -> Result<(usize, usize), String> {
154    // Pre-0.15.36 markers proved application but did not retain the receipt.
155    if value == "v1" {
156        return Ok((0, 0));
157    }
158    let receipt: PersistedLocalDataRebootstrapReceipt =
159        serde_json::from_str(value).map_err(|_| invalid_local_rebootstrap_receipt())?;
160    if receipt.version != LOCAL_REBOOTSTRAP_RECEIPT_VERSION
161        || receipt.retained_commits > MAX_JS_SAFE_INTEGER
162        || receipt.reset_subscriptions > MAX_JS_SAFE_INTEGER
163    {
164        return Err(invalid_local_rebootstrap_receipt());
165    }
166    Ok((
167        usize::try_from(receipt.retained_commits)
168            .map_err(|_| invalid_local_rebootstrap_receipt())?,
169        usize::try_from(receipt.reset_subscriptions)
170            .map_err(|_| invalid_local_rebootstrap_receipt())?,
171    ))
172}
173
174#[cfg(test)]
175mod observation_tests;
176
177/// Internal state for hosts that perform network I/O outside the client owner.
178#[doc(hidden)]
179pub struct CurrentBasePreparation {
180    client_commit_id: String,
181    sequence: i64,
182    units: Vec<UnitRecord>,
183    transport_epoch: u64,
184    registration_epoch: u64,
185    security_epoch: u64,
186}
187
188#[cfg(test)]
189mod units_tests;
190
191#[derive(Debug, Clone)]
192struct OutboxOp {
193    upsert: bool,
194    table: String,
195    row_id: String,
196    base_version: Option<i64>,
197    /// Schema-agnostic local form (§0): driver JSON values, encoded with
198    /// the current codec at send time. A key's presence IS the §6.1 sparse
199    /// presence set — both the overlay and the wire payload derive from it.
200    values: Option<Map<String, Value>>,
201    /// §7.1 insert intent: `mutate` recorded this upsert while the row was
202    /// absent from the visible replica. Replay and send-time encoding present
203    /// every current-schema column; a column `values` lacks is NULL.
204    insert: bool,
205}
206
207impl OutboxOp {
208    /// The values this upsert presents under the current `table` (§7.1). An
209    /// insert-intent operation presents every column, a missing one as NULL;
210    /// `None` when a missing column is non-nullable, which leaves the commit
211    /// unencodable (§7.4.4), or when the operation carries no values.
212    fn presented_values(&self, table: &TableSchema) -> Option<Cow<'_, Map<String, Value>>> {
213        let values = self.values.as_ref()?;
214        if !self.insert || table.columns.iter().all(|c| values.contains_key(&c.name)) {
215            return Some(Cow::Borrowed(values));
216        }
217        full_row_values(table, values.clone()).ok().map(Cow::Owned)
218    }
219}
220
221impl From<CommitOperation> for OutboxOp {
222    fn from(operation: CommitOperation) -> Self {
223        Self {
224            upsert: operation.op == "upsert",
225            table: operation.table,
226            row_id: operation.row_id,
227            base_version: operation.base_version,
228            values: operation.values,
229            insert: operation.insert,
230        }
231    }
232}
233
234/// §7.5 static authoring refusal; the table and column travel in details.
235fn invalid_authoring(message: &str, details: Value) -> ClientError {
236    ClientError {
237        code: "sync.invalid_request".to_owned(),
238        message: message.to_owned(),
239        details: Some(details),
240        retryable: false,
241    }
242}
243
244/// §7.1 chained base: `later` repeats `earlier`'s positive base on the same
245/// row and presents only columns `earlier` presents.
246fn chains_on(earlier: &OutboxOp, later: &OutboxOp) -> bool {
247    // Only an upsert carries values, and it always carries its primary key.
248    earlier.row_id == later.row_id
249        && earlier.table == later.table
250        && later.base_version.is_some_and(|base| base > 0)
251        && earlier.base_version == later.base_version
252        && later.values.as_ref().is_some_and(|values| {
253            values.keys().all(|column| {
254                earlier
255                    .values
256                    .as_ref()
257                    .is_some_and(|earlier| earlier.contains_key(column))
258            })
259        })
260}
261
262impl From<&OutboxOp> for CommitOperation {
263    fn from(operation: &OutboxOp) -> Self {
264        Self {
265            table: operation.table.clone(),
266            row_id: operation.row_id.clone(),
267            op: if operation.upsert { "upsert" } else { "delete" }.to_owned(),
268            base_version: operation.base_version,
269            values: operation.values.clone(),
270            insert: operation.insert,
271        }
272    }
273}
274
275#[derive(Debug, Clone)]
276struct OutboxCommit {
277    client_commit_id: String,
278    ops: Vec<OutboxOp>,
279}
280
281/// One commit's `(table, value keys, insert intent)` per upsert operation — the classification
282/// shape the §7.4.4 send-time drop and the RFC 0005 D6 audit share, so the two
283/// can never disagree about which commits cannot re-encode.
284fn commit_audit_operations(commit: &OutboxCommit) -> Vec<(&str, Vec<&str>, bool)> {
285    commit
286        .ops
287        .iter()
288        .filter(|operation| operation.upsert)
289        .map(|operation| {
290            (
291                operation.table.as_str(),
292                operation
293                    .values
294                    .as_ref()
295                    .map(|values| values.keys().map(String::as_str).collect())
296                    .unwrap_or_default(),
297                operation.insert,
298            )
299        })
300        .collect()
301}
302
303type FailedCommit = (OutboxCommit, Map<String, Value>);
304
305#[derive(Debug, Clone)]
306struct CompiledLocalDataPurgeTarget {
307    table: String,
308    selectors: Vec<(String, Vec<String>)>,
309}
310
311struct StoredCommitOutcomeRow {
312    sequence: i64,
313    client_commit_id: String,
314    status: String,
315    recorded_at_ms: i64,
316    results_json: String,
317    operations_json: Option<String>,
318    resolution: String,
319    resolved_at_ms: Option<i64>,
320    replacement_client_commit_id: Option<String>,
321}
322
323impl StoredCommitOutcomeRow {
324    fn from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result<Self> {
325        Ok(Self {
326            sequence: row.get(0)?,
327            client_commit_id: row.get(1)?,
328            status: row.get(2)?,
329            recorded_at_ms: row.get(3)?,
330            results_json: row.get(4)?,
331            operations_json: row.get(5)?,
332            resolution: row.get(6)?,
333            resolved_at_ms: row.get(7)?,
334            replacement_client_commit_id: row.get(8)?,
335        })
336    }
337}
338
339/// Section outcome distinguishing the §5.6 unit-local fail-closed
340/// path from a round-aborting failure (§1.4 rule 5).
341/// §4.4 a unit section's `SUB_START`.
342struct SectionStart {
343    id: String,
344    status: SubStatus,
345    reason_code: String,
346    bootstrap: bool,
347    fenced: Vec<FencedRecord>,
348}
349
350enum SectionError {
351    FailClosed,
352    /// §5.5: the server refused a segment with `sync.forbidden`.
353    Refused,
354    Abort(String, String),
355    Transport(TransportError),
356}
357
358#[derive(Debug)]
359pub(crate) struct RequestMeta {
360    pushed_ids: Vec<String>,
361    /// Hex of the registration digest this pull carried; the server stores
362    /// that list once the round completes (§4.1).
363    registration_digest: String,
364    pub(crate) accept: u8,
365    /// §6.1 splitBatch: outbox commits held back from THIS request because
366    /// the running operation count reached the push cap — the next round
367    /// pushes them (`sync_needed` stays set while any remain).
368    deferred_commits: usize,
369}
370
371#[derive(Default)]
372struct ChangeAccumulator {
373    /// None means table-wide; Some is the exact scoped domain.
374    tables: BTreeMap<String, Option<BTreeSet<String>>>,
375    windows: BTreeMap<String, BTreeSet<String>>,
376    status: bool,
377    conflicts: bool,
378    rejections: bool,
379    outcomes: bool,
380}
381
382impl ChangeAccumulator {
383    fn table(&mut self, table: &str) {
384        self.tables.insert(table.to_owned(), None);
385    }
386
387    fn scope(&mut self, table: &str, key: String) {
388        match self.tables.get_mut(table) {
389            Some(None) => {}
390            Some(Some(keys)) => {
391                keys.insert(key);
392            }
393            None => {
394                self.tables
395                    .insert(table.to_owned(), Some(BTreeSet::from([key])));
396            }
397        }
398    }
399
400    fn window(&mut self, base_key: &str, value: &str) {
401        self.windows
402            .entry(base_key.to_owned())
403            .or_default()
404            .insert(value.to_owned());
405    }
406
407    fn touched(&self) -> bool {
408        !self.tables.is_empty()
409            || !self.windows.is_empty()
410            || self.status
411            || self.conflicts
412            || self.rejections
413            || self.outcomes
414    }
415}
416
417/// §6.1: the server caps total operations per request (reference default
418/// 500) and rejects the whole batch with `sync.too_many_operations`; the
419/// client "splits and retries". Splitting happens at build time: commits are
420/// included IN ORDER until the operation budget is spent, the rest wait for
421/// the next round.
422const PUSH_OPS_PER_REQUEST: usize = 500;
423const MAX_LOCAL_PURGE_TARGETS: usize = 64;
424const MAX_LOCAL_PURGE_SELECTORS: usize = 8;
425const MAX_LOCAL_PURGE_VALUES: usize = 128;
426const MAX_LOCAL_PURGE_VALUE_LENGTH: usize = 256;
427pub const SECURITY_PREFLIGHT_REQUIRED_CODE: &str = "client.security_preflight_required";
428
429/// Dirty visible tables; `None` means a whole-replica recovery/reset.
430struct OverlayDirty(RefCell<Option<BTreeSet<String>>>);
431
432impl OverlayDirty {
433    fn clean() -> Self {
434        Self(RefCell::new(Some(BTreeSet::new())))
435    }
436    fn get(&self) -> bool {
437        self.0
438            .borrow()
439            .as_ref()
440            .is_none_or(|tables| !tables.is_empty())
441    }
442    fn set(&self, dirty: bool) {
443        *self.0.borrow_mut() = if dirty { None } else { Some(BTreeSet::new()) };
444    }
445    fn table(&self, table: &str) {
446        if let Some(tables) = self.0.borrow_mut().as_mut() {
447            tables.insert(table.to_owned());
448        }
449    }
450    fn snapshot(&self) -> Option<BTreeSet<String>> {
451        self.0.borrow().clone()
452    }
453    fn restore(&self, tables: Option<BTreeSet<String>>) {
454        *self.0.borrow_mut() = tables;
455    }
456}
457
458/// A copy-on-write in-memory mirror: a [`Rollback`] shares it, and the first
459/// write after a capture copies it once.
460#[derive(Debug, Clone, Default)]
461struct Shared<T>(Arc<T>);
462
463impl<T> std::ops::Deref for Shared<T> {
464    type Target = T;
465    fn deref(&self) -> &T {
466        &self.0
467    }
468}
469
470impl<T: Clone> std::ops::DerefMut for Shared<T> {
471    fn deref_mut(&mut self) -> &mut T {
472        Arc::make_mut(&mut self.0)
473    }
474}
475
476impl<T> From<T> for Shared<T> {
477    fn from(value: T) -> Self {
478        Self(Arc::new(value))
479    }
480}
481
482/// The in-memory mirrors of durable state, captured before a local
483/// transaction and restored when it rolls back. Queues only grow inside a
484/// transaction, so their lengths suffice.
485#[derive(Debug, Clone)]
486struct Rollback {
487    outbox: Shared<Vec<Arc<OutboxCommit>>>,
488    failed_commits: Shared<Vec<FailedCommit>>,
489    conflicts: Shared<Vec<ConflictRecord>>,
490    rejections: Shared<Vec<RejectionRecord>>,
491    schema_floor: Option<SchemaFloor>,
492    lease_state: Option<LeaseState>,
493    stopped: bool,
494    upgrading: bool,
495    sync_needed: bool,
496    active_round: Option<uuid::Uuid>,
497    overlay_dirty: Option<BTreeSet<String>>,
498    change_queue: usize,
499    sync_intent_queue: usize,
500    last_change: Option<DiagnosticLastChange>,
501}
502
503pub struct SyncClient {
504    #[cfg(feature = "bench-internals")]
505    benchmark_phases: Recorder,
506    conn: Connection,
507    schema: ClientSchema,
508    client_id: String,
509    limits: ClientLimits,
510    outbox: Shared<Vec<Arc<OutboxCommit>>>,
511    failed_commits: Shared<Vec<FailedCommit>>,
512    retain_failed_commits: bool,
513    conflicts: Shared<Vec<ConflictRecord>>,
514    rejections: Shared<Vec<RejectionRecord>>,
515    schema_floor: Option<SchemaFloor>,
516    /// §7.3.5: the opaque auth-lease state (from LEASE frames + lease errors).
517    lease_state: Option<LeaseState>,
518    /// §1.6: the schema-floor response stops syncing until an upgrade.
519    stopped: bool,
520    /// §7.4.5: true while a schema-bump reset + first re-bootstrap is in flight.
521    upgrading: bool,
522    /// §8.4 coalesced sync-needed signal.
523    sync_needed: bool,
524    /// §8.8: how sync rounds treat the realtime binding, and its exact
525    /// availability state with the evidence behind a lost or refused one.
526    realtime_policy: RealtimePolicy,
527    realtime_state: RealtimeState,
528    realtime_reason_code: Option<String>,
529    realtime_retry_delay_ms: Option<u64>,
530    /// §8.6 presence: scopeKey → (`actorId clientId` peer key → peer).
531    presence: HashMap<String, HashMap<String, PresencePeer>>,
532    /// Client clock (epoch ms) for the §5.4 `urlExpiresAtMs` check; the
533    /// host may pin it (conformance runs on a virtual clock).
534    now_ms: Option<i64>,
535    /// §5.11 client-side encryption keys (`keyId → key bytes`). Empty ⇒ E2EE
536    /// off. The encrypt/decrypt seam (`values.rs`) is compiled only under the
537    /// `e2ee` feature; without it, a schema with encrypted columns fails loud.
538    encryption: crate::values::EncryptionConfig,
539    /// Fail-closed host bootstrap gate. While set, command hosts permit only
540    /// status/lifecycle inspection and an exact authorized local purge.
541    security_preflight: bool,
542    security_epoch: u64,
543    transport_enabled: bool,
544    transport_epoch: u64,
545    /// Raised by every registration change; a current-base proof spanning
546    /// one reports missing coverage.
547    registration_epoch: u64,
548    /// RFC 0005 D2/D8: the previous-version capture config. `None` (default)
549    /// means the feature is off; the descriptor write and the container orphan
550    /// sweep still happen.
551    previous_version: Option<PreviousVersionContextConfig>,
552    /// Per-table primary-key upsert SQL, built once per (full table name) —
553    /// the row write path runs per row during bootstrap (§5.6), so the SQL
554    /// string (and, via `prepare_cached`, its compiled statement) is reused
555    /// instead of being rebuilt and re-prepared per row. Cleared on a §7.4.3
556    /// schema reset (the column lists may have changed).
557    insert_sql: RefCell<HashMap<String, String>>,
558    /// §7.1: reconcile only tables whose base/outbox changed. Whole-replica
559    /// reset and restart explicitly invalidate every table.
560    overlay_dirty: OverlayDirty,
561    // First SQLite failure of the current operation, before savepoint cleanup.
562    storage_failure: RefCell<Option<QueryReadFailure>>,
563    /// Test-only structural performance signal: response processing must not
564    /// turn a batch of acknowledgements into one full overlay rebuild each.
565    #[cfg(test)]
566    overlay_rebuild_count: Cell<usize>,
567    #[cfg(test)]
568    acknowledged_replay_count: Cell<usize>,
569    /// Test-only structural performance signal: outcome retention is enforced
570    /// once per response rather than once per acknowledgement.
571    #[cfg(test)]
572    outcome_prune_count: Cell<usize>,
573    /// Test-only pinned duration of every §5.3 image chunk, in ms.
574    #[cfg(test)]
575    image_chunk_ms: Cell<Option<f64>>,
576    /// Exact observer-transaction output drained by command/FFI hosts.
577    progress: ProgressObserver,
578    change_queue: VecDeque<ClientChangeBatch>,
579    sync_intent_queue: VecDeque<SyncIntent>,
580    /// Explicit exponential retry policy for transient transport failures.
581    retry_delay_ms: u64,
582    /// §7.7: the background retry the current round scheduled, if any.
583    round_retry_delay_ms: Option<u64>,
584    /// A round whose response never completed (§1.5): the client keeps its
585    /// code and reports it on every following request until a round receives a
586    /// complete, decodable response. The server already recorded the round, so
587    /// nothing else can tell it the round failed on this side.
588    round_failure: Option<String>,
589    active_round: Option<uuid::Uuid>,
590    last_round: Option<DiagnosticLastRound>,
591    last_change: Option<DiagnosticLastChange>,
592    /// §7.6: latest failed owned snapshot read per owner id, oldest first.
593    query_failures: Vec<DiagnosticQueryFailure>,
594    /// SPEC §1.4 rule 4: the open coalesced apply transaction of a response.
595    coalesced: Option<CoalescedApply>,
596    /// A coalesced apply transaction of the current response failed; every
597    /// later observation of that response fails until it ends.
598    coalesced_failure: Option<String>,
599}
600
601/// SPEC §1.4 rule 4: the frames and blocks of one response share one local
602/// write transaction of at most `COALESCED_APPLY_ROWS` rows. The in-memory
603/// state captured at its `BEGIN` is restored when its `COMMIT` fails.
604struct CoalescedApply {
605    rows: usize,
606    rollback: Rollback,
607}
608
609const COALESCED_APPLY_ROWS: usize = 1024;
610
611pub(crate) fn quote_ident(name: &str) -> String {
612    format!("\"{}\"", name.replace('"', "\"\""))
613}
614/// §7.4.1 `_syncular_meta` helpers. Free functions so RFC 0005's
615/// previous-version module reads and writes the SAME records as the client
616/// instead of growing a parallel meta layer.
617pub(crate) fn meta_get(conn: &Connection, key: &str) -> rusqlite::Result<Option<String>> {
618    conn.query_row(
619        "SELECT value FROM _syncular_meta WHERE key = ?1",
620        rusqlite::params![key],
621        |row| row.get::<_, String>(0),
622    )
623    .optional()
624}
625
626pub(crate) fn meta_set(conn: &Connection, key: &str, value: &str) -> rusqlite::Result<()> {
627    conn.execute(
628        "INSERT OR REPLACE INTO _syncular_meta (key, value) VALUES (?1, ?2)",
629        rusqlite::params![key, value],
630    )
631    .map(|_| ())
632}
633
634pub(crate) fn meta_delete(conn: &Connection, key: &str) -> rusqlite::Result<()> {
635    conn.execute(
636        "DELETE FROM _syncular_meta WHERE key = ?1",
637        rusqlite::params![key],
638    )
639    .map(|_| ())
640}
641
642/// Seek by the existing key while retaining the exact legacy text match.
643fn row_id_predicate(table: &TableSchema) -> String {
644    let key = quote_ident(&table.primary_key);
645    let text_match = format!("CAST({key} AS TEXT) = ?1");
646    match table.columns[table.pk_index].ty {
647        ColumnType::String | ColumnType::Json => {
648            format!("{key} = ?1 AND {text_match}")
649        }
650        // Native columns have no declared SQLite affinity. Match the stored
651        // integer bind type explicitly. Unary + removes expression affinity so
652        // SQLite can seek the typeless key; the text check preserves exact IDs.
653        ColumnType::Integer | ColumnType::Boolean => {
654            format!("{key} = +CAST(?1 AS INTEGER) AND {text_match}")
655        }
656        // REAL-to-text conversion can round distinct values to the same text.
657        // Narrowing those matches by numeric equality would change behavior.
658        _ => text_match,
659    }
660}
661
662/// §3.3/§4.8/§5.6 row ownership: rows whose local scope columns match
663/// `effective` and that no entry of `owners` matches. `owners` holds the
664/// pair scopes of every owner of the table (§3.3 row ownership); a scoped delete never removes a row one of them still covers.
665/// Scope values bind by the column's declared type, as `row_id_predicate`
666/// does. `Ok(None)` matches nothing (an empty map or value list); `Err(())`
667/// is a missing local scope-column mapping (the fail-closed case).
668fn local_scope_predicate(
669    table: &TableSchema,
670    effective: &[(String, Vec<String>)],
671    owners: &[Vec<(String, Vec<String>)>],
672) -> Result<Option<(String, Vec<SqlValue>)>, ()> {
673    if effective.is_empty() || effective.iter().any(|(_, values)| values.is_empty()) {
674        return Ok(None);
675    }
676    let column = |variable: &str| {
677        let name = table.scope_column(variable).ok_or(())?;
678        let column = table
679            .columns
680            .iter()
681            .find(|column| column.name == name)
682            .ok_or(())?;
683        Ok((quote_ident(name), column.ty == ColumnType::Integer))
684    };
685    let mut clauses = Vec::new();
686    let mut params = Vec::new();
687    for (variable, values) in effective {
688        let (key, integer) = column(variable)?;
689        let holes = vec!["?"; values.len()].join(", ");
690        clauses.push(if integer {
691            params.extend(values.iter().cloned().map(SqlValue::Text));
692            let seek = vec!["+CAST(? AS INTEGER)"; values.len()].join(", ");
693            format!("{key} IN ({seek}) AND CAST({key} AS TEXT) IN ({holes})")
694        } else {
695            format!("{key} IN ({holes})")
696        });
697        params.extend(values.iter().cloned().map(SqlValue::Text));
698    }
699    // Owners bind as one JSON parameter per variable set, so the statement
700    // keeps its size at any owner count; owners of one variable bind the
701    // union of their values. An owner holding other values of a variable
702    // `effective` names owns none of its rows and drops out. Values compare
703    // as text, the exact semantics of the effective clauses.
704    let mut groups: BTreeMap<Vec<&str>, Vec<Vec<&[String]>>> = BTreeMap::new();
705    for owner in owners {
706        if owner.is_empty()
707            || owner.iter().any(|(variable, values)| {
708                values.is_empty()
709                    || effective.iter().any(|(name, wanted)| {
710                        name == variable && wanted.iter().all(|value| !values.contains(value))
711                    })
712            })
713        {
714            continue;
715        }
716        let mut owner: Vec<_> = owner.iter().collect();
717        owner.sort_by(|a, b| a.0.cmp(&b.0));
718        groups
719            .entry(
720                owner
721                    .iter()
722                    .map(|(variable, _)| variable.as_str())
723                    .collect(),
724            )
725            .or_default()
726            .push(owner.iter().map(|(_, values)| values.as_slice()).collect());
727    }
728    let within = |variable: &str, list: &str| {
729        let (key, integer) = column(variable)?;
730        let key = if integer {
731            format!("CAST({key} AS TEXT)")
732        } else {
733            key
734        };
735        Ok(format!("{key} IN (SELECT value FROM json_each({list}))"))
736    };
737    for (variables, tuples) in groups {
738        if let [variable] = variables[..] {
739            let values: BTreeSet<&String> = tuples.iter().flatten().copied().flatten().collect();
740            params.push(SqlValue::Text(
741                Value::from_iter(values.into_iter().cloned()).to_string(),
742            ));
743            clauses.push(format!("NOT {}", within(variable, "?")?));
744        } else {
745            params.push(SqlValue::Text(
746                serde_json::to_string(&tuples).expect("owner tuples serialize"),
747            ));
748            let matches = variables
749                .iter()
750                .enumerate()
751                .map(|(index, variable)| within(variable, &format!("o.value,'$[{index}]'")))
752                .collect::<Result<Vec<_>, ()>>()?;
753            clauses.push(format!(
754                "NOT EXISTS (SELECT 1 FROM json_each(?) o WHERE {})",
755                matches.join(" AND ")
756            ));
757        }
758    }
759    Ok(Some((clauses.join(" AND "), params)))
760}
761
762/// [`local_scope_predicate`] over decoded row values (retained rows).
763fn owns_local_row(
764    table: &TableSchema,
765    effective: &[(String, Vec<String>)],
766    owners: &[Vec<(String, Vec<String>)>],
767    values: &Map<String, Value>,
768) -> bool {
769    let matches = |scopes: &[(String, Vec<String>)]| {
770        !scopes.is_empty()
771            && scopes.iter().all(|(variable, allowed)| {
772                table
773                    .scope_column(variable)
774                    .and_then(|column| values.get(column))
775                    .filter(|value| !value.is_null())
776                    .is_some_and(|value| {
777                        allowed.contains(
778                            &value
779                                .as_str()
780                                .map_or_else(|| value.to_string(), str::to_owned),
781                        )
782                    })
783            })
784    };
785    matches(effective) && !owners.iter().any(|owner| matches(owner))
786}
787
788fn is_local_operation_code_like(value: &str) -> bool {
789    let bytes = value.as_bytes();
790    bytes.first().is_some_and(u8::is_ascii_alphanumeric)
791        && bytes
792            .iter()
793            .all(|byte| byte.is_ascii_alphanumeric() || matches!(*byte, b'.' | b'_' | b':' | b'-'))
794}
795
796fn base_table(name: &str) -> String {
797    quote_ident(&format!("_syncular_base_{name}"))
798}
799
800/// §4.8 deferred eviction record: (unit id, table, effective scope map).
801type PendingEvict = (String, String, Vec<(String, Vec<String>)>);
802
803/// §4.8: a stable key for a window base: variable + canonical fixed scopes.
804fn window_base_key(base: &WindowBase) -> String {
805    format!(
806        "{}\0{}",
807        base.variable,
808        ssp2::registration::canonical_unit_json(&scope_pairs(&base.fixed_scopes))
809    )
810}
811
812/// The unit of one window value: the fixed scopes plus the value.
813fn window_unit_scopes(base: &WindowBase, value: &str) -> UnitScopes {
814    let mut scopes = base.fixed_scopes.clone();
815    scopes.insert(base.variable.clone(), value.to_owned());
816    scopes
817}
818
819/// A unit's scope map as the codec and the digests take it.
820fn scope_pairs(scopes: &UnitScopes) -> Vec<(String, String)> {
821    let mut pairs: Vec<(String, String)> = scopes
822        .iter()
823        .map(|(variable, value)| (variable.clone(), value.clone()))
824        .collect();
825    pairs.sort_by(|a, b| a.0.encode_utf16().cmp(b.0.encode_utf16()));
826    pairs
827}
828
829/// §4.1: the unit covers every table declaring all of its variables.
830fn covers(table: &TableSchema, scopes: &UnitScopes) -> bool {
831    scopes
832        .keys()
833        .all(|variable| table.scope_column(variable).is_some())
834}
835
836/// §4.4: the effective scopes of the (unit, table) pair under `echo`: the
837/// scope map as one-element lists plus the fenced lists. `None` when the
838/// fence revoked the pair.
839fn pair_scopes(
840    scopes: &UnitScopes,
841    table: &str,
842    echo: Option<&[EchoRecord]>,
843) -> Option<Vec<(String, Vec<String>)>> {
844    let fenced = echo.and_then(|echo| echo.iter().find(|record| record.table == table));
845    if fenced.is_some_and(|record| record.status == "revoked") {
846        return None;
847    }
848    let mut effective: BTreeMap<String, Vec<String>> = fenced
849        .map(|record| record.scopes.clone())
850        .unwrap_or_default();
851    for (variable, value) in scopes {
852        effective.insert(variable.clone(), vec![value.clone()]);
853    }
854    Some(effective.into_iter().collect())
855}
856
857/// §3.3 row ownership: the pair scopes of every owner of `table` except
858/// `except_id`. An owner is active with a persisted echo and live or
859/// post-bootstrap, or bootstrapping with a segment of the table in its
860/// current bootstrap.
861fn unit_owners(
862    units: &[UnitRecord],
863    table: &TableSchema,
864    except_id: &str,
865) -> Vec<Vec<(String, Vec<String>)>> {
866    units
867        .iter()
868        .filter(|unit| {
869            unit.id != except_id
870                && unit.status == UnitStatus::Active
871                && unit.echo.is_some()
872                && (!unit.bootstrapping() || unit.seeded.contains(&table.name))
873                && covers(table, &unit.scopes)
874        })
875        .filter_map(|unit| pair_scopes(&unit.scopes, &table.name, unit.echo.as_deref()))
876        .collect()
877}
878
879/// §5.5 / §5.6: the unit's scope map, each value a one-element list.
880fn unit_scope_lists(scopes: &UnitScopes) -> Vec<(String, Vec<String>)> {
881    scopes
882        .iter()
883        .map(|(variable, value)| (variable.clone(), vec![value.clone()]))
884        .collect()
885}
886
887/// §7.1 acknowledged intent of the rows `?1` names (`[[table, id], ...]`).
888/// It joins through the (tbl, id) index: a scan per acknowledgement would
889/// make draining a batch quadratic.
890const ACKED_ROWS_BY_ROW: &str = "SELECT a.op_json,a.intent_json FROM json_each(?1) k JOIN _syncular_acked_rows a ON a.tbl=json_extract(k.value,'$[0]') AND a.id=json_extract(k.value,'$[1]') WHERE ?2 IS NULL OR a.tbl IN(SELECT value FROM json_each(?2)) ORDER BY a.commit_seq,a.idx";
891
892const UNIT_COLUMNS: &str =
893    "id, scopes, explicit, status, reason_code, cursor, bootstrap_state, seeded, echo";
894
895/// A unit row; a field that does not parse is a conversion failure, which
896/// both read paths classify as `sync.local_corrupt`.
897fn unit_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result<UnitRecord> {
898    fn json<T: serde::de::DeserializeOwned>(index: usize, raw: &str) -> rusqlite::Result<T> {
899        serde_json::from_str(raw).map_err(|error| {
900            rusqlite::Error::FromSqlConversionFailure(
901                index,
902                rusqlite::types::Type::Text,
903                Box::new(error),
904            )
905        })
906    }
907    Ok(UnitRecord {
908        id: row.get(0)?,
909        scopes: json(1, &row.get::<_, String>(1)?)?,
910        explicit: row.get::<_, i64>(2)? == 1,
911        status: json(
912            3,
913            &serde_json::to_string(&row.get::<_, String>(3)?).expect("text"),
914        )?,
915        reason_code: row.get(4)?,
916        cursor: row.get(5)?,
917        bootstrap_state: row.get(6)?,
918        seeded: json(7, &row.get::<_, String>(7)?)?,
919        echo: row
920            .get::<_, Option<String>>(8)?
921            .map(|echo| json::<Vec<EchoRecord>>(8, &echo))
922            .transpose()?
923            .map(|echo| {
924                if echo
925                    .iter()
926                    .all(|record| record.status == "active" || record.status == "revoked")
927                {
928                    Ok(echo)
929                } else {
930                    Err(rusqlite::Error::FromSqlConversionFailure(
931                        8,
932                        rusqlite::types::Type::Text,
933                        "echo status is neither active nor revoked".into(),
934                    ))
935                }
936            })
937            .transpose()?,
938    })
939}
940
941/// Every unit in registration order (§4.1).
942fn load_units(conn: &Connection) -> rusqlite::Result<Vec<UnitRecord>> {
943    conn.prepare_cached(&format!(
944        "SELECT {UNIT_COLUMNS} FROM _syncular_units ORDER BY rowid"
945    ))?
946    .query_map([], unit_from_row)?
947    .collect()
948}
949
950fn get_unit(conn: &Connection, id: &str) -> rusqlite::Result<Option<UnitRecord>> {
951    conn.query_row(
952        &format!("SELECT {UNIT_COLUMNS} FROM _syncular_units WHERE id = ?1"),
953        [id],
954        unit_from_row,
955    )
956    .optional()
957}
958
959/// Insert or update in place: an update keeps the registration order.
960fn save_unit(conn: &Connection, unit: &UnitRecord) -> rusqlite::Result<()> {
961    conn.prepare_cached(&format!(
962        "INSERT INTO _syncular_units({UNIT_COLUMNS}) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9)
963         ON CONFLICT(id) DO UPDATE SET explicit=excluded.explicit, status=excluded.status,
964           reason_code=excluded.reason_code, cursor=excluded.cursor,
965           bootstrap_state=excluded.bootstrap_state, seeded=excluded.seeded, echo=excluded.echo"
966    ))?
967    .execute(rusqlite::params![
968        unit.id,
969        serde_json::to_string(&unit.scopes).expect("scope map serializes"),
970        i64::from(unit.explicit),
971        unit.status.name(),
972        unit.reason_code,
973        unit.cursor,
974        unit.bootstrap_state,
975        serde_json::to_string(&unit.seeded).expect("table list serializes"),
976        unit.echo
977            .as_ref()
978            .map(|echo| serde_json::to_string(echo).expect("echo serializes")),
979    ])
980    .map(|_| ())
981}
982
983/// §4.1 references: every window registry entry `(base, value)` holding a unit.
984fn window_refs(conn: &Connection, unit_id: &str) -> rusqlite::Result<Vec<(String, String)>> {
985    conn.prepare_cached("SELECT base, unit FROM _syncular_windows WHERE sub_id = ?1")?
986        .query_map([unit_id], |row| Ok((row.get(0)?, row.get(1)?)))?
987        .collect()
988}
989
990/// §7.4.3 reset, §2.1 log reset and local rebootstrap: keep the unit
991/// registrations and discard every synced state, so the next round sends the
992/// list and fresh-bootstraps every unit. The caller owns the transaction.
993fn reset_units(conn: &Connection) -> rusqlite::Result<()> {
994    conn.execute_batch(
995        "UPDATE _syncular_units SET status='active', reason_code=NULL, cursor=-1,
996           bootstrap_state=NULL, seeded='[]', echo=NULL;
997         DELETE FROM _syncular_meta WHERE key IN ('replicaCursor','echoDigest','registrationDigest');",
998    )
999}
1000
1001/// §4.5: the replica cursor, `-1` while no unit is live.
1002fn replica_cursor(conn: &Connection) -> rusqlite::Result<i64> {
1003    meta_get(conn, REPLICA_CURSOR_KEY)?.map_or(Ok(-1), |raw| {
1004        raw.parse().map_err(|error| {
1005            rusqlite::Error::FromSqlConversionFailure(
1006                0,
1007                rusqlite::types::Type::Text,
1008                Box::new(error),
1009            )
1010        })
1011    })
1012}
1013
1014fn visible_table(name: &str) -> String {
1015    quote_ident(name)
1016}
1017
1018const FTS_SOURCE_ID_COLUMN: &str = "_syncular_source_id";
1019
1020/// §7.4.3: is a `sqlite_master` table name a synced table (visible or base),
1021/// i.e. NOT one of the durable bookkeeping tables the reset preserves?
1022fn is_synced_table_name(name: &str) -> bool {
1023    if name.starts_with("sqlite_") {
1024        return false;
1025    }
1026    if name.starts_with("_syncular_base_") {
1027        return true; // the base half of a synced table pair
1028    }
1029    // Bookkeeping: outbox, units, meta, blob cache/uploads.
1030    !name.starts_with("_syncular_")
1031}
1032
1033/// `"sha256:" + hex` of the bytes — the content address (§5.9.1).
1034fn blob_id_for(bytes: &[u8]) -> String {
1035    let digest = Sha256::digest(bytes);
1036    format!("sha256:{}", bytes_to_hex(&digest))
1037}
1038
1039/// One [`SyncClient::write_row`] bind parameter, borrowing the row-codec
1040/// value it wraps — strings/JSON/bytes bind as borrowed TEXT/BLOB (no copy
1041/// per row on the §5.6 bootstrap path), scalars bind owned.
1042enum RowParam<'a> {
1043    Cell(&'a Option<ColumnValue>),
1044    Version(i64),
1045    Text(Option<&'a str>),
1046}
1047
1048impl rusqlite::ToSql for RowParam<'_> {
1049    fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
1050        Ok(match self {
1051            RowParam::Version(v) => ToSqlOutput::Owned(SqlValue::Integer(*v)),
1052            RowParam::Text(text) => match text {
1053                Some(text) => ToSqlOutput::Borrowed(ValueRef::Text(text.as_bytes())),
1054                None => ToSqlOutput::Owned(SqlValue::Null),
1055            },
1056            RowParam::Cell(cell) => match cell {
1057                None => ToSqlOutput::Owned(SqlValue::Null),
1058                Some(ColumnValue::String(s)) => ToSqlOutput::Borrowed(ValueRef::Text(s.as_bytes())),
1059                Some(ColumnValue::Integer(i)) => ToSqlOutput::Owned(SqlValue::Integer(*i)),
1060                Some(ColumnValue::Float(f)) => ToSqlOutput::Owned(SqlValue::Real(*f)),
1061                Some(ColumnValue::Boolean(b)) => {
1062                    ToSqlOutput::Owned(SqlValue::Integer(i64::from(*b)))
1063                }
1064                Some(ColumnValue::Json(raw)) | Some(ColumnValue::BlobRef(raw)) => {
1065                    ToSqlOutput::Borrowed(ValueRef::Text(raw.0.as_bytes()))
1066                }
1067                // §5.10: crdt bytes store as BLOB, like bytes.
1068                Some(ColumnValue::Bytes(b)) | Some(ColumnValue::Crdt(b)) => {
1069                    ToSqlOutput::Borrowed(ValueRef::Blob(b))
1070                }
1071            },
1072        })
1073    }
1074}
1075
1076// Retain a typed primary key across the current import chunk without copying row payloads.
1077fn owned_sql_value(param: impl rusqlite::ToSql) -> Result<SqlValue, String> {
1078    match param.to_sql().map_err(|error| error.to_string())? {
1079        ToSqlOutput::Owned(value) => Ok(value),
1080        ToSqlOutput::Borrowed(value) => {
1081            SqlValue::try_from(value).map_err(|error| error.to_string())
1082        }
1083        _ => Err("sync.invalid_request: unsupported SQLite key parameter".into()),
1084    }
1085}
1086
1087/// §5.3 image cell → bind parameter, strict per the declared column type
1088/// (`boolean` from INTEGER 0/1, `json` from its raw TEXT, NULL only when
1089/// nullable). Mismatches are image-producer violations. Returns the cell
1090/// borrowed when the stored representation already matches what the row
1091/// codec would write, or the normalized scalar (boolean → 0/1, float from
1092/// INTEGER → REAL) otherwise — the local write is byte-identical to the
1093/// old convert-then-insert path without allocating per cell.
1094fn image_cell_param<'a>(column: &Column, value: ValueRef<'a>) -> Result<ToSqlOutput<'a>, String> {
1095    use ssp2::segment::ColumnType;
1096    let mismatch = || {
1097        Err(format!(
1098            "image column {:?} holds a value of the wrong type",
1099            column.name
1100        ))
1101    };
1102    match value {
1103        ValueRef::Null => {
1104            if !column.nullable {
1105                return Err(format!(
1106                    "image column {:?} is NULL but not nullable",
1107                    column.name
1108                ));
1109            }
1110            Ok(ToSqlOutput::Owned(SqlValue::Null))
1111        }
1112        ValueRef::Integer(i) => match column.ty {
1113            ColumnType::Integer => Ok(ToSqlOutput::Borrowed(value)),
1114            ColumnType::Boolean => Ok(ToSqlOutput::Owned(SqlValue::Integer(i64::from(i != 0)))),
1115            ColumnType::Float => Ok(ToSqlOutput::Owned(SqlValue::Real(i as f64))),
1116            _ => mismatch(),
1117        },
1118        ValueRef::Real(_) => match column.ty {
1119            ColumnType::Float => Ok(ToSqlOutput::Borrowed(value)),
1120            _ => mismatch(),
1121        },
1122        ValueRef::Text(t) => {
1123            std::str::from_utf8(t)
1124                .map_err(|_| format!("image column {:?} is not UTF-8", column.name))?;
1125            match column.ty {
1126                ColumnType::String | ColumnType::Json | ColumnType::BlobRef => {
1127                    Ok(ToSqlOutput::Borrowed(value))
1128                }
1129                _ => mismatch(),
1130            }
1131        }
1132        ValueRef::Blob(_) => match column.ty {
1133            // §5.10: a crdt column stores its opaque bytes as BLOB, like bytes.
1134            ColumnType::Bytes | ColumnType::Crdt => Ok(ToSqlOutput::Borrowed(value)),
1135            _ => mismatch(),
1136        },
1137    }
1138}
1139
1140pub(crate) fn sql_ref_to_json(column: &Column, value: rusqlite::types::ValueRef<'_>) -> Value {
1141    use rusqlite::types::ValueRef;
1142    match value {
1143        ValueRef::Null => Value::Null,
1144        ValueRef::Integer(i) => match column.ty {
1145            ssp2::segment::ColumnType::Boolean => Value::Bool(i != 0),
1146            ssp2::segment::ColumnType::Float => {
1147                serde_json::Number::from_f64(i as f64).map_or(Value::Null, Value::Number)
1148            }
1149            _ => Value::from(i),
1150        },
1151        ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
1152        ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
1153        ValueRef::Blob(b) => {
1154            let mut map = Map::new();
1155            map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
1156            Value::Object(map)
1157        }
1158    }
1159}
1160
1161/// Bind a driver JSON value form as a rusqlite parameter for [`SyncClient::query`].
1162/// Objects are accepted in lossless `{"$bytes": hex}` and
1163/// `{"$bigint": decimal}` envelope forms.
1164fn json_param_to_sql(value: &Value) -> Result<SqlValue, String> {
1165    Ok(match value {
1166        Value::Null => SqlValue::Null,
1167        Value::Bool(b) => SqlValue::Integer(i64::from(*b)),
1168        Value::Number(n) => {
1169            if let Some(i) = n.as_i64() {
1170                SqlValue::Integer(i)
1171            } else if let Some(f) = n.as_f64() {
1172                SqlValue::Real(f)
1173            } else {
1174                return Err(format!("query param number {n} is out of range"));
1175            }
1176        }
1177        Value::String(s) => SqlValue::Text(s.clone()),
1178        Value::Object(_) => {
1179            if let Some(hex) = value.get("$bytes").and_then(Value::as_str) {
1180                SqlValue::Blob(crate::values::hex_to_bytes(hex)?)
1181            } else if let Some(decimal) = value.get("$bigint").and_then(Value::as_str) {
1182                SqlValue::Integer(decimal.parse::<i64>().map_err(|_| {
1183                    format!("query bigint param {decimal:?} is outside SQLite's i64 range")
1184                })?)
1185            } else {
1186                return Err(
1187                    "query object param must be a {$bytes: hex} or {$bigint: decimal} value"
1188                        .to_owned(),
1189                );
1190            }
1191        }
1192        Value::Array(_) => return Err("query array params are not supported".to_owned()),
1193    })
1194}
1195
1196/// Map a rusqlite value with no schema column to consult (arbitrary query
1197/// output): integers/reals/text pass through by stored affinity, blobs ride
1198/// as `{"$bytes": hex}`. Distinct from [`sql_ref_to_json`], which uses the
1199/// schema column type to recover booleans/floats/json.
1200fn sql_ref_to_json_dynamic(value: rusqlite::types::ValueRef<'_>) -> Value {
1201    use rusqlite::types::ValueRef;
1202    match value {
1203        ValueRef::Null => Value::Null,
1204        ValueRef::Integer(i) => {
1205            // JSON/Tauri IPC cannot represent every SQLite i64 exactly. Keep
1206            // ordinary UI-sized integers ergonomic and envelope only values
1207            // beyond JavaScript's safe range.
1208            const MAX_SAFE_INTEGER: i64 = 9_007_199_254_740_991;
1209            if (-MAX_SAFE_INTEGER..=MAX_SAFE_INTEGER).contains(&i) {
1210                Value::from(i)
1211            } else {
1212                let mut map = Map::new();
1213                map.insert("$bigint".to_owned(), Value::from(i.to_string()));
1214                Value::Object(map)
1215            }
1216        }
1217        ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
1218        ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
1219        ValueRef::Blob(b) => {
1220            let mut map = Map::new();
1221            map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
1222            Value::Object(map)
1223        }
1224    }
1225}
1226
1227/// §7.5: SQLite failures classify by result code into [`QueryReadFailure`].
1228fn query_connection(
1229    conn: &Connection,
1230    sql: &str,
1231    params: &[QueryValue],
1232) -> Result<Vec<QueryRow>, QueryReadFailure> {
1233    let lowered_sql = crate::query_guard::lower_public_query_sql(sql);
1234    let mut stmt = crate::query_guard::prepare_read_only(conn, &lowered_sql)?;
1235    let bound: Vec<SqlValue> = params
1236        .iter()
1237        .map(json_param_to_sql)
1238        .collect::<Result<_, _>>()?;
1239    // §2.4: an unaliased physical version column stays engine-internal, so a
1240    // `SELECT *` row feeds straight back into mutation values. An explicit
1241    // alias (`_sync_version AS v`) returns it as ordinary query data. The
1242    // §5.11 sealed columns follow the same rule.
1243    let columns: Vec<(usize, String)> = stmt
1244        .column_names()
1245        .enumerate()
1246        .filter(|(_, name)| {
1247            !matches!(
1248                *name,
1249                "_syncular_version" | "_sync_sealed_key" | "_sync_sealed"
1250            )
1251        })
1252        .map(|(i, name)| (i, name.to_owned()))
1253        .collect();
1254    let bound_refs: Vec<&dyn rusqlite::ToSql> =
1255        bound.iter().map(|v| v as &dyn rusqlite::ToSql).collect();
1256    let mut sql_rows = stmt.query(bound_refs.as_slice())?;
1257    let mut out = Vec::new();
1258    while let Some(row) = sql_rows.next()? {
1259        let mut record = Map::new();
1260        for (i, name) in &columns {
1261            let value = row.get_ref(*i)?;
1262            record.insert(name.clone(), sql_ref_to_json_dynamic(value));
1263        }
1264        out.push(record);
1265    }
1266    Ok(out)
1267}
1268
1269fn persisted_window_state(
1270    conn: &Connection,
1271    base: &WindowBase,
1272) -> Result<WindowState, QueryReadFailure> {
1273    let mut stmt = conn.prepare(
1274        "SELECT windows.unit, units.status, units.cursor, units.bootstrap_state
1275           FROM _syncular_windows AS windows
1276           LEFT JOIN _syncular_units AS units ON units.id = windows.sub_id
1277          WHERE windows.base = ?1
1278          ORDER BY windows.unit ASC",
1279    )?;
1280    let rows = stmt.query_map(rusqlite::params![window_base_key(base)], |row| {
1281        Ok((
1282            row.get::<_, String>(0)?,
1283            row.get::<_, Option<String>>(1)?,
1284            row.get::<_, Option<i64>>(2)?,
1285            row.get::<_, Option<String>>(3)?,
1286        ))
1287    })?;
1288    let mut units = Vec::new();
1289    let mut pending = Vec::new();
1290    for row in rows {
1291        let (unit, status, cursor, bootstrap_state) = row?;
1292        // §4.8: complete iff registered, active and not bootstrapping.
1293        if status.as_deref() != Some("active")
1294            || cursor.is_some_and(|cursor| cursor < 0 || bootstrap_state.is_some())
1295        {
1296            pending.push(unit.clone());
1297        }
1298        units.push(unit);
1299    }
1300    Ok(WindowState { units, pending })
1301}
1302
1303/// One read-only statement in a §7.5 snapshot read.
1304#[derive(Debug, Clone, Copy)]
1305pub struct SnapshotStatement<'a> {
1306    pub sql: &'a str,
1307    pub params: &'a [Value],
1308}
1309
1310/// A §7.5 atomic read request: read-only statements, window coverage, unit
1311/// catch-up states, and commit deliveries, all resolved from one SQLite read
1312/// transaction and one revision.
1313#[derive(Debug, Clone, Default)]
1314pub struct SnapshotReadRequest<'a> {
1315    pub statements: Vec<SnapshotStatement<'a>>,
1316    pub coverage: &'a [WindowCoverage],
1317    pub units: Vec<String>,
1318    pub commit_ids: Vec<String>,
1319}
1320
1321#[cfg(test)]
1322thread_local! {
1323    /// Test-only rendezvous that runs between the statements of one snapshot
1324    /// read. A test uses it to commit on another connection mid-read and prove
1325    /// the read transaction's snapshot and revision stay stable.
1326    static SNAPSHOT_READ_HOOK: std::cell::RefCell<Option<Box<dyn FnOnce()>>> =
1327        std::cell::RefCell::new(None);
1328}
1329
1330#[cfg(test)]
1331fn run_snapshot_read_hook() {
1332    SNAPSHOT_READ_HOOK.with(|slot| {
1333        if let Some(hook) = slot.borrow_mut().take() {
1334            hook();
1335        }
1336    });
1337}
1338
1339/// §7.5: resolve a request over one SQLite read transaction. The revision,
1340/// every statement's rows, coverage, catch-up states, and deliveries are
1341/// captured under one `SAVEPOINT`; any statement's failure rolls the whole
1342/// read back and releases the connection.
1343fn snapshot_read_connection(
1344    conn: &Connection,
1345    request: &SnapshotReadRequest<'_>,
1346) -> Result<SnapshotRead, QueryReadFailure> {
1347    conn.execute_batch("SAVEPOINT syncular_snapshot_read")?;
1348    let corrupt = |message: &str| QueryReadFailure {
1349        code: Some("sync.local_corrupt"),
1350        sqlite_code: None,
1351        sqlite_message: None,
1352        rollback_failure: None,
1353        message: message.to_owned(),
1354    };
1355    let stored_failure = |error: rusqlite::Error, message: &str| match error {
1356        rusqlite::Error::Utf8Error(..)
1357        | rusqlite::Error::InvalidColumnType(..)
1358        | rusqlite::Error::FromSqlConversionFailure(..)
1359        | rusqlite::Error::IntegralValueOutOfRange(..) => corrupt(message),
1360        _ => QueryReadFailure::from(error),
1361    };
1362    let result: Result<SnapshotRead, QueryReadFailure> = (|| {
1363        // The durable revision as a canonical decimal `u64`; a missing marker
1364        // is the legacy zero, and any other non-canonical value is corrupt.
1365        let revision: Option<String> = conn
1366            .query_row(
1367                "SELECT value FROM _syncular_meta WHERE key = ?1",
1368                rusqlite::params![LOCAL_REVISION_KEY],
1369                |row| row.get(0),
1370            )
1371            .optional()
1372            .map_err(|error| stored_failure(error, "persisted local revision is invalid"))?;
1373        let revision = match revision {
1374            None => "0".to_owned(),
1375            Some(value)
1376                if value == "0"
1377                    || (matches!(value.as_bytes().first(), Some(b'1'..=b'9'))
1378                        && value.bytes().all(|b| b.is_ascii_digit())
1379                        && value.parse::<u64>().is_ok()) =>
1380            {
1381                value
1382            }
1383            Some(_) => return Err(corrupt("persisted local revision is invalid")),
1384        };
1385        let mut queries = Vec::with_capacity(request.statements.len());
1386        for statement in &request.statements {
1387            queries.push(query_connection(conn, statement.sql, statement.params)?);
1388            #[cfg(test)]
1389            run_snapshot_read_hook();
1390        }
1391        let mut pending = Vec::new();
1392        let mut missing = Vec::new();
1393        for requested in request.coverage {
1394            let base_key = window_base_key(&requested.base);
1395            let state = persisted_window_state(conn, &requested.base)?;
1396            for unit in BTreeSet::from_iter(requested.units.iter().cloned()) {
1397                let reference = WindowUnitRef {
1398                    base_key: base_key.clone(),
1399                    unit: unit.clone(),
1400                };
1401                if !state.units.iter().any(|held| held == &unit) {
1402                    missing.push(reference);
1403                } else if state.pending.iter().any(|held| held == &unit) {
1404                    pending.push(reference);
1405                }
1406            }
1407        }
1408        let mut units = Vec::with_capacity(request.units.len());
1409        for id in &request.units {
1410            let exists: bool = conn.query_row(
1411                "SELECT EXISTS(SELECT 1 FROM _syncular_units WHERE id = ?1)",
1412                [id],
1413                |row| row.get(0),
1414            )?;
1415            if !exists {
1416                units.push(UnitCatchup::Unknown { id: id.clone() });
1417                continue;
1418            }
1419            // Every persisted field is required to be well typed; a corrupt
1420            // record fails typed instead of inventing a state.
1421            let unit = get_unit(conn, id)
1422                .map_err(|error| stored_failure(error, "persisted unit state is invalid"))?
1423                .ok_or_else(|| corrupt("persisted unit state is invalid"))?;
1424            let cursor = match unit.cursor {
1425                Some(cursor) => cursor,
1426                None => replica_cursor(conn)
1427                    .map_err(|error| stored_failure(error, "persisted unit state is invalid"))?,
1428            };
1429            if cursor.unsigned_abs() > MAX_JS_SAFE_INTEGER {
1430                return Err(corrupt("persisted unit state is invalid"));
1431            }
1432            let active = unit.status == UnitStatus::Active;
1433            let pending = unit.bootstrapping();
1434            units.push(UnitCatchup::Known {
1435                id: id.clone(),
1436                status: unit.status,
1437                live: unit.cursor.is_none(),
1438                cursor,
1439                has_resume_token: unit.bootstrap_state.is_some(),
1440                bootstrap_complete: active && !pending,
1441                known_pending_pages: active && pending,
1442                reason_code: unit.reason_code,
1443            });
1444        }
1445        let mut deliveries = Vec::with_capacity(request.commit_ids.len());
1446        for id in &request.commit_ids {
1447            let pending: bool = conn.query_row(
1448                "SELECT EXISTS(SELECT 1 FROM _syncular_outbox WHERE commit_id = ?1)",
1449                rusqlite::params![id],
1450                |row| row.get(0),
1451            )?;
1452            if pending {
1453                deliveries.push(CommitDelivery::Pending {
1454                    client_commit_id: id.clone(),
1455                });
1456                continue;
1457            }
1458            let row = conn
1459                .query_row(
1460                    "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
1461                            resolution, resolved_at_ms, replacement_client_commit_id
1462                       FROM _syncular_commit_outcomes WHERE client_commit_id = ?1",
1463                    rusqlite::params![id],
1464                    StoredCommitOutcomeRow::from_row,
1465                )
1466                .optional()
1467                .map_err(|error| stored_failure(error, "persisted commit outcome is invalid"))?;
1468            deliveries.push(match row {
1469                Some(row) => CommitDelivery::Known {
1470                    client_commit_id: id.clone(),
1471                    outcome: SyncClient::outcome_from_row(row)
1472                        .map_err(|_| corrupt("persisted commit outcome is invalid"))?,
1473                },
1474                None => CommitDelivery::Unknown {
1475                    client_commit_id: id.clone(),
1476                },
1477            });
1478        }
1479        Ok(SnapshotRead {
1480            revision,
1481            queries,
1482            coverage: CoverageSnapshot {
1483                complete: pending.is_empty() && missing.is_empty(),
1484                pending,
1485                missing,
1486            },
1487            units,
1488            deliveries,
1489        })
1490    })();
1491    match result {
1492        Ok(read) => {
1493            conn.execute_batch("RELEASE syncular_snapshot_read")?;
1494            Ok(read)
1495        }
1496        Err(mut error) => {
1497            if let Err(rollback) = conn
1498                .execute_batch("ROLLBACK TO syncular_snapshot_read; RELEASE syncular_snapshot_read")
1499            {
1500                error.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
1501            }
1502            Err(error)
1503        }
1504    }
1505}
1506
1507fn snapshot_connection(
1508    conn: &Connection,
1509    sql: &str,
1510    params: &[Value],
1511    coverage: &[WindowCoverage],
1512) -> Result<QuerySnapshot, QueryReadFailure> {
1513    let read = snapshot_read_connection(
1514        conn,
1515        &SnapshotReadRequest {
1516            statements: vec![SnapshotStatement { sql, params }],
1517            coverage,
1518            units: Vec::new(),
1519            commit_ids: Vec::new(),
1520        },
1521    )?;
1522    Ok(QuerySnapshot {
1523        revision: read.revision,
1524        rows: read
1525            .queries
1526            .into_iter()
1527            .next()
1528            .expect("single-query snapshot returns one result"),
1529        coverage: read.coverage,
1530    })
1531}
1532
1533/// A long-lived read-only SQLite sidecar for latency-critical native views.
1534/// Network rounds stay serialized on the mutable core owner, while atomic
1535/// query snapshots use this independent connection and therefore never queue
1536/// behind HTTP/WebSocket latency.
1537pub struct FileQuerySnapshotReader {
1538    path: String,
1539    conn: Option<Connection>,
1540}
1541
1542impl FileQuerySnapshotReader {
1543    #[must_use]
1544    pub fn new(path: impl Into<String>) -> Self {
1545        Self {
1546            path: path.into(),
1547            conn: None,
1548        }
1549    }
1550
1551    fn connection(&mut self) -> Result<&Connection, QueryReadFailure> {
1552        if self.conn.is_none() {
1553            let conn = Connection::open_with_flags(
1554                &self.path,
1555                OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
1556            )
1557            .map_err(QueryReadFailure::from)?;
1558            conn.busy_timeout(std::time::Duration::from_millis(250))
1559                .map_err(QueryReadFailure::from)?;
1560            self.conn = Some(conn);
1561        }
1562        self.conn
1563            .as_ref()
1564            .ok_or_else(|| QueryReadFailure::from("read sidecar connection missing".to_owned()))
1565    }
1566
1567    /// One atomic read over the shared primitive: every statement, coverage,
1568    /// catch-up, and delivery under one `SAVEPOINT` and one revision.
1569    pub fn snapshot_read(
1570        &mut self,
1571        request: &SnapshotReadRequest<'_>,
1572    ) -> Result<SnapshotRead, QueryReadFailure> {
1573        snapshot_read_connection(self.connection()?, request)
1574    }
1575
1576    /// The current durable local revision. A host can skip a read when it is
1577    /// unchanged; a corrupt or unreadable marker fails typed.
1578    pub fn local_revision(&mut self) -> Result<String, QueryReadFailure> {
1579        Ok(self
1580            .snapshot_read(&SnapshotReadRequest::default())?
1581            .revision)
1582    }
1583
1584    /// One unit's persisted catch-up state (see the §7.5 contract).
1585    pub fn unit_catchup(&mut self, id: &str) -> Result<UnitCatchup, QueryReadFailure> {
1586        let read = self.snapshot_read(&SnapshotReadRequest {
1587            units: vec![id.to_owned()],
1588            ..Default::default()
1589        })?;
1590        Ok(read
1591            .units
1592            .into_iter()
1593            .next()
1594            .expect("one requested catch-up"))
1595    }
1596
1597    /// One client commit's delivery status: pending, its retained outcome, or
1598    /// an explicit unknown (never held, or pruned by retention).
1599    pub fn commit_delivery(&mut self, id: &str) -> Result<CommitDelivery, QueryReadFailure> {
1600        let read = self.snapshot_read(&SnapshotReadRequest {
1601            commit_ids: vec![id.to_owned()],
1602            ..Default::default()
1603        })?;
1604        Ok(read
1605            .deliveries
1606            .into_iter()
1607            .next()
1608            .expect("one requested delivery"))
1609    }
1610
1611    /// Hosts report an owned failure, and the first success after it, to the
1612    /// owning core through [`SyncClient::record_query_read`] (§7.6).
1613    pub fn query_snapshot(
1614        &mut self,
1615        sql: &str,
1616        params: &[Value],
1617        coverage: &[WindowCoverage],
1618    ) -> Result<QuerySnapshot, QueryReadFailure> {
1619        snapshot_connection(self.connection()?, sql, params, coverage)
1620    }
1621}
1622
1623fn validate_authority_reads(
1624    schema: &ClientSchema,
1625    declarations: &[crate::api::AuthorityReadDeclaration],
1626) -> Result<(), String> {
1627    let mut seen = BTreeSet::new();
1628    for read in declarations {
1629        let table = schema
1630            .table(&read.table)
1631            .ok_or("client.authority_read_forbidden: authority table is not declared")?;
1632        let plain = |name: &str| {
1633            table.columns.iter().enumerate().any(|(i, c)| {
1634                c.name == name
1635                    && name != "authority_version"
1636                    && !name.starts_with("_sync")
1637                    && !table.encrypted_columns.iter().any(|e| e.index == i)
1638                    && matches!(
1639                        c.ty,
1640                        ssp2::segment::ColumnType::String
1641                            | ssp2::segment::ColumnType::Integer
1642                            | ssp2::segment::ColumnType::Float
1643                            | ssp2::segment::ColumnType::Boolean
1644                            | ssp2::segment::ColumnType::Json
1645                    )
1646            })
1647        };
1648        if !seen.insert(&read.table)
1649            || read.columns.is_empty()
1650            || read.columns.iter().collect::<BTreeSet<_>>().len() != read.columns.len()
1651            || !read.columns.contains(&table.primary_key)
1652            || !read.columns.iter().all(|c| plain(c))
1653            || read.scopes.is_empty()
1654            || !read.scopes.iter().all(|(v, values)| {
1655                table
1656                    .scope_column(v)
1657                    .is_some_and(|c| read.columns.iter().any(|column| column == c) && plain(c))
1658                    && !values.is_empty()
1659                    && values.iter().all(|v| !v.is_empty() && v != "*")
1660            })
1661        {
1662            return Err("client.authority_read_forbidden: authority reads require plain columns, a primary key and nonempty plain scope selectors".into());
1663        }
1664    }
1665    Ok(())
1666}
1667
1668// §7.4.2: every open and recreation checks before any persistent writes.
1669/// A marker read that hit a recognized SQLite storage failure keeps that
1670/// classification (BUSY/LOCKED stay retryable); only an unclassified read
1671/// failure is the invalid-marker `sync.local_corrupt`.
1672fn schema_marker_read_failure(error: rusqlite::Error) -> String {
1673    let failure = QueryReadFailure::from(error);
1674    if failure.code.is_some() {
1675        failure.to_string()
1676    } else {
1677        "sync.local_corrupt: persisted local schema marker is unreadable or invalid".to_owned()
1678    }
1679}
1680
1681/// §7.1: reject an invalid configured client limit before any storage is
1682/// opened or written. Zero would wedge the push queue and an out-of-range
1683/// value would differ per target; the TS core enforces the same bounds.
1684fn validate_client_limits(limits: &crate::api::ClientLimits) -> Result<(), String> {
1685    if limits.outcome_retention_max_entries == Some(0) {
1686        return Err("sync.invalid_request: outcomeRetentionMaxEntries must be positive".to_owned());
1687    }
1688    if [
1689        limits.max_push_commits_per_request,
1690        limits.max_push_operations_per_request,
1691        limits.max_push_request_bytes,
1692    ]
1693    .into_iter()
1694    .flatten()
1695    .any(|value| value == 0 || value > u32::MAX as usize)
1696    {
1697        return Err(
1698            "sync.invalid_request: push limit must be an integer in 1..=4294967295".to_owned(),
1699        );
1700    }
1701    Ok(())
1702}
1703
1704fn read_local_schema_version(conn: &Connection, requested: i32) -> Result<Option<i32>, String> {
1705    let corrupt =
1706        || "sync.local_corrupt: persisted local schema marker is unreadable or invalid".to_owned();
1707    let table_type: Option<String> = conn
1708        .query_row(
1709            "SELECT type FROM sqlite_master WHERE name = '_syncular_meta'",
1710            [],
1711            |row| row.get(0),
1712        )
1713        .optional()
1714        .map_err(schema_marker_read_failure)?;
1715    let Some(table_type) = table_type else {
1716        return Ok(None);
1717    };
1718    if table_type != "table" {
1719        return Err(corrupt());
1720    }
1721    // §7.4.2: a duplicated marker cannot be resolved. `query_row` would take
1722    // whichever row SQLite returns first, and an older one would drive a
1723    // destructive reset over a newer replica.
1724    let mut statement = conn
1725        .prepare("SELECT value FROM _syncular_meta WHERE key = ?1")
1726        .map_err(schema_marker_read_failure)?;
1727    let mut rows = statement
1728        .query([LOCAL_SCHEMA_VERSION_KEY])
1729        .map_err(schema_marker_read_failure)?;
1730    let value: Option<String> = rows
1731        .next()
1732        .map_err(schema_marker_read_failure)?
1733        .map(|row| row.get(0))
1734        .transpose()
1735        .map_err(schema_marker_read_failure)?;
1736    if rows.next().map_err(schema_marker_read_failure)?.is_some() {
1737        return Err(corrupt());
1738    }
1739    let Some(value) = value else {
1740        let has_descriptor = conn
1741            .query_row(
1742                "SELECT 1 FROM _syncular_meta WHERE key = ?1 LIMIT 1",
1743                [crate::previous_version::LOCAL_SCHEMA_DESCRIPTOR_KEY],
1744                |_| Ok(()),
1745            )
1746            .optional()
1747            .map_err(schema_marker_read_failure)?
1748            .is_some();
1749        if has_descriptor {
1750            return Err(corrupt());
1751        }
1752        return Ok(None);
1753    };
1754    if !matches!(value.as_bytes().first(), Some(b'1'..=b'9'))
1755        || !value.bytes().all(|b| b.is_ascii_digit())
1756    {
1757        return Err(corrupt());
1758    }
1759    let version = value.parse::<i32>().map_err(|_| corrupt())?;
1760    if version > requested {
1761        return Err(
1762            "client.schema_downgrade: persisted local schema is newer than the requested schema"
1763                .to_owned(),
1764        );
1765    }
1766    Ok(Some(version))
1767}
1768
1769/// A file replica on a browser worker must recover a hot journal before any
1770/// statement reads it. `wasm_vfs` replaces the installed OPFS VFS's reserved
1771/// lock callback, so install it before the connection opens its file.
1772#[cfg(all(target_arch = "wasm32", target_os = "unknown"))]
1773fn prepare_wasm_replica(path: &str) -> Result<(), String> {
1774    if path.is_empty() || path == ":memory:" {
1775        return Ok(());
1776    }
1777    crate::wasm_vfs::disable_foreign_lock_check()
1778}
1779
1780#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))]
1781fn prepare_wasm_replica(_path: &str) -> Result<(), String> {
1782    Ok(())
1783}
1784
1785impl SyncClient {
1786    pub fn new_with_identity(
1787        client_id: Option<String>,
1788        schema_json: &Value,
1789        limits: ClientLimits,
1790    ) -> Result<Self, String> {
1791        let resolved = client_id.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
1792        let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
1793        Self::with_connection(resolved, schema_json, limits, conn)
1794    }
1795
1796    pub fn new(
1797        client_id: String,
1798        schema_json: &Value,
1799        limits: ClientLimits,
1800    ) -> Result<Self, String> {
1801        let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
1802        Self::with_connection(client_id, schema_json, limits, conn)
1803    }
1804
1805    /// Build a client backed by an on-disk SQLite database at `path` — the
1806    /// seam a native host (Tauri plugin, FFI file-DB variant) uses to persist
1807    /// across process restarts. `create_tables` runs `IF NOT EXISTS`, so
1808    /// re-opening the same file reuses the persisted rows. Keeps rusqlite out
1809    /// of the command router's dependency set (the router only holds a path).
1810    pub fn open_path(
1811        client_id: String,
1812        schema_json: &Value,
1813        limits: ClientLimits,
1814        path: &str,
1815    ) -> Result<Self, String> {
1816        validate_client_limits(&limits)?;
1817        parse_schema_json(schema_json)?;
1818        prepare_wasm_replica(path)?;
1819        let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
1820        Self::with_connection(client_id, schema_json, limits, conn)
1821    }
1822
1823    /// Open a file replica and resolve its client id. Native targets require
1824    /// WAL for the independent snapshot reader; `wasm32-unknown-unknown`
1825    /// accepts the sqlite-wasm worker journal instead.
1826    pub fn open_path_with_identity(
1827        client_id: Option<String>,
1828        schema_json: &Value,
1829        limits: ClientLimits,
1830        path: &str,
1831    ) -> Result<Self, String> {
1832        Self::open_path_with_identity_journal(
1833            client_id,
1834            schema_json,
1835            limits,
1836            path,
1837            !cfg!(all(target_arch = "wasm32", target_os = "unknown")),
1838        )
1839    }
1840
1841    fn open_path_with_identity_journal(
1842        client_id: Option<String>,
1843        schema_json: &Value,
1844        limits: ClientLimits,
1845        path: &str,
1846        require_wal: bool,
1847    ) -> Result<Self, String> {
1848        validate_client_limits(&limits)?;
1849        // §7.4.2: an invalid requested version is refused before the replica is
1850        // even created, so a rejected constructor leaves no file behind.
1851        parse_schema_json(schema_json)?;
1852        prepare_wasm_replica(path)?;
1853        let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
1854        // A file-backed native client uses an independent read connection for
1855        // latency-critical snapshots. WAL is SQLite's intended reader/writer
1856        // concurrency mode: a view read never holds a rollback-journal lock
1857        // that delays the client's next commit. The busy timeout must precede
1858        // this replica's own startup transaction; WAL is applied only after
1859        // startup succeeded, because a refused open must leave the replica's
1860        // journal mode and contents untouched (§7.4.2).
1861        conn.busy_timeout(std::time::Duration::from_millis(250))
1862            .map_err(|error| format!("configure db {path:?} busy timeout: {error}"))?;
1863        // Durability is the caller's contract here, not the VFS default: a
1864        // rollback-journal commit needs FULL before it writes the database
1865        // pages, or a killed worker can leave a half-written image behind.
1866        // WAL below inherits the same setting.
1867        conn.pragma_update(None, "synchronous", "FULL")
1868            .map_err(|error| format!("configure db {path:?} synchronous: {error}"))?;
1869        let client = Self::with_connection_identity(client_id, schema_json, limits, conn)?;
1870        // A browser worker's sqlite-wasm VFS has no WAL mode, and the wasm
1871        // host has no independent snapshot reader. The WAL requirement exists
1872        // for native file replicas, where it is the reader/writer boundary.
1873        if require_wal {
1874            let journal_mode: String = client
1875                .conn
1876                .pragma_update_and_check(None, "journal_mode", "WAL", |row| row.get(0))
1877                .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
1878            if journal_mode != "wal" {
1879                return Err(
1880                    "sync.invalid_request: file-backed client requires WAL journal mode".to_owned(),
1881                );
1882            }
1883        }
1884        Ok(client)
1885    }
1886
1887    /// Delete a file replica at `path`, including its `-journal`, `-wal`, and
1888    /// `-shm` sidecars, through the VFS that owns it. Use this after
1889    /// `client.storage_corrupt`: an OPFS SAH pool stores those files under
1890    /// opaque names, so deleting the pool directory behind SQLite bypasses the
1891    /// pool's slot accounting.
1892    ///
1893    /// Close the client and every [`FileQuerySnapshotReader`] on `path` first;
1894    /// a VFS cannot delete a file another handle still owns. A file that does
1895    /// not exist counts as deleted, so a cleanly closed replica without
1896    /// sidecars resets. The next open
1897    /// creates a fresh replica (new client id) and re-bootstraps from the
1898    /// server. A memory replica has nothing to delete and fails with
1899    /// `sync.invalid_request`.
1900    pub fn reset_replica(path: &str) -> Result<(), String> {
1901        if path.is_empty() || path == ":memory:" {
1902            return Err("sync.invalid_request: a memory replica has nothing to reset".to_owned());
1903        }
1904        // SAFETY: a null VFS name asks SQLite for its process-wide default VFS,
1905        // the same one `Connection::open(path)` above selects. `xDelete` is
1906        // that VFS's own function pointer, and `c_name` outlives the call.
1907        let vfs = unsafe { rusqlite::ffi::sqlite3_vfs_find(std::ptr::null()) };
1908        if vfs.is_null() {
1909            return Err("client.storage_io: no SQLite VFS is registered".to_owned());
1910        }
1911        let delete = unsafe { (*vfs).xDelete }
1912            .ok_or_else(|| "client.storage_io: the SQLite VFS cannot delete files".to_owned())?;
1913        for suffix in ["-wal", "-shm", "-journal", ""] {
1914            let file_name = format!("{path}{suffix}");
1915            let c_name = std::ffi::CString::new(file_name.as_str()).map_err(|_| {
1916                format!("sync.invalid_request: replica path {file_name:?} contains a NUL byte")
1917            })?;
1918            // SAFETY: `vfs` and its `xDelete` pointer come from the live
1919            // SQLite library; `c_name` is a valid NUL-terminated path.
1920            let code = unsafe { delete(vfs, c_name.as_ptr(), 1) };
1921            // A cleanly closed WAL replica has no sidecars; a missing file is
1922            // already deleted.
1923            if code != rusqlite::ffi::SQLITE_OK && code != rusqlite::ffi::SQLITE_IOERR_DELETE_NOENT
1924            {
1925                return Err(format!(
1926                    "client.storage_io: deleting replica file {file_name:?} failed with SQLite code {code}"
1927                ));
1928            }
1929        }
1930        Ok(())
1931    }
1932
1933    #[must_use]
1934    pub fn client_id(&self) -> &str {
1935        &self.client_id
1936    }
1937
1938    /// Rotate the caller's client id (§1.5) and return the new one. Use it
1939    /// after a `sync.invalid_client_id` refusal (`recommendedAction:
1940    /// resetClientId`): the next round carries the new id. A live realtime
1941    /// socket is closed so the next connection registers with the new id.
1942    /// The replica and its outbox stay in place; wipe them yourself when the
1943    /// stored data belongs to the account that signed out.
1944    pub fn reset_client_id(&mut self, transport: &mut dyn Transport) -> Result<String, String> {
1945        self.disconnect_realtime(transport);
1946        let client_id = uuid::Uuid::new_v4().to_string();
1947        self.set_meta(CLIENT_ID_KEY, &client_id)?;
1948        self.client_id.clone_from(&client_id);
1949        self.round_failure = None;
1950        Ok(client_id)
1951    }
1952
1953    /// Build a client over a caller-supplied rusqlite connection — the seam a
1954    /// native host (Tauri plugin, FFI file-DB variant) uses to back the core
1955    /// with an on-disk database (`Connection::open(path)`) rather than the
1956    /// default `:memory:`. The connection MUST be fresh (no pre-existing
1957    /// syncular tables); `create_tables` runs `IF NOT EXISTS`, so re-opening
1958    /// the same file across process restarts reuses the persisted rows.
1959    pub fn with_connection(
1960        client_id: String,
1961        schema_json: &Value,
1962        limits: ClientLimits,
1963        conn: Connection,
1964    ) -> Result<Self, String> {
1965        Self::with_connection_identity(Some(client_id), schema_json, limits, conn)
1966    }
1967
1968    /// [`Self::with_connection`] for `open_path_with_identity`: no requested id
1969    /// means a durable replica's persisted `clientId` wins over a fresh one.
1970    fn with_connection_identity(
1971        client_id: Option<String>,
1972        schema_json: &Value,
1973        limits: ClientLimits,
1974        conn: Connection,
1975    ) -> Result<Self, String> {
1976        validate_client_limits(&limits)?;
1977        let schema = parse_schema_json(schema_json)?;
1978        validate_authority_reads(&schema, &limits.authority_reads)?;
1979        // RFC 0005 D8: resolve the previous-version config before the opening
1980        // reset runs — the capture happens inside it. Bad bounds fail loud.
1981        let previous_version = match limits.previous_version_context {
1982            Some(config) => {
1983                config.validate()?;
1984                Some(config)
1985            }
1986            None => None,
1987        };
1988        let mut client = SyncClient {
1989            #[cfg(feature = "bench-internals")]
1990            benchmark_phases: Recorder::default(),
1991            conn,
1992            schema,
1993            client_id: client_id
1994                .clone()
1995                .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()),
1996            limits,
1997            outbox: Shared::default(),
1998            failed_commits: Shared::default(),
1999            retain_failed_commits: false,
2000            conflicts: Shared::default(),
2001            rejections: Shared::default(),
2002            schema_floor: None,
2003            lease_state: None,
2004            stopped: false,
2005            upgrading: false,
2006            sync_needed: false,
2007            realtime_policy: RealtimePolicy::Optional,
2008            realtime_state: RealtimeState::Disconnected,
2009            realtime_reason_code: None,
2010            realtime_retry_delay_ms: None,
2011            presence: HashMap::new(),
2012            now_ms: None,
2013            encryption: crate::values::EncryptionConfig::default(),
2014            security_preflight: false,
2015            security_epoch: 0,
2016            transport_enabled: true,
2017            transport_epoch: 0,
2018            registration_epoch: 0,
2019            previous_version,
2020            insert_sql: RefCell::new(HashMap::new()),
2021            overlay_dirty: OverlayDirty::clean(),
2022            storage_failure: RefCell::new(None),
2023            #[cfg(test)]
2024            overlay_rebuild_count: Cell::new(0),
2025            #[cfg(test)]
2026            acknowledged_replay_count: Cell::new(0),
2027            #[cfg(test)]
2028            outcome_prune_count: Cell::new(0),
2029            #[cfg(test)]
2030            image_chunk_ms: Cell::new(None),
2031            progress: ProgressObserver::default(),
2032            change_queue: VecDeque::new(),
2033            sync_intent_queue: VecDeque::new(),
2034            retry_delay_ms: 250,
2035            round_retry_delay_ms: None,
2036            round_failure: None,
2037            active_round: None,
2038            last_round: None,
2039            last_change: None,
2040            query_failures: Vec::new(),
2041            coalesced: None,
2042            coalesced_failure: None,
2043        };
2044        // The row write path leans on the prepared-statement cache (two
2045        // insert statements per synced table, plus the bookkeeping
2046        // statements); size it so a multi-table schema never thrashes.
2047        client
2048            .conn
2049            .set_prepared_statement_cache_capacity(64.max(client.schema.tables.len() * 4));
2050        // §7.4.2: one write transaction (SQLite `BEGIN IMMEDIATE`, a reserved
2051        // writer) covers the marker read, the identity check, and every
2052        // bookkeeping/schema write, so a concurrent open that upgraded the
2053        // replica cannot interleave between the guard and the writes, and a
2054        // refusal leaves the replica unchanged.
2055        client
2056            .conn
2057            .execute_batch("BEGIN IMMEDIATE")
2058            .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
2059        let startup = (|| -> Result<(), String> {
2060            // The marker guard runs inside the write transaction, so the version
2061            // it validates is the version every write below commits under.
2062            let marker = read_local_schema_version(&client.conn, client.schema.version)?;
2063            // New app indexes may reference columns that only exist after a
2064            // version-bump reset.
2065            client.create_bookkeeping_tables()?;
2066            match client.get_meta(CLIENT_ID_KEY)? {
2067                Some(existing) => match client_id {
2068                    Some(requested) if existing != requested => {
2069                        return Err(format!(
2070                            "client.identity_mismatch: this database belongs to {existing:?}; refusing to rebind it to {requested:?}"
2071                        ))
2072                    }
2073                    _ => client.client_id = existing,
2074                },
2075                None => client.set_meta(CLIENT_ID_KEY, &client.client_id)?,
2076            }
2077            client.restore_persisted_state()?;
2078            client.failed_commits = client.load_failed_commits()?.into();
2079            match marker {
2080                None => {
2081                    client.create_synced_tables()?;
2082                    client.persist_schema_version()?;
2083                    client.save_unit_scope_schema()?;
2084                }
2085                Some(version) if version == client.schema.version => {
2086                    client.create_synced_tables()?;
2087                    // RFC 0005 D1: the same-version open backfills the descriptor
2088                    // for a database first opened by an unaware binary, so the
2089                    // NEXT bump can capture without needing a schema bump first.
2090                    set_local_schema_descriptor(&client.conn, &client.schema)
2091                        .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
2092                    client.save_unit_scope_schema()?;
2093                }
2094                Some(_) => client.run_schema_reset()?,
2095            }
2096            // RFC 0005 D9: the reset sweep cannot see a container whose capture
2097            // committed in its OWN file while the replica's savepoint never
2098            // released — a crash at that seam leaves a container beside a matching
2099            // OLD marker, and the next open at that version runs no reset. Discard
2100            // it before anything can read it. No-op when no container exists.
2101            client.reconcile_previous_version_at_boot()?;
2102            client.clear_satisfied_persisted_schema_floor()?;
2103            client.prune_unknown_units(false)?;
2104            // §7.1/§7.4.4: recover intrinsically codec-invalid persisted values at
2105            // the full startup/reset reconciliation boundary, inside this open
2106            // transaction and after the schema and identity guards, so a commit
2107            // an earlier version accepted is dropped with durable rejection
2108            // evidence before any replay. A table the current schema removed
2109            // keeps the §7.4.4 send-time classification, and an unresolved
2110            // encryption key stays with the send seam because hosts configure
2111            // keys after open.
2112            client.drop_unencodable_outbox(false, None)?;
2113            if marker == Some(client.schema.version)
2114                && (!client.outbox.is_empty()
2115                    || !client.failed_commits.is_empty()
2116                    || client.has_acknowledged_rows()?)
2117            {
2118                // Reconstruct the visible optimistic overlay from the durable
2119                // base plus outbox instead of trusting a process-interrupted
2120                // mirror.
2121                client.overlay_dirty.set(true);
2122                client.rebuild_overlay()?;
2123            }
2124            // Every persisted active unit needs one catch-up pull on
2125            // open: realtime only covers changes after connection, while an
2126            // idempotent setWindow correctly creates no fresh command effect.
2127            // Pending outbox work has the same restart requirement. The core
2128            // owns this intent so native hosts never poll or require an
2129            // application-issued sync().
2130            client.enqueue_startup_sync_if_needed()?;
2131            Ok(())
2132        })();
2133        match startup {
2134            Ok(()) => match client.conn.execute_batch("COMMIT") {
2135                // §4.8: an eviction a crash interrupted finishes before any
2136                // read, with or without a transport; a replica reopened in
2137                // preflight finishes it at activation.
2138                Ok(()) => {
2139                    if !client.security_preflight {
2140                        client.drain_pending_evictions()?;
2141                    }
2142                    Ok(client)
2143                }
2144                Err(error) => {
2145                    let failure = Self::sqlite_failure(&client.storage_failure, error);
2146                    if let Err(rollback) = client.conn.execute_batch("ROLLBACK") {
2147                        // Keep a failed rollback's typed storage classification.
2148                        let _ = Self::sqlite_failure(&client.storage_failure, rollback);
2149                    }
2150                    Err(failure)
2151                }
2152            },
2153            Err(error) => {
2154                if let Err(rollback) = client.conn.execute_batch("ROLLBACK") {
2155                    // Keep a failed rollback's typed storage classification.
2156                    let _ = Self::sqlite_failure(&client.storage_failure, rollback);
2157                }
2158                Err(error)
2159            }
2160        }
2161    }
2162
2163    /// Pin the client clock (epoch ms) — the §5.4 expiry check runs
2164    /// against this instead of system time (conformance virtual clock).
2165    pub fn set_now_ms(&mut self, now_ms: i64) {
2166        self.now_ms = Some(now_ms);
2167    }
2168
2169    /// §5.11: install the client-side encryption keys (`keyId → key bytes`).
2170    /// The command router parses these from the `create` command's
2171    /// `encryption` config (keys as `{$bytes: hex}`).
2172    pub fn set_encryption(&mut self, encryption: crate::values::EncryptionConfig) {
2173        self.encryption = encryption;
2174    }
2175
2176    #[must_use]
2177    pub fn security_lifecycle(&self) -> &'static str {
2178        if self.security_preflight {
2179            "preflight"
2180        } else {
2181            "active"
2182        }
2183    }
2184
2185    #[must_use]
2186    pub fn security_preflight(&self) -> bool {
2187        self.security_preflight
2188    }
2189
2190    /// Host-owned, non-persisted network gate. Local operations remain available.
2191    pub fn set_transport_enabled(&mut self, transport: &mut dyn Transport, enabled: bool) {
2192        if self.transport_enabled != enabled {
2193            self.transport_enabled = enabled;
2194            self.transport_epoch = self.transport_epoch.wrapping_add(1);
2195            self.sync_intent_queue.clear();
2196            self.sync_intent_queue
2197                .push_back(if enabled && !self.security_preflight {
2198                    SyncIntent::Interactive
2199                } else {
2200                    SyncIntent::None
2201                });
2202        }
2203        // A captured reply keeps its normal apply/revocation semantics.
2204        if !enabled && self.active_round.is_none() {
2205            self.disconnect_realtime(transport);
2206        }
2207    }
2208
2209    pub fn transport_enabled(&self) -> bool {
2210        self.transport_enabled
2211    }
2212
2213    /// Quarantine this replica: enter the fail-closed gate, release all
2214    /// core-owned key material, and record the gate in the database so
2215    /// reopening re-enters preflight until `activate_security` clears it.
2216    ///
2217    /// Use [`SyncClient::seal_security_on_teardown`] for the shutdown barrier,
2218    /// which must NOT leave that durable mark.
2219    pub fn begin_security_preflight(&mut self) -> Result<(), String> {
2220        let entering = !self.security_preflight;
2221        self.seal_security_on_teardown();
2222        // Persist the quarantine so it survives handle teardown and restart:
2223        // reopening this replica re-enters preflight until activation clears it.
2224        self.set_meta(SECURITY_PREFLIGHT_PENDING_KEY, "1")?;
2225        if entering {
2226            self.emit_status_batch()?;
2227        }
2228        Ok(())
2229    }
2230
2231    /// One change batch carrying the status snapshot (§5.11 lifecycle).
2232    fn emit_status_batch(&mut self) -> Result<(), String> {
2233        self.begin_observation("syncular_status")?;
2234        let batch = ChangeAccumulator {
2235            status: true,
2236            ..ChangeAccumulator::default()
2237        };
2238        self.finish_observation("syncular_status", batch)
2239            .inspect_err(|_| self.rollback_observation("syncular_status"))
2240    }
2241
2242    /// Release core-owned key material as a host tears the client down,
2243    /// WITHOUT recording a quarantine.
2244    ///
2245    /// Shutting an activated client down is not a quarantine event. Persisting
2246    /// the gate here marks every cleanly closed replica as pending, and the
2247    /// next plain `create` is then refused permanently: the reopen path
2248    /// restores the flag from the marker and the create guard rejects it. The
2249    /// in-memory flag still closes the gate for anything still holding this
2250    /// instance, which is all a teardown barrier needs.
2251    pub fn seal_security_on_teardown(&mut self) {
2252        self.cancel_sync_round();
2253        self.security_preflight = true;
2254        self.security_epoch = self.security_epoch.wrapping_add(1);
2255        self.encryption = crate::values::EncryptionConfig::default();
2256        self.sync_intent_queue.clear();
2257    }
2258
2259    /// Install the post-authentication keyring and release the host loop.
2260    pub fn activate_security(
2261        &mut self,
2262        encryption: crate::values::EncryptionConfig,
2263    ) -> Result<(), String> {
2264        if !self.security_preflight {
2265            return Err(
2266                "sync.invalid_request: activateSecurity requires security preflight".to_owned(),
2267            );
2268        }
2269        // §4.8: an interrupted eviction finishes before protected reads open.
2270        self.drain_pending_evictions()?;
2271        self.enqueue_startup_sync_if_needed()?;
2272        self.delete_meta(SECURITY_PREFLIGHT_PENDING_KEY)?;
2273        self.encryption = encryption;
2274        self.security_preflight = false;
2275        // §5.11: rows sealed under a key this keyring now holds open at once.
2276        self.reveal_sealed_rows()
2277            .map_err(|error| format!("{}: {}", error.code, error.message))?;
2278        self.emit_status_batch()
2279    }
2280
2281    fn clock_now_ms(&self) -> i64 {
2282        self.now_ms.unwrap_or_else(crate::wall_clock_ms)
2283    }
2284
2285    fn create_bookkeeping_tables(&self) -> Result<(), String> {
2286        let exists = |name: &str| {
2287            self.conn
2288                .query_row(
2289                    "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?1)",
2290                    [name],
2291                    |row| row.get::<_, bool>(0),
2292                )
2293                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2294        };
2295        let outbox_existed = exists("_syncular_outbox")?;
2296        // Durable client bookkeeping (outbox + units + meta).
2297        self.conn
2298            .execute_batch(
2299                "CREATE TABLE IF NOT EXISTS _syncular_outbox (
2300                   seq INTEGER PRIMARY KEY AUTOINCREMENT,
2301                   commit_id TEXT NOT NULL UNIQUE, ops_json TEXT NOT NULL);
2302                 CREATE TABLE IF NOT EXISTS _syncular_row_deliveries (
2303                   tbl TEXT NOT NULL, id TEXT NOT NULL, commit_seq INTEGER NOT NULL, PRIMARY KEY(tbl,id));
2304                 CREATE TABLE IF NOT EXISTS _syncular_acked_rows (
2305                   commit_id TEXT NOT NULL, idx INTEGER NOT NULL, tbl TEXT NOT NULL, id TEXT NOT NULL,
2306                   commit_seq INTEGER NOT NULL, op_json TEXT NOT NULL, intent_json TEXT,
2307                   PRIMARY KEY(commit_id, idx));
2308                 CREATE INDEX IF NOT EXISTS _syncular_acked_rows_row
2309                   ON _syncular_acked_rows(tbl, id);
2310                 CREATE TABLE IF NOT EXISTS _syncular_failed_commits (
2311                   seq INTEGER PRIMARY KEY AUTOINCREMENT, client_commit_id TEXT NOT NULL UNIQUE,
2312                   operations_json TEXT NOT NULL, initial_json TEXT NOT NULL);
2313                 CREATE TABLE IF NOT EXISTS _syncular_commit_outcomes (
2314                   seq INTEGER PRIMARY KEY AUTOINCREMENT,
2315                   client_commit_id TEXT NOT NULL UNIQUE,
2316                   status TEXT NOT NULL CHECK(status IN ('applied', 'cached', 'conflict', 'rejected')),
2317                   recorded_at_ms INTEGER NOT NULL,
2318                   results_json TEXT NOT NULL,
2319                   operations_json TEXT,
2320                   resolution TEXT NOT NULL DEFAULT 'active'
2321                     CHECK(resolution IN ('active', 'resolved_keep_server', 'superseded', 'dismissed')),
2322                   resolved_at_ms INTEGER,
2323                   replacement_client_commit_id TEXT);
2324                 CREATE INDEX IF NOT EXISTS _syncular_commit_outcomes_resolution_seq
2325                   ON _syncular_commit_outcomes(resolution, seq);
2326                 CREATE TABLE IF NOT EXISTS _syncular_units (
2327                   id TEXT PRIMARY KEY, scopes TEXT NOT NULL,
2328                   explicit INTEGER NOT NULL DEFAULT 0,
2329                   status TEXT NOT NULL DEFAULT 'active', reason_code TEXT,
2330                   cursor INTEGER DEFAULT -1, bootstrap_state TEXT,
2331                   seeded TEXT NOT NULL DEFAULT '[]', echo TEXT);
2332                 CREATE TABLE IF NOT EXISTS _syncular_meta (
2333                   key TEXT PRIMARY KEY, value TEXT NOT NULL);
2334                 CREATE TABLE IF NOT EXISTS _syncular_windows (
2335                   base TEXT NOT NULL, unit TEXT NOT NULL, sub_id TEXT NOT NULL,
2336                   PRIMARY KEY (base, unit));
2337                 CREATE INDEX IF NOT EXISTS _syncular_windows_unit ON _syncular_windows(sub_id);
2338                 CREATE TABLE IF NOT EXISTS _syncular_pending_evictions (
2339                   unit_id TEXT NOT NULL, tbl TEXT NOT NULL,
2340                   effective_scopes TEXT NOT NULL, PRIMARY KEY (unit_id, tbl));
2341                 CREATE TABLE IF NOT EXISTS _syncular_blob_commit_refs(
2342                   commit_id TEXT NOT NULL, blob_id TEXT NOT NULL,
2343                   PRIMARY KEY(commit_id, blob_id));
2344                 CREATE INDEX IF NOT EXISTS _syncular_blob_commit_refs_body
2345                   ON _syncular_blob_commit_refs(blob_id);",
2346            )
2347            .map_err(|e| e.to_string())?;
2348        // Migrate an outcome journal created before failed aggregate
2349        // envelopes were retained. Historical rows intentionally stay NULL.
2350        let _ = self
2351            .conn
2352            .execute_batch("ALTER TABLE _syncular_commit_outcomes ADD COLUMN operations_json TEXT");
2353        if self.get_meta(LOCAL_REVISION_KEY)?.is_none() {
2354            self.set_meta(LOCAL_REVISION_KEY, "0")?;
2355        }
2356        // §7.1: `outboxId` names the outbox sequence. A new outbox table starts
2357        // a new sequence, so it replaces the id in the same transaction.
2358        if !outbox_existed || self.get_meta(OUTBOX_ID_KEY)?.is_none() {
2359            self.set_meta(OUTBOX_ID_KEY, &uuid::Uuid::new_v4().to_string())?;
2360        }
2361        // §9: a replica written under wire version 3 keeps its outbox, journal,
2362        // rows and pending evictions and registers its units fresh. Its
2363        // subscriptions, cursors, echoes, resume tokens and window entries go;
2364        // units left from an earlier version 4 life restart.
2365        if exists("_syncular_window_pending_evict")? {
2366            let invalid: bool = self
2367                .conn
2368                .query_row(
2369                    "SELECT EXISTS(SELECT 1 FROM _syncular_window_pending_evict
2370                       WHERE NOT json_valid(effective_scopes)
2371                          OR json_type(effective_scopes) <> 'object'
2372                          OR EXISTS(SELECT 1 FROM json_each(effective_scopes)
2373                               WHERE type <> 'array'
2374                                  OR EXISTS(SELECT 1 FROM json_each(value) AS v
2375                                       WHERE v.type <> 'text')))",
2376                    [],
2377                    |row| row.get(0),
2378                )
2379                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
2380            if invalid {
2381                return Err(
2382                    "sync.local_corrupt: a version 3 pending eviction cannot be upgraded"
2383                        .to_owned(),
2384                );
2385            }
2386            self.conn
2387                .execute_batch(
2388                    "INSERT OR REPLACE INTO _syncular_pending_evictions
2389                       SELECT sub_id, tbl, effective_scopes FROM _syncular_window_pending_evict;
2390                     DROP TABLE _syncular_window_pending_evict;
2391                     DROP TABLE IF EXISTS _syncular_subscriptions;
2392                     DELETE FROM _syncular_windows;",
2393                )
2394                .and_then(|()| reset_units(&self.conn))
2395                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
2396        }
2397        // §5.9.7 blob cache. Mutable upload state stays off the body row so
2398        // completing a large upload does not rewrite its BLOB record.
2399        if self.schema_has_blobs() {
2400            if exists("_syncular_blobs")? {
2401                let mut statement = self
2402                    .conn
2403                    .prepare("PRAGMA table_info(_syncular_blobs)")
2404                    .map_err(|e| e.to_string())?;
2405                let mut columns = statement
2406                    .query_map([], |row| row.get::<_, String>(1))
2407                    .map_err(|e| e.to_string())?
2408                    .collect::<Result<Vec<_>, _>>()
2409                    .map_err(|e| e.to_string())?;
2410                columns.sort();
2411                let mut expected = vec![
2412                    "blob_id",
2413                    "byte_length",
2414                    "bytes",
2415                    "created_at_ms",
2416                    "media_type",
2417                ];
2418                expected.sort_unstable();
2419                if columns != expected {
2420                    return Err(
2421                        "sync.schema_mismatch: Local blob schema is incompatible".to_owned()
2422                    );
2423                }
2424            } else {
2425                self.conn
2426                    .execute_batch(
2427                        "CREATE TABLE _syncular_blobs(
2428                           blob_id TEXT PRIMARY KEY,
2429                           bytes BLOB NOT NULL,
2430                           byte_length INTEGER NOT NULL,
2431                           media_type TEXT,
2432                           created_at_ms INTEGER NOT NULL);",
2433                    )
2434                    .map_err(|e| e.to_string())?;
2435            }
2436            self.conn
2437                .execute_batch(
2438                    "CREATE TABLE IF NOT EXISTS _syncular_blob_uploads(
2439                       blob_id TEXT PRIMARY KEY,
2440                       media_type TEXT,
2441                       created_at_ms INTEGER NOT NULL);",
2442                )
2443                .map_err(|e| e.to_string())?;
2444        }
2445        Ok(())
2446    }
2447
2448    /// True iff any synced table declares a `blob_ref` column (§5.9).
2449    fn schema_has_blobs(&self) -> bool {
2450        self.schema
2451            .tables
2452            .iter()
2453            .any(|t| t.columns.iter().any(|c| c.ty == ColumnType::BlobRef))
2454    }
2455
2456    // -- meta (§7.4.1 marker, bookkeeping) ------------------------------------
2457
2458    fn get_meta(&self, key: &str) -> Result<Option<String>, String> {
2459        meta_get(&self.conn, key)
2460            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2461    }
2462
2463    // The caller holds the startup/reset transaction. Either both schema
2464    // records persist or the enclosing operation fails and rolls back.
2465    fn persist_schema_version(&self) -> Result<(), String> {
2466        self.conn
2467            .execute(
2468                "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
2469                rusqlite::params![LOCAL_SCHEMA_VERSION_KEY, self.schema.version.to_string()],
2470            )
2471            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
2472        set_local_schema_descriptor(&self.conn, &self.schema)
2473            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2474    }
2475
2476    pub(crate) fn set_meta(&self, key: &str, value: &str) -> Result<(), String> {
2477        meta_set(&self.conn, key, value)
2478            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2479    }
2480
2481    fn delete_meta(&self, key: &str) -> Result<(), String> {
2482        meta_delete(&self.conn, key)
2483            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2484    }
2485
2486    /// §7.1: the durable outbox in FIFO order, in schema-agnostic local form.
2487    fn load_outbox(&self) -> Result<Vec<Arc<OutboxCommit>>, String> {
2488        let mut stmt = self
2489            .conn
2490            .prepare("SELECT commit_id, ops_json FROM _syncular_outbox ORDER BY seq ASC")
2491            .map_err(|error| error.to_string())?;
2492        let rows = stmt
2493            .query_map([], |row| {
2494                Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
2495            })
2496            .map_err(|error| error.to_string())?;
2497        let mut commits = Vec::new();
2498        for row in rows {
2499            let (client_commit_id, raw) = row.map_err(|error| error.to_string())?;
2500            let entries: Vec<Value> = serde_json::from_str(&raw).map_err(|error| {
2501                format!("invalid persisted outbox {client_commit_id:?}: {error}")
2502            })?;
2503            let mut ops = Vec::with_capacity(entries.len());
2504            for entry in entries {
2505                let op = entry.get("op").and_then(Value::as_str).unwrap_or("delete");
2506                ops.push(OutboxOp {
2507                    upsert: op == "upsert",
2508                    table: entry
2509                        .get("table")
2510                        .and_then(Value::as_str)
2511                        .ok_or_else(|| "persisted outbox operation missing table".to_owned())?
2512                        .to_owned(),
2513                    row_id: entry
2514                        .get("rowId")
2515                        .and_then(Value::as_str)
2516                        .ok_or_else(|| "persisted outbox operation missing rowId".to_owned())?
2517                        .to_owned(),
2518                    base_version: entry.get("baseVersion").and_then(Value::as_i64),
2519                    // A legacy entry's `changedFields` key is ignored:
2520                    // the values map is the presence set (§6.1).
2521                    values: entry.get("values").and_then(Value::as_object).cloned(),
2522                    insert: entry.get("insert") == Some(&Value::Bool(true)),
2523                });
2524            }
2525            commits.push(Arc::new(OutboxCommit {
2526                client_commit_id,
2527                ops,
2528            }));
2529        }
2530        Ok(commits)
2531    }
2532
2533    fn restore_persisted_state(&mut self) -> Result<(), String> {
2534        self.outbox = self.load_outbox()?.into();
2535
2536        self.prune_commit_outcomes()?;
2537        let active = self.commit_outcomes(CommitOutcomeQuery {
2538            active_only: true,
2539            ..CommitOutcomeQuery::default()
2540        })?;
2541        self.conflicts = active
2542            .iter()
2543            .flat_map(|outcome| outcome.results.iter())
2544            .filter_map(|result| match result {
2545                CommitOperationOutcome::Conflict { conflict } => Some(conflict.clone()),
2546                _ => None,
2547            })
2548            .collect::<Vec<_>>()
2549            .into();
2550        self.rejections = active
2551            .iter()
2552            .flat_map(|outcome| outcome.results.iter())
2553            .filter_map(|result| match result {
2554                CommitOperationOutcome::Error { rejection } => Some(rejection.clone()),
2555                _ => None,
2556            })
2557            .collect::<Vec<_>>()
2558            .into();
2559
2560        self.lease_state = self
2561            .get_meta(LEASE_STATE_KEY)?
2562            .map(|raw| serde_json::from_str(&raw))
2563            .transpose()
2564            .map_err(|error| format!("invalid persisted lease state: {error}"))?;
2565        self.schema_floor = self
2566            .get_meta(SCHEMA_FLOOR_KEY)?
2567            .map(|raw| serde_json::from_str(&raw))
2568            .transpose()
2569            .map_err(|error| format!("invalid persisted schema floor: {error}"))?;
2570        self.stopped = self.schema_floor.is_some();
2571        // A replica left in unactivated preflight reopens gated: the quarantine
2572        // decision persists with the data across handle teardown and restart.
2573        if self.get_meta(SECURITY_PREFLIGHT_PENDING_KEY)?.as_deref() == Some("1") {
2574            self.security_preflight = true;
2575        }
2576        Ok(())
2577    }
2578
2579    fn save_unit_scope_schema(&self) -> Result<(), String> {
2580        let declarations: BTreeMap<_, BTreeMap<_, _>> = self
2581            .schema
2582            .tables
2583            .iter()
2584            .map(|table| {
2585                (
2586                    table.name.clone(),
2587                    table
2588                        .scope_variables
2589                        .iter()
2590                        .map(|scope| {
2591                            (
2592                                scope.variable.clone(),
2593                                [scope.prefix.clone(), scope.column.clone()],
2594                            )
2595                        })
2596                        .collect(),
2597                )
2598            })
2599            .collect();
2600        self.set_meta(
2601            "subscriptionScopeSchema",
2602            &serde_json::to_string(&declarations).expect("scope schema serialization"),
2603        )
2604    }
2605
2606    /// Drop units that cover no table of the running schema, with their window
2607    /// registry rows and pending evictions; never translate scopes. At a bump
2608    /// (§7.4.3) a unit also needs every variable unchanged in prefix and column
2609    /// on every table it covered before.
2610    fn prune_unknown_units(&mut self, bump: bool) -> Result<(), String> {
2611        let previous: Option<BTreeMap<String, BTreeMap<String, [String; 2]>>> = if bump {
2612            self.get_meta("subscriptionScopeSchema")?
2613                .map(|raw| serde_json::from_str(&raw))
2614                .transpose()
2615                .map_err(|_| "sync.local_corrupt: invalid persisted scope schema".to_owned())?
2616        } else {
2617            None
2618        };
2619        // A table keeps the variables' meaning when each has the same prefix
2620        // and column as before. An old binary stored no declaration evidence;
2621        // matching names cannot prove unchanged meaning, so the app registers
2622        // again.
2623        let unchanged = |name: &str, variables: &[&String]| {
2624            let (Some(old), Some(table)) = (
2625                previous.as_ref().and_then(|previous| previous.get(name)),
2626                self.schema.table(name),
2627            ) else {
2628                return false;
2629            };
2630            variables.iter().all(|variable| {
2631                table
2632                    .scope_variables
2633                    .iter()
2634                    .find(|scope| &scope.variable == *variable)
2635                    .is_some_and(|scope| {
2636                        old.get(*variable) == Some(&[scope.prefix.clone(), scope.column.clone()])
2637                    })
2638            })
2639        };
2640        let compatible = |scopes: &UnitScopes| {
2641            if !self.schema.tables.iter().any(|table| covers(table, scopes)) {
2642                return false;
2643            }
2644            if !bump {
2645                return true;
2646            }
2647            let Some(previous) = &previous else {
2648                return false;
2649            };
2650            previous.iter().all(|(name, old)| {
2651                !scopes.keys().all(|variable| old.contains_key(variable))
2652                    || unchanged(name, &scopes.keys().collect::<Vec<_>>())
2653            })
2654        };
2655        let mut stale: BTreeSet<String> = self
2656            .stored(load_units(&self.conn))?
2657            .into_iter()
2658            .filter(|unit| !compatible(&unit.scopes))
2659            .map(|unit| unit.id)
2660            .collect();
2661        // A shrunk window's deferred eviction outlives its unit row and keeps
2662        // its per-table scopes. A table the schema no longer has cannot be
2663        // evicted, and at a bump a table whose scope declarations changed must
2664        // not be evicted by the outdated rule.
2665        for (id, table, effective) in self.load_pending_evictions()? {
2666            if self.schema.table(&table).is_none()
2667                || (bump
2668                    && !unchanged(
2669                        &table,
2670                        &effective
2671                            .iter()
2672                            .map(|(variable, _)| variable)
2673                            .collect::<Vec<_>>(),
2674                    ))
2675            {
2676                stale.insert(id);
2677            }
2678        }
2679        for id in &stale {
2680            for sql in [
2681                "DELETE FROM _syncular_windows WHERE sub_id = ?1",
2682                "DELETE FROM _syncular_pending_evictions WHERE unit_id = ?1",
2683                "DELETE FROM _syncular_units WHERE id = ?1",
2684            ] {
2685                self.conn
2686                    .execute(sql, rusqlite::params![id])
2687                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
2688            }
2689        }
2690        Ok(())
2691    }
2692
2693    pub fn local_revision(&self) -> Result<u64, String> {
2694        Ok(self
2695            .get_meta(LOCAL_REVISION_KEY)?
2696            .and_then(|value| value.parse().ok())
2697            .unwrap_or(0))
2698    }
2699
2700    /// §7.5: resolve a snapshot read on the mutable owner connection through
2701    /// the same primitive the read-only sidecar uses, so the command surface
2702    /// and the file sidecar return one revision, rows, coverage, catch-up, and
2703    /// delivery shape. The structured [`QueryReadFailure`] is preserved; an
2704    /// `owner` records its failure (and its next success) in diagnostics.
2705    pub fn snapshot_read(
2706        &mut self,
2707        request: &SnapshotReadRequest<'_>,
2708        owner: Option<QueryOwner<'_>>,
2709    ) -> Result<SnapshotRead, QueryReadFailure> {
2710        let result = snapshot_read_connection(&self.conn, request);
2711        if let Some(owner) = owner {
2712            self.record_query_read(owner, result.as_ref().err());
2713        }
2714        result
2715    }
2716
2717    #[must_use]
2718    pub fn status_snapshot(&self) -> SyncStatusSnapshot {
2719        SyncStatusSnapshot {
2720            current_schema_version: self.schema.version,
2721            outbox: self.outbox.len(),
2722            upgrading: self.upgrading,
2723            lease_state: self.lease_state.clone(),
2724            schema_floor: self.schema_floor.clone(),
2725            sync_needed: self.sync_needed,
2726            previous_version_context: self.previous_version_status(),
2727            security_lifecycle: self.security_lifecycle(),
2728        }
2729    }
2730
2731    /// RFC 0005 A2: container presence for a host that must act on it (Diego's
2732    /// rollback precondition), read without creating the file.
2733    fn previous_version_status(&self) -> PreviousVersionStatus {
2734        let Some(replica_path) = self.previous_version_replica_path() else {
2735            return PreviousVersionStatus {
2736                present: false,
2737                created_at_ms: None,
2738            };
2739        };
2740        match crate::previous_version::read_previous_version_record(&replica_path) {
2741            Ok(Some(record)) => PreviousVersionStatus {
2742                present: true,
2743                created_at_ms: Some(record.created_at_ms),
2744            },
2745            _ => PreviousVersionStatus {
2746                present: std::path::Path::new(
2747                    &crate::previous_version::previous_version_container_path(&replica_path),
2748                )
2749                .exists(),
2750                created_at_ms: None,
2751            },
2752        }
2753    }
2754
2755    /// RFC 0005 D7: read captured rows from the previous local schema. `state`
2756    /// is always `previousVersion`; the container never makes
2757    /// `query_snapshot()` coverage complete. Lease stop/expiry, scope
2758    /// revocation, coverage completion and the TTL DISCARD the container and
2759    /// close the read with their own reason.
2760    pub fn previous_version_snapshot(
2761        &self,
2762        spec: &PreviousVersionReadSpec,
2763    ) -> Result<PreviousVersionSnapshot, String> {
2764        let now_ms = self.clock_now_ms();
2765        crate::previous_version::previous_version_snapshot(
2766            &self.conn,
2767            self.previous_version_replica_path().as_deref(),
2768            self.previous_version.as_ref(),
2769            PreviousVersionLifecycle {
2770                lease_inactive: self.previous_version_lease_inactive(now_ms),
2771                scope_revoked: self
2772                    .stored(load_units(&self.conn))?
2773                    .iter()
2774                    .any(|unit| unit.status == UnitStatus::Revoked),
2775                coverage_complete: self.previous_version_coverage_complete()?,
2776            },
2777            now_ms,
2778            self.schema.version,
2779            spec,
2780        )
2781    }
2782
2783    /// RFC 0005 D6: the advisory pre-reset compatibility audit.
2784    pub fn previous_version_audit(
2785        &self,
2786    ) -> Result<Option<crate::previous_version::PreviousVersionAudit>, String> {
2787        crate::previous_version::stored_previous_version_audit(&self.conn)
2788            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2789    }
2790
2791    /// RFC 0005 A2/D9: the executable downgrade step. Drop the container file
2792    /// and both metadata records, and report whether anything was present.
2793    /// Idempotent: absence is a successful no-op.
2794    pub fn previous_version_discard(
2795        &self,
2796    ) -> Result<crate::previous_version::PreviousVersionDiscardOutcome, String> {
2797        self.drop_previous_version()
2798    }
2799
2800    fn drop_previous_version(
2801        &self,
2802    ) -> Result<crate::previous_version::PreviousVersionDiscardOutcome, String> {
2803        let Some(replica_path) = self.previous_version_replica_path() else {
2804            return Ok(crate::previous_version::PreviousVersionDiscardOutcome {
2805                present: false,
2806                discarded: false,
2807            });
2808        };
2809        crate::previous_version::discard_previous_version(&self.conn, &replica_path)
2810    }
2811
2812    /// RFC 0005 D7: `leaseState.errorCode` or an expired lease.
2813    fn previous_version_lease_inactive(&self, now_ms: i64) -> bool {
2814        let Some(lease) = &self.lease_state else {
2815            return false;
2816        };
2817        if lease.error_code.is_some() {
2818            return true;
2819        }
2820        lease.expires_at_ms.is_some_and(|expires| expires <= now_ms)
2821    }
2822
2823    /// RFC 0005 D7: §7.4.5 replacement coverage — no active unit is
2824    /// bootstrapping. An empty active set is not completion (mirrors the TS
2825    /// predicate exactly).
2826    fn previous_version_coverage_complete(&self) -> Result<bool, String> {
2827        let units = self.stored(load_units(&self.conn))?;
2828        let mut active = units
2829            .iter()
2830            .filter(|unit| unit.status == UnitStatus::Active)
2831            .peekable();
2832        Ok(active.peek().is_some() && !active.any(UnitRecord::bootstrapping))
2833    }
2834
2835    /// RFC 0005 D7: lifetime triggers evaluated once per sync round, after the
2836    /// apply. A container that no longer has a reason to exist is discarded so
2837    /// the rows do not outlive the window they were kept for.
2838    fn previous_version_lifetime_check(&self) {
2839        let Some(config) = self.previous_version.filter(|config| config.enabled) else {
2840            return;
2841        };
2842        let now_ms = self.clock_now_ms();
2843        if self.previous_version_lease_inactive(now_ms) {
2844            let _ = self.drop_previous_version();
2845            return;
2846        }
2847        let Some(replica_path) = self.previous_version_replica_path() else {
2848            return;
2849        };
2850        let Ok(Some(record)) = crate::previous_version::read_previous_version_record(&replica_path)
2851        else {
2852            return;
2853        };
2854        let Ok(units) = load_units(&self.conn) else {
2855            return;
2856        };
2857        if units.iter().any(|unit| unit.status == UnitStatus::Revoked) {
2858            let _ = self.drop_previous_version();
2859            return;
2860        }
2861        if now_ms - record.created_at_ms > config.max_age_ms {
2862            let _ = self.drop_previous_version();
2863            return;
2864        }
2865        if self.previous_version_coverage_complete() == Ok(true) {
2866            let _ = self.drop_previous_version();
2867        }
2868    }
2869
2870    pub fn diagnostics_snapshot(&self) -> Result<ClientDiagnosticsSnapshot, String> {
2871        let registered = self.stored(load_units(&self.conn))?;
2872        // Both meta values in one statement: the FFI host recomputes
2873        // diagnostics after every command.
2874        let (local_revision, replica_cursor): (Option<String>, Option<String>) = self.stored(
2875            self.conn
2876                .prepare_cached(
2877                    "SELECT (SELECT value FROM _syncular_meta WHERE key = ?1),
2878                            (SELECT value FROM _syncular_meta WHERE key = ?2)",
2879                )
2880                .and_then(|mut statement| {
2881                    statement.query_row([LOCAL_REVISION_KEY, REPLICA_CURSOR_KEY], |row| {
2882                        Ok((row.get(0)?, row.get(1)?))
2883                    })
2884                }),
2885        )?;
2886        let mut units = DiagnosticUnits::default();
2887        for unit in &registered {
2888            *match unit.status {
2889                UnitStatus::Revoked => &mut units.revoked,
2890                UnitStatus::Failed => &mut units.failed,
2891                UnitStatus::Active if !unit.bootstrapping() => &mut units.complete,
2892                UnitStatus::Active
2893                    if unit.reason_code.as_deref() == Some("sync.cursor_expired") =>
2894                {
2895                    &mut units.reset
2896                }
2897                UnitStatus::Active => &mut units.bootstrapping,
2898            } += 1;
2899        }
2900        units.reason_codes = registered
2901            .iter()
2902            .filter_map(|unit| unit.reason_code.as_deref().map(Self::diagnostic_code))
2903            .collect::<BTreeSet<_>>()
2904            .into_iter()
2905            .collect();
2906        let captured_at_ms = self.clock_now_ms();
2907        let lease = if let Some(error_code) = self
2908            .lease_state
2909            .as_ref()
2910            .and_then(|state| state.error_code.clone())
2911        {
2912            ClientDiagnosticsLease {
2913                state: "stopped".to_owned(),
2914                expires_at_ms: self
2915                    .lease_state
2916                    .as_ref()
2917                    .and_then(|state| state.expires_at_ms),
2918                error_code: Some(Self::diagnostic_code(&error_code)),
2919            }
2920        } else if let Some(expires_at_ms) = self
2921            .lease_state
2922            .as_ref()
2923            .and_then(|state| state.expires_at_ms)
2924        {
2925            ClientDiagnosticsLease {
2926                state: if expires_at_ms <= captured_at_ms {
2927                    "expired".to_owned()
2928                } else {
2929                    "active".to_owned()
2930                },
2931                expires_at_ms: Some(expires_at_ms),
2932                error_code: None,
2933            }
2934        } else {
2935            ClientDiagnosticsLease {
2936                state: "none".to_owned(),
2937                expires_at_ms: None,
2938                error_code: None,
2939            }
2940        };
2941        let connectivity = match self.last_round.as_ref() {
2942            _ if !self.transport_enabled && self.active_round.is_none() => "offline",
2943            Some(round) if round.status == "succeeded" => "online",
2944            Some(round)
2945                if round.status == "failed"
2946                    && matches!(
2947                        round.cause_kind,
2948                        Some(
2949                            crate::TransportCauseKind::Network | crate::TransportCauseKind::Connect
2950                        )
2951                    ) =>
2952            {
2953                "offline"
2954            }
2955            Some(round) if round.status == "failed" && round.http_status.is_some() => "online",
2956            _ => "unknown",
2957        };
2958        Ok(ClientDiagnosticsSnapshot {
2959            version: CLIENT_DIAGNOSTICS_VERSION,
2960            captured_at_ms,
2961            host: ClientDiagnosticsHost {
2962                kind: "direct".to_owned(),
2963                role: "single".to_owned(),
2964                connectivity: connectivity.to_owned(),
2965                realtime: self.realtime_state,
2966                realtime_policy: self.realtime_policy,
2967                realtime_reason_code: self.realtime_reason_code.clone(),
2968                realtime_retry_delay_ms: self.realtime_retry_delay_ms,
2969            },
2970            security_lifecycle: self.security_lifecycle().to_owned(),
2971            schema: ClientDiagnosticsSchema {
2972                current_version: self.schema.version,
2973                upgrading: self.upgrading,
2974                required_version: self
2975                    .schema_floor
2976                    .as_ref()
2977                    .and_then(|floor| floor.required_schema_version),
2978                latest_version: self
2979                    .schema_floor
2980                    .as_ref()
2981                    .and_then(|floor| floor.latest_schema_version),
2982            },
2983            replica: ClientDiagnosticsReplica {
2984                local_revision: local_revision
2985                    .and_then(|value| value.parse::<u64>().ok())
2986                    .unwrap_or(0)
2987                    .to_string(),
2988                sync_needed: self.sync_needed,
2989                pending_outbox: self.outbox.len(),
2990            },
2991            lease,
2992            active_units: registered
2993                .iter()
2994                .filter(|unit| unit.status == UnitStatus::Active)
2995                .count(),
2996            units,
2997            replica_cursor: replica_cursor.map_or(Ok(-1), |raw| {
2998                raw.parse()
2999                    .map_err(|_| "sync.local_corrupt: persisted unit state is invalid".to_owned())
3000            })?,
3001            last_round: self.last_round.clone(),
3002            last_change: self.last_change.clone(),
3003            storage: self.diagnostics_storage(),
3004            query_failures: self.query_failures.clone(),
3005        })
3006    }
3007
3008    fn diagnostics_storage(&self) -> ClientDiagnosticsStorage {
3009        let read = || -> Result<ClientDiagnosticsStorage, rusqlite::Error> {
3010            let page_count: i64 = self
3011                .conn
3012                .query_row("PRAGMA page_count", [], |row| row.get(0))?;
3013            let page_size: i64 = self
3014                .conn
3015                .query_row("PRAGMA page_size", [], |row| row.get(0))?;
3016            // One cached aggregate, since the FFI host recomputes diagnostics
3017            // after every command; every invocation still reads current storage.
3018            let (outbox_bytes, outcome_entries, outcome_bytes): (i64, i64, i64) = self
3019                .conn
3020                .prepare_cached(
3021                    "SELECT (SELECT COALESCE(SUM(LENGTH(ops_json)), 0) FROM _syncular_outbox),
3022                            COUNT(*), COALESCE(SUM(LENGTH(results_json) + COALESCE(LENGTH(operations_json), 0)), 0)
3023                       FROM _syncular_commit_outcomes",
3024                )?
3025                .query_row([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))?;
3026            let blob_bytes = if self.schema_has_blobs() {
3027                self.conn
3028                    .prepare_cached("SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs")?
3029                    .query_row([], |row| row.get(0))?
3030            } else {
3031                0
3032            };
3033            let pressure = self
3034                .limits
3035                .blob_cache_max_bytes
3036                .is_some_and(|limit| blob_bytes > limit);
3037            Ok(ClientDiagnosticsStorage {
3038                status: if pressure { "pressure" } else { "healthy" }.to_owned(),
3039                database_bytes_approx: Some(page_count.saturating_mul(page_size).max(0)),
3040                pending_outbox_bytes_approx: Some(outbox_bytes.max(0)),
3041                retained_outcome_bytes_approx: Some(outcome_bytes.max(0)),
3042                retained_outcome_entries: Some(outcome_entries.max(0)),
3043                blob_cache_bytes_approx: Some(blob_bytes.max(0)),
3044                pressure_reason_code: pressure.then(|| "client.blob_cache_over_limit".to_owned()),
3045            })
3046        };
3047        read().unwrap_or_else(|_| ClientDiagnosticsStorage {
3048            status: "unreadable".to_owned(),
3049            database_bytes_approx: None,
3050            pending_outbox_bytes_approx: None,
3051            retained_outcome_bytes_approx: None,
3052            retained_outcome_entries: None,
3053            blob_cache_bytes_approx: None,
3054            pressure_reason_code: None,
3055        })
3056    }
3057
3058    pub fn drain_change_batches(&mut self) -> Vec<ClientChangeBatch> {
3059        self.change_queue.drain(..).collect()
3060    }
3061
3062    pub fn drain_sync_intents(&mut self) -> Vec<SyncIntent> {
3063        let intents = self.sync_intent_queue.drain(..).collect();
3064        if self.transport_enabled {
3065            intents
3066        } else {
3067            vec![SyncIntent::None]
3068        }
3069    }
3070
3071    fn schedule_background_retry(&mut self) -> u64 {
3072        let delay_ms = self.retry_delay_ms;
3073        self.sync_intent_queue
3074            .push_back(SyncIntent::Background { delay_ms });
3075        self.retry_delay_ms = (self.retry_delay_ms * 2).min(30_000);
3076        self.round_retry_delay_ms = Some(delay_ms);
3077        delay_ms
3078    }
3079
3080    fn reset_background_retry(&mut self) {
3081        self.retry_delay_ms = 250;
3082    }
3083
3084    fn retryable_transport_code(code: &str) -> bool {
3085        code == "transport.failed"
3086            || code == "transport.timeout"
3087            || code == "transport.unavailable"
3088            || code == "sync.transport_failed"
3089    }
3090
3091    /// §1.8: the transport reported a refused or unreachable endpoint. The
3092    /// native transport says `connect` (DNS failure, connection refused,
3093    /// reset, or aborted); a browser fetch transport says `network`. Repeating
3094    /// the request on a backoff repairs neither, so no retry is scheduled.
3095    fn unreachable_transport(details: &Option<serde_json::Value>) -> bool {
3096        matches!(
3097            details
3098                .as_ref()
3099                .and_then(|details| details.get("causeKind"))
3100                .and_then(serde_json::Value::as_str),
3101            Some("connect" | "network")
3102        )
3103    }
3104
3105    /// §1.5: a segment fetch that fails inside an otherwise complete round is
3106    /// invisible to the server (the round response arrived whole), so its code
3107    /// rides the following requests like a socket-round failure. A code the
3108    /// frame grammar rejects is not reportable and the previous code stays.
3109    fn segment_fetch_failed(&mut self, error: TransportError) -> SectionError {
3110        if ssp2::model::valid_round_failure_code(&error.code) {
3111            self.round_failure = Some(error.code.clone());
3112        }
3113        SectionError::Transport(error)
3114    }
3115
3116    /// A failed round worth a background retry: a transport failure, or a
3117    /// §10.2 code the catalog marks retryable (clients may hardcode the
3118    /// catalog metadata, §10.1). Matches the TypeScript core, which retries
3119    /// on the server's `retryable` flag.
3120    fn retryable_failure_code(code: &str) -> bool {
3121        Self::retryable_transport_code(code)
3122            || matches!(
3123                code,
3124                "sync.auth_required"
3125                    | "sync.auth_lease_required"
3126                    | "sync.auth_lease_revoked"
3127                    | "sync.internal_error"
3128                    | "sync.idempotency_cache_miss"
3129                    | "sync.schema_not_ready"
3130                    | "sync.segment_expired"
3131                    | "sync.rate_limited"
3132                    | "sync.websocket_connection_limit"
3133            )
3134    }
3135
3136    fn diagnostic_code(code: &str) -> String {
3137        let valid = !code.is_empty()
3138            && code.len() <= 96
3139            && code.contains('.')
3140            && code
3141                .bytes()
3142                .next()
3143                .is_some_and(|byte| byte.is_ascii_lowercase())
3144            && code.bytes().all(|byte| {
3145                byte.is_ascii_lowercase()
3146                    || byte.is_ascii_digit()
3147                    || matches!(byte, b'.' | b'_' | b'-')
3148            });
3149        if valid {
3150            code.to_owned()
3151        } else {
3152            "client.unknown_failure".to_owned()
3153        }
3154    }
3155
3156    fn set_sync_needed(&mut self, value: bool, interactive: bool) {
3157        if self.sync_needed != value {
3158            if self.begin_observation("syncular_status").is_ok() {
3159                self.sync_needed = value;
3160                let batch = ChangeAccumulator {
3161                    status: true,
3162                    ..ChangeAccumulator::default()
3163                };
3164                if self.finish_observation("syncular_status", batch).is_err() {
3165                    self.rollback_observation("syncular_status");
3166                }
3167            } else {
3168                self.sync_needed = value;
3169            }
3170        }
3171        if value && interactive {
3172            self.sync_intent_queue.push_back(SyncIntent::Interactive);
3173        }
3174    }
3175
3176    /// A unit-store result: a row that does not parse is `sync.local_corrupt`;
3177    /// every other SQLite error keeps its storage classification.
3178    fn stored<T>(&self, result: rusqlite::Result<T>) -> Result<T, String> {
3179        result.map_err(|error| match error {
3180            rusqlite::Error::FromSqlConversionFailure(..) => {
3181                "sync.local_corrupt: persisted unit state is invalid".to_owned()
3182            }
3183            error => Self::sqlite_failure(&self.storage_failure, error),
3184        })
3185    }
3186
3187    fn sqlite_failure(
3188        storage_failure: &RefCell<Option<QueryReadFailure>>,
3189        error: rusqlite::Error,
3190    ) -> String {
3191        let message = error.to_string();
3192        let failure = QueryReadFailure::from(error);
3193        let code = failure.code;
3194        if code.is_some() && storage_failure.borrow().is_none() {
3195            *storage_failure.borrow_mut() = Some(failure);
3196        }
3197        match code {
3198            Some(code) => format!("{code}: {message}"),
3199            None => message,
3200        }
3201    }
3202
3203    fn begin_observation(&self, name: &str) -> Result<(), String> {
3204        if let Some(failure) = &self.coalesced_failure {
3205            return Err(failure.clone());
3206        }
3207        // SQLite rolls a whole transaction back on some I/O and capacity
3208        // errors; later frames must not commit on their own after that.
3209        if self.coalesced.is_some() && self.conn.is_autocommit() {
3210            return Err("the coalesced apply transaction was rolled back".to_owned());
3211        }
3212        self.conn
3213            .execute_batch(&format!("SAVEPOINT {name}"))
3214            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
3215    }
3216
3217    /// SPEC §1.4 rule 4: open a coalesced apply transaction unless a caller
3218    /// already owns one.
3219    fn begin_coalesced_apply(&mut self) -> Result<(), String> {
3220        if self.coalesced.is_some() || !self.conn.is_autocommit() {
3221            return Ok(());
3222        }
3223        self.conn
3224            .execute_batch("BEGIN")
3225            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3226        self.coalesced = Some(CoalescedApply {
3227            rows: 0,
3228            rollback: self.capture(),
3229        });
3230        Ok(())
3231    }
3232
3233    fn capture(&self) -> Rollback {
3234        Rollback {
3235            outbox: self.outbox.clone(),
3236            failed_commits: self.failed_commits.clone(),
3237            conflicts: self.conflicts.clone(),
3238            rejections: self.rejections.clone(),
3239            schema_floor: self.schema_floor.clone(),
3240            lease_state: self.lease_state.clone(),
3241            stopped: self.stopped,
3242            upgrading: self.upgrading,
3243            sync_needed: self.sync_needed,
3244            active_round: self.active_round,
3245            overlay_dirty: self.overlay_dirty.snapshot(),
3246            change_queue: self.change_queue.len(),
3247            sync_intent_queue: self.sync_intent_queue.len(),
3248            last_change: self.last_change.clone(),
3249        }
3250    }
3251
3252    fn restore(&mut self, rollback: Rollback) {
3253        self.outbox = rollback.outbox;
3254        self.failed_commits = rollback.failed_commits;
3255        self.conflicts = rollback.conflicts;
3256        self.rejections = rollback.rejections;
3257        self.schema_floor = rollback.schema_floor;
3258        self.lease_state = rollback.lease_state;
3259        self.stopped = rollback.stopped;
3260        self.upgrading = rollback.upgrading;
3261        self.sync_needed = rollback.sync_needed;
3262        self.active_round = rollback.active_round;
3263        self.overlay_dirty.restore(rollback.overlay_dirty);
3264        self.change_queue.truncate(rollback.change_queue);
3265        self.sync_intent_queue.truncate(rollback.sync_intent_queue);
3266        self.last_change = rollback.last_change;
3267    }
3268
3269    /// Account the rows of the next frame or block. A unit that would take the
3270    /// coalesced transaction past the bound commits the earlier units first.
3271    fn coalesce_unit(&mut self, rows: usize) -> Result<(), String> {
3272        let Some(open) = &self.coalesced else {
3273            return Ok(());
3274        };
3275        if open.rows > 0 && open.rows + rows > COALESCED_APPLY_ROWS {
3276            self.commit_coalesced_apply()?;
3277            self.begin_coalesced_apply()?;
3278        }
3279        if let Some(open) = &mut self.coalesced {
3280            open.rows += rows;
3281        }
3282        Ok(())
3283    }
3284
3285    /// COMMIT the coalesced transaction. A failed COMMIT rolls every coalesced
3286    /// frame back, restores the in-memory state of its `BEGIN`, drops the
3287    /// change batches and sync intents queued since, and fails the rest of the
3288    /// response.
3289    fn commit_coalesced_apply(&mut self) -> Result<(), String> {
3290        let Some(open) = self.coalesced.take() else {
3291            return Ok(());
3292        };
3293        let committed = if self.conn.is_autocommit() {
3294            Err("the coalesced apply transaction was rolled back".to_owned())
3295        } else {
3296            self.conn
3297                .execute_batch("COMMIT")
3298                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
3299        };
3300        let Err(error) = committed else {
3301            return Ok(());
3302        };
3303        if !self.conn.is_autocommit() {
3304            if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
3305                let _ = Self::sqlite_failure(&self.storage_failure, rollback);
3306            }
3307        }
3308        self.restore(open.rollback);
3309        self.coalesced_failure = Some(error.clone());
3310        Err(error)
3311    }
3312
3313    /// End the response's coalescing: COMMIT what is open and clear a failure.
3314    fn finish_coalesced_apply(&mut self) -> Result<(), String> {
3315        let committed = self.commit_coalesced_apply();
3316        match self.coalesced_failure.take() {
3317            Some(failure) => Err(failure),
3318            None => committed,
3319        }
3320    }
3321
3322    fn rollback_observation(&self, name: &str) {
3323        if let Err(error) = self
3324            .conn
3325            .execute_batch(&format!("ROLLBACK TO {name}; RELEASE {name}"))
3326        {
3327            if let Some(failure) = self.storage_failure.borrow_mut().as_mut() {
3328                if failure.rollback_failure.is_none() {
3329                    failure.rollback_failure = Some(Box::new(QueryReadFailure::from(error)));
3330                }
3331            }
3332        }
3333    }
3334
3335    fn finish_observation(
3336        &mut self,
3337        name: &str,
3338        mut batch: ChangeAccumulator,
3339    ) -> Result<(), String> {
3340        #[cfg(feature = "bench-internals")]
3341        let _phase = self.benchmark_phases.start(Phase::ObservationCommit);
3342        if self.failed_commits.iter().any(|(commit, _)| {
3343            commit
3344                .ops
3345                .iter()
3346                .any(|operation| batch.tables.contains_key(&operation.table))
3347        }) {
3348            batch.outcomes = true;
3349        }
3350        if !batch.touched() {
3351            self.conn
3352                .execute_batch(&format!("RELEASE {name}"))
3353                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3354            return Ok(());
3355        }
3356        let revision = self
3357            .local_revision()?
3358            .checked_add(1)
3359            .ok_or_else(|| "local revision exhausted u64".to_owned())?;
3360        self.conn
3361            .execute(
3362                "INSERT OR REPLACE INTO _syncular_meta(key, value) VALUES (?1, ?2)",
3363                rusqlite::params![LOCAL_REVISION_KEY, revision.to_string()],
3364            )
3365            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3366        let status = batch.status.then(|| self.status_snapshot());
3367        let event = ClientChangeBatch {
3368            revision: revision.to_string(),
3369            tables: batch
3370                .tables
3371                .into_iter()
3372                .map(|(table, scope_keys)| TableChange {
3373                    table,
3374                    scope_keys: scope_keys.map(|keys| keys.into_iter().collect()),
3375                })
3376                .collect(),
3377            windows: batch
3378                .windows
3379                .into_iter()
3380                .map(|(base_key, units)| WindowChange {
3381                    base_key,
3382                    units: units.into_iter().collect(),
3383                })
3384                .collect(),
3385            status,
3386            conflicts_changed: batch.conflicts,
3387            rejections_changed: batch.rejections,
3388            outcomes_changed: batch.outcomes,
3389            applied_commits: Vec::new(),
3390        };
3391        self.conn
3392            .execute_batch(&format!("RELEASE {name}"))
3393            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3394        let mut diagnostic_tables = event
3395            .tables
3396            .iter()
3397            .map(|entry| entry.table.clone())
3398            .collect::<BTreeSet<_>>()
3399            .into_iter()
3400            .collect::<Vec<_>>();
3401        // §7.6: the selected variables of the changed window bases.
3402        let mut diagnostic_windows = event
3403            .windows
3404            .iter()
3405            .map(|entry| {
3406                entry
3407                    .base_key
3408                    .split('\0')
3409                    .next()
3410                    .unwrap_or_default()
3411                    .to_owned()
3412            })
3413            .collect::<BTreeSet<_>>()
3414            .into_iter()
3415            .collect::<Vec<_>>();
3416        let domains_truncated = diagnostic_tables.len() > MAX_DIAGNOSTIC_DOMAINS
3417            || diagnostic_windows.len() > MAX_DIAGNOSTIC_DOMAINS;
3418        diagnostic_tables.truncate(MAX_DIAGNOSTIC_DOMAINS);
3419        diagnostic_windows.truncate(MAX_DIAGNOSTIC_DOMAINS);
3420        self.last_change = Some(DiagnosticLastChange {
3421            revision: event.revision.clone(),
3422            recorded_at_ms: self.clock_now_ms(),
3423            tables: diagnostic_tables,
3424            windows: diagnostic_windows,
3425            domains_truncated,
3426            status_changed: event.status.is_some(),
3427            conflicts_changed: event.conflicts_changed,
3428            rejections_changed: event.rejections_changed,
3429            outcomes_changed: event.outcomes_changed,
3430        });
3431        self.change_queue.push_back(event);
3432        Ok(())
3433    }
3434
3435    fn record_scope_map(
3436        &self,
3437        batch: &mut ChangeAccumulator,
3438        table_name: &str,
3439        scopes: &[(String, Vec<String>)],
3440    ) {
3441        let Some(table) = self.schema.table(table_name) else {
3442            return;
3443        };
3444        // A bulk scoped delete whose map leaves a declared variable unnamed
3445        // cannot list the deleted rows' keys for it: the change is table-wide.
3446        if table.scope_variables.iter().any(|scope| {
3447            !scopes
3448                .iter()
3449                .any(|(variable, _)| *variable == scope.variable)
3450        }) {
3451            batch.table(table_name);
3452            return;
3453        }
3454        for (variable, values) in scopes {
3455            let Some(scope) = table
3456                .scope_variables
3457                .iter()
3458                .find(|scope| &scope.variable == variable)
3459            else {
3460                continue;
3461            };
3462            for value in values {
3463                batch.scope(table_name, format!("{}:{value}", scope.prefix));
3464            }
3465        }
3466    }
3467
3468    /// Record a row's current scope keys from the base or visible table.
3469    fn record_row_scopes(
3470        &self,
3471        batch: &mut ChangeAccumulator,
3472        table_name: &str,
3473        row_id: &str,
3474        base: bool,
3475    ) -> bool {
3476        let Some(table) = self.schema.table(table_name) else {
3477            return false;
3478        };
3479        if table.scope_variables.is_empty() {
3480            return false;
3481        }
3482        let columns = table
3483            .scope_variables
3484            .iter()
3485            .map(|scope| quote_ident(&scope.column))
3486            .collect::<Vec<_>>()
3487            .join(", ");
3488        let full_table = if base {
3489            base_table(table_name)
3490        } else {
3491            visible_table(table_name)
3492        };
3493        let sql = format!(
3494            "SELECT {columns} FROM {full_table} WHERE {} LIMIT 1",
3495            row_id_predicate(table)
3496        );
3497        let Ok(mut stmt) = self.conn.prepare_cached(&sql) else {
3498            return false;
3499        };
3500        let values = stmt.query_row(rusqlite::params![row_id], |row| {
3501            let mut values = Vec::with_capacity(table.scope_variables.len());
3502            for index in 0..table.scope_variables.len() {
3503                values.push(row.get::<_, Option<String>>(index)?);
3504            }
3505            Ok(values)
3506        });
3507        let Ok(values) = values else {
3508            return false;
3509        };
3510        let mut recorded = false;
3511        for (scope, value) in table.scope_variables.iter().zip(values) {
3512            if let Some(value) = value {
3513                batch.scope(table_name, format!("{}:{value}", scope.prefix));
3514                recorded = true;
3515            }
3516        }
3517        recorded
3518    }
3519
3520    fn record_commit_changes(
3521        &self,
3522        batch: &mut ChangeAccumulator,
3523        tables: &[String],
3524        changes: &[ssp2::model::Change],
3525    ) {
3526        #[cfg(feature = "bench-internals")]
3527        let _phase = self.benchmark_phases.start(Phase::ObservationPrepare);
3528        for change in changes {
3529            let Some(table_name) = tables.get(change.table_index as usize) else {
3530                continue;
3531            };
3532            let mut precise = self.record_row_scopes(batch, table_name, &change.row_id, true);
3533            if let Some(table) = self.schema.table(table_name) {
3534                for (variable, value) in &change.scopes {
3535                    if let Some(scope) = table
3536                        .scope_variables
3537                        .iter()
3538                        .find(|scope| &scope.variable == variable)
3539                    {
3540                        batch.scope(table_name, format!("{}:{value}", scope.prefix));
3541                        precise = true;
3542                    }
3543                }
3544            }
3545            if !precise {
3546                batch.table(table_name);
3547            }
3548        }
3549    }
3550
3551    /// Whether a base or visible row matches the scope predicate; `false`
3552    /// without a local scope-column mapping. A SQLite failure is an error.
3553    fn scoped_rows_exist(
3554        &self,
3555        table_name: &str,
3556        effective: &[(String, Vec<String>)],
3557        owners: &[Vec<(String, Vec<String>)>],
3558    ) -> Result<bool, String> {
3559        let Some(table) = self.schema.table(table_name) else {
3560            return Ok(false);
3561        };
3562        let Ok(Some((predicate, params))) = local_scope_predicate(table, effective, owners) else {
3563            return Ok(false);
3564        };
3565        for source in [base_table(table_name), visible_table(table_name)] {
3566            if self
3567                .conn
3568                .query_row(
3569                    &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {predicate})"),
3570                    rusqlite::params_from_iter(&params),
3571                    |row| row.get::<_, bool>(0),
3572                )
3573                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
3574            {
3575                return Ok(true);
3576            }
3577        }
3578        Ok(false)
3579    }
3580
3581    /// §5.6 first-page clear of `unit_id`'s pair: delete its rows except
3582    /// those another owner covers (§3.3), failing closed without a local
3583    /// scope-column mapping. Owners are read only when a row matches, inside
3584    /// the caller's transaction. Returns whether rows matched.
3585    fn clear_scope(
3586        &mut self,
3587        table: &TableSchema,
3588        effective: &[(String, Vec<String>)],
3589        unit_id: &str,
3590    ) -> Result<bool, SectionError> {
3591        let storage = |message| SectionError::Abort("storage.failed".to_owned(), message);
3592        let matched = self
3593            .scoped_rows_exist(&table.name, effective, &[])
3594            .map_err(storage)?;
3595        let owners = if matched {
3596            unit_owners(
3597                &self.stored(load_units(&self.conn)).map_err(storage)?,
3598                table,
3599                unit_id,
3600            )
3601        } else {
3602            Vec::new()
3603        };
3604        let changed = matched
3605            && (owners.is_empty()
3606                || self
3607                    .scoped_rows_exist(&table.name, effective, &owners)
3608                    .map_err(storage)?);
3609        if !self
3610            .purge_scope_rows(&table.name, effective, &owners)
3611            .map_err(storage)?
3612        {
3613            return Err(SectionError::FailClosed);
3614        }
3615        Ok(changed)
3616    }
3617
3618    /// §7.4.5: true while a schema-bump reset + first re-bootstrap runs.
3619    pub fn upgrading(&self) -> bool {
3620        self.upgrading
3621    }
3622
3623    /// §7.4.2 "app ships new code": swap to a NEW generated schema while
3624    /// keeping this client's local database (identity, outbox, tables). The
3625    /// §7.4.1 marker check then fires the wipe/re-bootstrap flow when the
3626    /// version increases. Recreation runs the same marker guard as opening
3627    /// a durable replica, holds the guard and every write it authorizes under
3628    /// one write transaction, and preserves the current client (compiled
3629    /// schema, outbox, tables) on refusal.
3630    pub fn recreate_with_schema(&mut self, schema_json: &Value) -> Result<(), String> {
3631        let new_schema = parse_schema_json(schema_json)?;
3632        validate_authority_reads(&new_schema, &self.limits.authority_reads)?;
3633        self.conn
3634            .execute_batch("BEGIN IMMEDIATE")
3635            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3636        let prior_schema = std::mem::replace(&mut self.schema, new_schema);
3637        // §7.4.3/§7.4.4: a reset inside this transaction mutates in-memory
3638        // state (dropped outbox commits, rejections, upgrade and floor flags,
3639        // overlay dirtiness) and appends change/intent batches. A COMMIT that
3640        // fails after the reset succeeded rolls the database back, so every
3641        // one of those fields is captured here and restored with it.
3642        let rollback = self.capture();
3643        let guarded = (|| -> Result<(), String> {
3644            // §7.4.2: the marker is re-read under the write lock the caller
3645            // holds, so a replica another process upgraded meanwhile refuses
3646            // this recreation instead of having its marker and tables
3647            // downgraded.
3648            let marker = read_local_schema_version(&self.conn, self.schema.version)?;
3649            self.create_bookkeeping_tables()?;
3650            match marker {
3651                Some(version) if version < self.schema.version => self.run_schema_reset()?,
3652                None => {
3653                    self.create_synced_tables()?;
3654                    self.persist_schema_version()?;
3655                }
3656                _ => {}
3657            }
3658            // §7.4.2: recreation IS the boot for the in-memory core, so reload the
3659            // durable outbox the same-version path is about to replay. Only the
3660            // outbox is reloaded: the recovery below reads it, and the caller's
3661            // rollback snapshot already covers this field.
3662            self.outbox = self.load_outbox()?.into();
3663            // §7.1/§7.4.4: recover intrinsically codec-invalid persisted values at
3664            // this guarded boundary once the schema and marker agree. A bump went
3665            // through `run_schema_reset`, which performs the same recovery after
3666            // it creates the new physical tables and before its final replay.
3667            self.drop_unencodable_outbox(false, None)?;
3668            if marker == Some(self.schema.version)
3669                && (!self.outbox.is_empty()
3670                    || !self.failed_commits.is_empty()
3671                    || self.has_acknowledged_rows()?)
3672            {
3673                self.overlay_dirty.set(true);
3674                self.rebuild_overlay()?;
3675            }
3676            self.prune_unknown_units(false)?;
3677            self.save_unit_scope_schema()?;
3678            // The conformance recreate is the in-memory equivalent of reopening
3679            // a durable client. Apply the same startup catch-up contract even
3680            // when the schema itself did not change.
3681            self.enqueue_startup_sync_if_needed()?;
3682            Ok(())
3683        })();
3684        let outcome = match guarded {
3685            Ok(()) => self
3686                .conn
3687                .execute_batch("COMMIT")
3688                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error)),
3689            Err(error) => Err(error),
3690        };
3691        if let Err(error) = outcome {
3692            if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
3693                // Keep a failed rollback's typed storage classification.
3694                let _ = Self::sqlite_failure(&self.storage_failure, rollback);
3695            }
3696            self.schema = prior_schema;
3697            self.restore(rollback);
3698            // Prepared insert statements were derived from the reverted schema.
3699            self.insert_sql.borrow_mut().clear();
3700            return Err(error);
3701        }
3702        if !self.security_preflight {
3703            self.drain_pending_evictions()?;
3704        }
3705        Ok(())
3706    }
3707
3708    fn enqueue_startup_sync_if_needed(&mut self) -> Result<(), String> {
3709        let startup_work = !self.stopped
3710            && (!self.outbox.is_empty()
3711                || self.has_acknowledged_rows()?
3712                || !self.load_pending_evictions()?.is_empty()
3713                || self
3714                    .stored(load_units(&self.conn))?
3715                    .iter()
3716                    .any(|unit| unit.status == UnitStatus::Active));
3717        if startup_work {
3718            self.sync_needed = true;
3719            self.sync_intent_queue.push_back(SyncIntent::Interactive);
3720        }
3721        Ok(())
3722    }
3723
3724    /// A native client persists schema-floor stops across process restarts.
3725    /// Once the running app already satisfies that floor, the persisted stop
3726    /// is only stale evidence from an older server round (for example, the
3727    /// server was restarted after catching up to an app that was ahead).
3728    /// Clear it and let the normal startup pull re-negotiate. If the server is
3729    /// still incompatible it will return the floor again in that first round.
3730    fn clear_satisfied_persisted_schema_floor(&mut self) -> Result<(), String> {
3731        let satisfied = self
3732            .schema_floor
3733            .as_ref()
3734            .and_then(|floor| floor.required_schema_version)
3735            .is_some_and(|required| self.schema.version >= required);
3736        if !satisfied {
3737            return Ok(());
3738        }
3739        self.delete_meta(SCHEMA_FLOOR_KEY)?;
3740        self.schema_floor = None;
3741        self.stopped = false;
3742        Ok(())
3743    }
3744
3745    /// RFC 0005 D1: discard an orphan or stale container at boot. The reset
3746    /// path already swept it; this covers a same-version or fresh-install open,
3747    /// which runs no reset and therefore no sweep — plus the D7 TTL, which only
3748    /// an aware binary can apply.
3749    fn reconcile_previous_version_at_boot(&self) -> Result<(), String> {
3750        let Some(replica_path) = self.previous_version_replica_path() else {
3751            return Ok(());
3752        };
3753        let max_age_ms = self
3754            .previous_version
3755            .filter(|config| config.enabled)
3756            .map(|config| config.max_age_ms);
3757        reconcile_previous_version_at_boot(
3758            &self.conn,
3759            &replica_path,
3760            self.schema.version,
3761            max_age_ms,
3762            self.clock_now_ms(),
3763        )
3764    }
3765
3766    /// RFC 0005 D3: the sibling container path, or `None` when the replica has
3767    /// no file — an in-memory replica returns an empty path, and a `file:` URI
3768    /// has no plain sibling path — so the feature resolves as off there.
3769    fn previous_version_replica_path(&self) -> Option<String> {
3770        let path = self.conn.path()?;
3771        if path.is_empty() || path.starts_with("file:") {
3772            return None;
3773        }
3774        Some(path.to_owned())
3775    }
3776
3777    /// §7.4.3 reset: whole-database local reset EXCEPT the outbox, clientId,
3778    /// and leaseState. Drops/recreates every synced table from the new
3779    /// schema, resets unit sync state (keeping registrations), clears
3780    /// the schema-floor stop state, rewrites the marker, drops outbox commits
3781    /// that cannot re-encode (§7.4.4), and replays the survivors on top.
3782    fn run_schema_reset(&mut self) -> Result<(), String> {
3783        self.begin_observation("syncular_schema_reset")?;
3784        let rollback = self.capture();
3785        let mut batch = ChangeAccumulator::default();
3786        let result = self
3787            .run_schema_reset_observed(&mut batch, true)
3788            .and_then(|()| self.finish_observation("syncular_schema_reset", batch));
3789        if let Err(error) = result {
3790            self.rollback_observation("syncular_schema_reset");
3791            self.restore(rollback);
3792            return Err(error);
3793        }
3794        Ok(())
3795    }
3796
3797    fn run_log_epoch_reset(&mut self, log_epoch: &str) -> Result<Vec<String>, String> {
3798        if self.get_meta(LOG_EPOCH_KEY)?.is_none() {
3799            self.begin_observation("syncular_epoch_acquisition")?;
3800            let rollback = self.capture();
3801            if let Err(error) = self.conn.execute(
3802                "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
3803                rusqlite::params![LOG_EPOCH_KEY, log_epoch],
3804            ) {
3805                self.rollback_observation("syncular_epoch_acquisition");
3806                return Err(Self::sqlite_failure(&self.storage_failure, error));
3807            }
3808            self.sync_needed = true;
3809            let batch = ChangeAccumulator {
3810                status: true,
3811                ..ChangeAccumulator::default()
3812            };
3813            if let Err(error) = self.finish_observation("syncular_epoch_acquisition", batch) {
3814                self.rollback_observation("syncular_epoch_acquisition");
3815                self.restore(rollback);
3816                return Err(error);
3817            }
3818            self.sync_intent_queue.push_back(SyncIntent::Interactive);
3819            return Ok(Vec::new());
3820        }
3821        let resets = self
3822            .stored(load_units(&self.conn))?
3823            .into_iter()
3824            .map(|unit| unit.id)
3825            .collect();
3826        self.begin_observation("syncular_log_epoch_reset")?;
3827        let rollback = self.capture();
3828        let mut batch = ChangeAccumulator::default();
3829        let result = self
3830            .run_schema_reset_observed(&mut batch, false)
3831            .and_then(|()| {
3832                self.conn
3833                    .execute(
3834                        "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
3835                        rusqlite::params![LOG_EPOCH_KEY, log_epoch],
3836                    )
3837                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3838                self.sync_needed = true;
3839                batch.status = true;
3840                self.finish_observation("syncular_log_epoch_reset", batch)
3841            });
3842        if let Err(error) = result {
3843            self.rollback_observation("syncular_log_epoch_reset");
3844            self.restore(rollback);
3845            return Err(error);
3846        }
3847        self.sync_intent_queue.push_back(SyncIntent::Interactive);
3848        Ok(resets)
3849    }
3850
3851    fn run_schema_reset_observed(
3852        &mut self,
3853        batch: &mut ChangeAccumulator,
3854        capture: bool,
3855    ) -> Result<(), String> {
3856        // §7.4.2: the reset is the only path that wipes local tables for a
3857        // version bump, and it rewrites the marker last. Re-read the marker
3858        // under this reset's transaction: a replica another process upgraded
3859        // meanwhile refuses here instead of being reset to this client's
3860        // older schema.
3861        read_local_schema_version(&self.conn, self.schema.version)?;
3862        self.conn
3863            .execute_batch(
3864                "DELETE FROM _syncular_acked_rows; DELETE FROM _syncular_row_deliveries;",
3865            )
3866            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3867        self.cancel_sync_round();
3868        self.upgrading = true;
3869        batch.status = true;
3870        // RFC 0005 D6: classify the pending outbox against the NEW compiled
3871        // schema BEFORE the wipe, and record the advisory audit. This pass
3872        // drops nothing, so the audit sees the original pending intent; the
3873        // §7.4.4 send-time drop and the intrinsic value recovery later in this
3874        // reset are the paths that remove a commit. The classification is
3875        // shared with the send-time drop so the two cannot disagree about what
3876        // is incompatible.
3877        if self.previous_version.is_some_and(|config| config.enabled) {
3878            let previous_version = self
3879                .get_meta(LOCAL_SCHEMA_VERSION_KEY)?
3880                .and_then(|value| value.parse::<i32>().ok())
3881                .unwrap_or(self.schema.version);
3882            let pending = self
3883                .outbox
3884                .iter()
3885                .map(|commit| PendingCommitAudit {
3886                    commit_id: &commit.client_commit_id,
3887                    operations: commit_audit_operations(commit),
3888                })
3889                .collect::<Vec<_>>();
3890            crate::previous_version::write_previous_version_audit(
3891                &self.conn,
3892                &crate::previous_version::build_previous_version_audit(
3893                    &self.schema,
3894                    &pending,
3895                    previous_version,
3896                    self.schema.version,
3897                    self.clock_now_ms(),
3898                ),
3899            )?;
3900        }
3901        // RFC 0005 D5: the container lives in its own FILE, so the reset can
3902        // never reach it. Sweep first (unconditional — also on the log-epoch
3903        // reset, which captures nothing), then capture BEFORE the wipe drops
3904        // any row. A refused capture records its reason and leaves no file.
3905        if let Some(replica_path) = self.previous_version_replica_path() {
3906            sweep_previous_version_container(&self.conn, &replica_path)?;
3907            if capture {
3908                if let Some(config) = self.previous_version.filter(|config| config.enabled) {
3909                    let previous_version = self
3910                        .get_meta(LOCAL_SCHEMA_VERSION_KEY)?
3911                        .and_then(|value| value.parse::<i32>().ok())
3912                        .unwrap_or(self.schema.version);
3913                    capture_previous_version_from_replica(
3914                        &self.conn,
3915                        &replica_path,
3916                        previous_version,
3917                        self.schema.version,
3918                        &config,
3919                        self.clock_now_ms(),
3920                    )?;
3921                }
3922            }
3923        }
3924        for table in &self.schema.tables {
3925            batch.table(&table.name);
3926        }
3927        for (base_key, value) in self.load_window_registry()? {
3928            batch.window(&base_key, &value);
3929        }
3930        if capture {
3931            self.prune_unknown_units(true)?;
3932        }
3933        // The per-table insert SQL is derived from the OLD column lists.
3934        self.insert_sql.borrow_mut().clear();
3935        self.overlay_dirty.set(true);
3936        // Drop every synced table (base + visible) that currently exists —
3937        // discovered from sqlite_master so a bump that adds/removes tables is
3938        // handled. Bookkeeping tables (`_syncular_outbox/_units/_meta`
3939        // and the blob cache) are preserved; base tables are `_syncular_base_*`
3940        // so they are matched explicitly, not by the bookkeeping filter.
3941        // Drop virtual tables first so SQLite removes every FTS shadow table
3942        // atomically instead of the generic discovery tearing it apart.
3943        let virtual_tables: Vec<String> = {
3944            let mut stmt = self
3945                .conn
3946                .prepare(
3947                    "SELECT name FROM sqlite_master WHERE type = 'table' AND sql LIKE 'CREATE VIRTUAL TABLE%'",
3948                )
3949                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3950            let rows = stmt
3951                .query_map([], |row| row.get::<_, String>(0))
3952                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3953            rows.filter_map(Result::ok)
3954                .filter(|name| is_synced_table_name(name))
3955                .collect()
3956        };
3957        for name in virtual_tables {
3958            self.conn
3959                .execute_batch(&format!(
3960                    "DROP TABLE IF EXISTS {}",
3961                    quote_ident(&format!("_syncular_fts_{name}"))
3962                ))
3963                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3964            self.conn
3965                .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
3966                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3967        }
3968        let existing: Vec<String> = {
3969            let mut stmt = self
3970                .conn
3971                .prepare("SELECT name FROM sqlite_master WHERE type = 'table'")
3972                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3973            let rows = stmt
3974                .query_map([], |row| row.get::<_, String>(0))
3975                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3976            rows.filter_map(Result::ok)
3977                .filter(|name| is_synced_table_name(name))
3978                .collect()
3979        };
3980        for name in &existing {
3981            if let Some(table) = name.strip_prefix("_syncular_base_") {
3982                batch.table(table);
3983            } else if !name.starts_with("_syncular_") {
3984                batch.table(name);
3985            }
3986        }
3987        for name in existing {
3988            self.conn
3989                .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
3990                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3991        }
3992        // Recreate the synced tables from the NEW schema.
3993        self.create_synced_tables()?;
3994        // Reset every unit's sync state, keeping the registration.
3995        reset_units(&self.conn)
3996            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3997        // The stop state is over: this client now ships a servable schema.
3998        self.stopped = false;
3999        self.schema_floor = None;
4000        self.delete_meta(SCHEMA_FLOOR_KEY)?;
4001        // Rewrite the marker LAST so a crash mid-reset re-runs the reset.
4002        self.persist_schema_version()?;
4003        self.save_unit_scope_schema()?;
4004        // §7.1/§7.4.4: the previous-version capture and audit above saw the
4005        // original pending intent; the synced tables now carry the new schema, so
4006        // recover intrinsically codec-invalid persisted values here, before the
4007        // final replay. A pending upsert that names a column the new schema
4008        // removed stays pending for the send-time `drop_incompatible_outbox`
4009        // prepass. The reset wrapper snapshots and restores the outbox,
4010        // rejections, and change queues, so a failed commit rolls this recovery
4011        // back with the rest of the reset.
4012        // §7.4.4: the enclosing reset observation owns the publication, so the
4013        // recovery contributes its rejection, outcome, and table domains to it.
4014        self.drop_unencodable_outbox(false, Some(batch))?;
4015        // Re-apply the surviving outbox optimistically over the empty tables.
4016        self.rebuild_overlay()?;
4017        Ok(())
4018    }
4019
4020    /// §7.4.4 send-time prepass: a persisted upsert whose values reference a
4021    /// column the current schema lacks (or a removed table) cannot be encoded
4022    /// (`commit_audit_operations` inspects upserts, so a value-free delete stays
4023    /// encodable and is validated by the server), nor can an insert-intent
4024    /// upsert that lacks a non-nullable column (§7.1, the audit's
4025    /// `missing-required-column`). Drop the commit, undo
4026    /// its optimistic projection, and raise a client-local
4027    /// `sync.outbox_incompatible` rejection.
4028    fn drop_incompatible_outbox(&mut self) -> Result<bool, String> {
4029        let incompatible = self
4030            .outbox
4031            .iter()
4032            .filter(|commit| {
4033                first_incompatibility(&self.schema, &commit_audit_operations(commit)).is_some()
4034            })
4035            .cloned()
4036            .collect::<Vec<_>>();
4037        if incompatible.is_empty() {
4038            return Ok(false);
4039        }
4040        self.drop_outbox_commits(
4041            &incompatible,
4042            OUTBOX_INCOMPATIBLE_CODE,
4043            "the persisted commit cannot encode under the current schema",
4044        )?;
4045        Ok(true)
4046    }
4047
4048    /// §7.4.4 incompatible-commit path, also taken by a refused blob upload
4049    /// (§5.9.7) with the server's code: reject every operation of `commits`,
4050    /// remove them from the outbox, and undo their optimistic rows.
4051    fn drop_outbox_commits(
4052        &mut self,
4053        incompatible: &[Arc<OutboxCommit>],
4054        code: &str,
4055        message: &str,
4056    ) -> Result<(), String> {
4057        // §7.4.4: one observation covers the durable outcome writes, the outbox
4058        // removal, the in-memory drop, and the overlay rebuild, so a storage
4059        // fault leaves the replica and the client unchanged and publishes no
4060        // change; a successful drop publishes exactly one batch carrying the
4061        // rejections, outcomes, status, and the tables whose projection changed.
4062        let rollback = self.capture();
4063        let owns_transaction = self.conn.is_autocommit();
4064        self.begin_observation("syncular_outbox_drop")?;
4065        let mut batch = ChangeAccumulator::default();
4066        let dropped = (|| -> Result<(), String> {
4067            let mut rejections = Vec::new();
4068            for commit in incompatible {
4069                let results = commit
4070                    .ops
4071                    .iter()
4072                    .enumerate()
4073                    .map(|(op_index, operation)| {
4074                        let rejection = RejectionRecord {
4075                            client_commit_id: commit.client_commit_id.clone(),
4076                            op_index: op_index as i32,
4077                            code: code.to_owned(),
4078                            message: message.to_owned(),
4079                            retryable: false,
4080                            details: None,
4081                            operation: Some(CommitOperation::from(operation)),
4082                        };
4083                        rejections.push(rejection.clone());
4084                        CommitOperationOutcome::Error { rejection }
4085                    })
4086                    .collect::<Vec<_>>();
4087                self.persist_commit_outcome(
4088                    &commit.client_commit_id,
4089                    CommitOutcomeStatus::Rejected,
4090                    &results,
4091                    Some(&commit.ops),
4092                )?;
4093                self.delete_outbox_persisted(&commit.client_commit_id)?;
4094                // A removed table has no local projection to invalidate; every
4095                // surviving table the dropped commit touched does.
4096                for operation in &commit.ops {
4097                    if self.schema.table(&operation.table).is_some() {
4098                        batch.table(&operation.table);
4099                    }
4100                }
4101            }
4102            self.prune_commit_outcomes()?;
4103            let incompatible_ids = incompatible
4104                .iter()
4105                .map(|commit| commit.client_commit_id.as_str())
4106                .collect::<BTreeSet<_>>();
4107            self.outbox
4108                .retain(|commit| !incompatible_ids.contains(commit.client_commit_id.as_str()));
4109            self.rejections.extend(rejections);
4110            // The dropped commits' purely-optimistic rows are undone: the
4111            // visible projection is re-derived from the surviving base + outbox.
4112            self.overlay_dirty.set(true);
4113            self.rebuild_overlay()?;
4114            batch.status = true;
4115            batch.rejections = true;
4116            batch.outcomes = true;
4117            Ok(())
4118        })();
4119        let result = dropped.and_then(|()| self.finish_observation("syncular_outbox_drop", batch));
4120        if let Err(error) = result {
4121            // An outer RELEASE is the COMMIT. Roll back that transaction
4122            // directly: retrying RELEASE after ROLLBACK TO still needs the
4123            // reader's lock. A nested observation only rolls back its savepoint.
4124            if owns_transaction {
4125                if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
4126                    let mut failure = self.storage_failure.borrow_mut();
4127                    let failure =
4128                        failure.get_or_insert_with(|| QueryReadFailure::from(error.clone()));
4129                    failure.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
4130                }
4131            } else {
4132                self.rollback_observation("syncular_outbox_drop");
4133            }
4134            self.restore(rollback);
4135            return Err(error);
4136        }
4137        Ok(())
4138    }
4139
4140    /// §7.4.3: (re)create the base + visible table pair for every synced
4141    /// table in the CURRENT schema (idempotent — `IF NOT EXISTS`).
4142    fn create_synced_tables(&self) -> Result<(), String> {
4143        for table in &self.schema.tables {
4144            // The base half + the visible half form the synced-table pair. An
4145            // index name is global in SQLite, so the base half's indexes are
4146            // name-prefixed (`_syncular_base_<index>`) to stay distinct.
4147            for (full, index_prefix) in [
4148                (base_table(&table.name), "_syncular_base_"),
4149                (visible_table(&table.name), ""),
4150            ] {
4151                let mut cols: Vec<String> =
4152                    table.columns.iter().map(|c| quote_ident(&c.name)).collect();
4153                cols.push("\"_syncular_version\" INTEGER NOT NULL".to_owned());
4154                cols.extend(
4155                    table
4156                        .sealed_columns()
4157                        .iter()
4158                        .map(|name| format!("{} TEXT", quote_ident(name))),
4159                );
4160                let sql = format!(
4161                    "CREATE TABLE IF NOT EXISTS {full} ({} , PRIMARY KEY ({}))",
4162                    cols.join(", "),
4163                    quote_ident(&table.primary_key)
4164                );
4165                self.conn
4166                    .execute(&sql, [])
4167                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4168                // §5.11: a replica created before sealed apply gains the
4169                // reserved columns in place, on both halves in the same order,
4170                // so data, indexes and the FTS projection stay.
4171                for name in table.sealed_columns() {
4172                    let present: bool = self
4173                        .conn
4174                        .query_row(
4175                            "SELECT EXISTS(SELECT 1 FROM pragma_table_info(?1) WHERE name = ?2)",
4176                            rusqlite::params![format!("{index_prefix}{}", table.name), name],
4177                            |row| row.get(0),
4178                        )
4179                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4180                    if !present {
4181                        self.conn
4182                            .execute(
4183                                &format!(
4184                                    "ALTER TABLE {full} ADD COLUMN {} TEXT",
4185                                    quote_ident(name)
4186                                ),
4187                                [],
4188                            )
4189                            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4190                    }
4191                }
4192                // The reveal at every round start reads the sealed rows of the
4193                // base half through this partial index.
4194                if table.has_encrypted_columns() && !index_prefix.is_empty() {
4195                    self.conn
4196                        .execute(
4197                            &format!(
4198                                "CREATE INDEX IF NOT EXISTS {} ON {full} (\"_sync_sealed_key\") WHERE \"_sync_sealed_key\" IS NOT NULL",
4199                                quote_ident(&format!("_syncular_sealed_{}", table.name))
4200                            ),
4201                            [],
4202                        )
4203                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4204                }
4205                // Local secondary indexes (CREATE INDEX subset). Created on
4206                // both halves so mirror reads hit an index on either. Runs on
4207                // both the initial create and the §7.4.3 reset recreate path.
4208                for index in &table.indexes {
4209                    let unique = if index.unique { "UNIQUE " } else { "" };
4210                    let index_name = quote_ident(&format!("{index_prefix}{}", index.name));
4211                    let cols_sql = index
4212                        .columns
4213                        .iter()
4214                        .map(|c| quote_ident(c))
4215                        .collect::<Vec<_>>()
4216                        .join(", ");
4217                    let index_sql = format!(
4218                        "CREATE {unique}INDEX IF NOT EXISTS {index_name} ON {full} ({cols_sql})"
4219                    );
4220                    self.conn
4221                        .execute(&index_sql, [])
4222                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4223                }
4224            }
4225        }
4226        // FTS exists only on the visible half. It is a local search
4227        // projection, never a synced/base table or a wire-schema member.
4228        for table in &self.schema.tables {
4229            for index in &table.fts_indexes {
4230                self.create_fts_projection(table, index)?;
4231            }
4232        }
4233        Ok(())
4234    }
4235
4236    fn fts_projection_exists(&self, index: &FtsIndexSchema) -> Result<bool, String> {
4237        let count: i64 = self
4238            .conn
4239            .query_row(
4240                "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?1",
4241                rusqlite::params![index.name],
4242                |row| row.get(0),
4243            )
4244            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4245        Ok(count > 0)
4246    }
4247
4248    fn drop_fts_triggers(&self, index: &FtsIndexSchema) -> Result<(), String> {
4249        for suffix in ["ai", "ad", "au"] {
4250            self.conn
4251                .execute(
4252                    &format!(
4253                        "DROP TRIGGER IF EXISTS {}",
4254                        quote_ident(&format!("{}_{suffix}", index.name))
4255                    ),
4256                    [],
4257                )
4258                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4259        }
4260        Ok(())
4261    }
4262
4263    fn create_fts_triggers(
4264        &self,
4265        table: &TableSchema,
4266        index: &FtsIndexSchema,
4267    ) -> Result<(), String> {
4268        let fts = quote_ident(&index.name);
4269        let source = visible_table(&table.name);
4270        let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
4271        let pk = quote_ident(&table.primary_key);
4272        let projection_columns = std::iter::once(source_id.clone())
4273            .chain(index.columns.iter().map(|column| quote_ident(column)))
4274            .collect::<Vec<_>>()
4275            .join(", ");
4276        let new_values = std::iter::once(format!("CAST(new.{pk} AS TEXT)"))
4277            .chain(
4278                index
4279                    .columns
4280                    .iter()
4281                    .map(|column| format!("new.{}", quote_ident(column))),
4282            )
4283            .collect::<Vec<_>>()
4284            .join(", ");
4285        let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
4286        let delete_new = format!("DELETE FROM {fts} WHERE rowid = (SELECT id FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)); DELETE FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)");
4287        let delete_old = format!("DELETE FROM {fts} WHERE rowid = (SELECT id FROM {mapping} WHERE source_id = CAST(old.{pk} AS TEXT)); DELETE FROM {mapping} WHERE source_id = CAST(old.{pk} AS TEXT)");
4288        let insert_new = format!("INSERT INTO {mapping}(source_id) VALUES (CAST(new.{pk} AS TEXT)); INSERT INTO {fts} (rowid, {projection_columns}) VALUES ((SELECT id FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)), {new_values})");
4289        let bi = quote_ident(&format!("{}_bi", index.name));
4290        let ai = quote_ident(&format!("{}_ai", index.name));
4291        let ad = quote_ident(&format!("{}_ad", index.name));
4292        let au = quote_ident(&format!("{}_au", index.name));
4293        let replacement_exists = format!("EXISTS (SELECT 1 FROM {source} WHERE {pk} = new.{pk})");
4294        let sql = format!(
4295            "DROP TRIGGER IF EXISTS {bi};
4296             DROP TRIGGER IF EXISTS {ai};
4297             DROP TRIGGER IF EXISTS {ad};
4298             DROP TRIGGER IF EXISTS {au};
4299             CREATE TRIGGER {bi} BEFORE INSERT ON {source} WHEN {replacement_exists} BEGIN {delete_new}; END;
4300             CREATE TRIGGER {ai} AFTER INSERT ON {source} BEGIN {insert_new}; END;
4301             CREATE TRIGGER {ad} AFTER DELETE ON {source} BEGIN {delete_old}; END;
4302             CREATE TRIGGER {au} AFTER UPDATE ON {source} BEGIN {delete_old}; {delete_new}; {insert_new}; END;",
4303        );
4304        self.conn
4305            .execute_batch(&sql)
4306            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
4307    }
4308
4309    fn rebuild_fts_projection(
4310        &self,
4311        table: &TableSchema,
4312        index: &FtsIndexSchema,
4313    ) -> Result<(), String> {
4314        let fts = quote_ident(&index.name);
4315        let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
4316        let pk = quote_ident(&table.primary_key);
4317        let indexed_columns = index
4318            .columns
4319            .iter()
4320            .map(|column| quote_ident(column))
4321            .collect::<Vec<_>>();
4322        let projection_columns = std::iter::once(source_id.clone())
4323            .chain(indexed_columns.iter().cloned())
4324            .collect::<Vec<_>>()
4325            .join(", ");
4326        let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
4327        let sql = format!(
4328            "DELETE FROM {fts}; DELETE FROM {mapping}; INSERT INTO {fts} ({projection_columns}) SELECT CAST({pk} AS TEXT), {columns} FROM {source}; INSERT INTO {mapping}(id, source_id) SELECT rowid, {source_id} FROM {fts};",
4329            columns = indexed_columns.join(", "),
4330            source = visible_table(&table.name),
4331        );
4332        self.conn
4333            .execute_batch(&sql)
4334            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
4335    }
4336
4337    fn create_fts_projection(
4338        &self,
4339        table: &TableSchema,
4340        index: &FtsIndexSchema,
4341    ) -> Result<(), String> {
4342        self.begin_observation("syncular_fts_schema")?;
4343        let result = (|| {
4344            let existed = self.fts_projection_exists(index)?;
4345            let tokenizer = index.tokenize.replace('\'', "''");
4346            let columns = index
4347                .columns
4348                .iter()
4349                .map(|column| quote_ident(column))
4350                .collect::<Vec<_>>()
4351                .join(", ");
4352            let sql = format!(
4353            "CREATE VIRTUAL TABLE IF NOT EXISTS {fts} USING fts5({source_id} UNINDEXED, {columns}, tokenize='{tokenizer}')",
4354            fts = quote_ident(&index.name),
4355            source_id = quote_ident(FTS_SOURCE_ID_COLUMN),
4356        );
4357            self.conn.execute(&sql, []).map_err(|error| {
4358                format!(
4359                    "cannot create local FTS5 projection {:?}: {error}",
4360                    index.name
4361                )
4362            })?;
4363            let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
4364            let mapped: bool = self
4365                .conn
4366                .query_row(
4367                    "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?1)",
4368                    [format!("_syncular_fts_{}", index.name)],
4369                    |row| row.get(0),
4370                )
4371                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4372            self.conn.execute_batch(&format!("CREATE TABLE IF NOT EXISTS {mapping}(id INTEGER PRIMARY KEY, source_id TEXT NOT NULL UNIQUE)")).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4373            if !mapped && existed {
4374                self.conn
4375                    .execute_batch(&format!(
4376                        "INSERT INTO {mapping}(id, source_id) SELECT rowid, {} FROM {}",
4377                        quote_ident(FTS_SOURCE_ID_COLUMN),
4378                        quote_ident(&index.name)
4379                    ))
4380                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4381            }
4382            self.create_fts_triggers(table, index)?;
4383            if !existed {
4384                self.rebuild_fts_projection(table, index)?;
4385            }
4386            Ok(())
4387        })();
4388        if let Err(error) = result {
4389            self.rollback_observation("syncular_fts_schema");
4390            return Err(error);
4391        }
4392        if let Err(error) = self.conn.execute_batch("RELEASE syncular_fts_schema") {
4393            self.rollback_observation("syncular_fts_schema");
4394            return Err(error.to_string());
4395        }
4396        Ok(())
4397    }
4398
4399    // -- persistence write-through --------------------------------------------
4400
4401    fn persist_outbox_insert(&self, commit: &OutboxCommit) -> Result<(), String> {
4402        let ops: Vec<Value> = commit
4403            .ops
4404            .iter()
4405            .map(|op| {
4406                let mut entry = serde_json::json!({
4407                    "op": if op.upsert { "upsert" } else { "delete" },
4408                    "table": op.table,
4409                    "rowId": op.row_id,
4410                    "baseVersion": op.base_version,
4411                    "values": op.values.clone().map(Value::Object),
4412                });
4413                if op.insert {
4414                    entry["insert"] = Value::Bool(true);
4415                }
4416                entry
4417            })
4418            .collect();
4419        self.conn
4420            .execute(
4421                "INSERT OR REPLACE INTO _syncular_outbox (commit_id, ops_json) VALUES (?1, ?2)",
4422                rusqlite::params![commit.client_commit_id, Value::Array(ops).to_string()],
4423            )
4424            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4425        for operation in &commit.ops {
4426            let Some(values) = &operation.values else {
4427                continue;
4428            };
4429            let Some(table) = self.schema.table(&operation.table) else {
4430                continue;
4431            };
4432            for column in table
4433                .columns
4434                .iter()
4435                .filter(|column| column.ty == ColumnType::BlobRef)
4436            {
4437                let Some(Value::String(raw)) = values.get(&column.name) else {
4438                    continue;
4439                };
4440                let Ok(blob_ref) = serde_json::from_str::<Value>(raw) else {
4441                    continue;
4442                };
4443                let Some(blob_id) = blob_ref.get("blobId").and_then(Value::as_str) else {
4444                    continue;
4445                };
4446                self.conn
4447                    .execute(
4448                        "INSERT OR IGNORE INTO _syncular_blob_commit_refs(commit_id, blob_id)
4449                         SELECT ?1, ?2 WHERE EXISTS (SELECT 1 FROM _syncular_blobs WHERE blob_id = ?2)",
4450                        rusqlite::params![commit.client_commit_id, blob_id],
4451                    )
4452                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4453            }
4454        }
4455        Ok(())
4456    }
4457
4458    fn delete_outbox_persisted(&self, client_commit_id: &str) -> Result<(), String> {
4459        self.conn
4460            .execute(
4461                "DELETE FROM _syncular_blob_commit_refs WHERE commit_id = ?1",
4462                rusqlite::params![client_commit_id],
4463            )
4464            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4465        self.conn
4466            .execute(
4467                "DELETE FROM _syncular_outbox WHERE commit_id = ?1",
4468                rusqlite::params![client_commit_id],
4469            )
4470            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4471        self.conn.execute("DELETE FROM _syncular_row_deliveries WHERE NOT EXISTS(SELECT 1 FROM _syncular_outbox,json_each(ops_json) WHERE json_extract(json_each.value,'$.table')=tbl AND json_extract(json_each.value,'$.rowId')=id)", [])
4472            .map(|_| ())
4473            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
4474    }
4475
4476    fn outcome_status_name(status: CommitOutcomeStatus) -> &'static str {
4477        match status {
4478            CommitOutcomeStatus::Applied => "applied",
4479            CommitOutcomeStatus::Cached => "cached",
4480            CommitOutcomeStatus::Conflict => "conflict",
4481            CommitOutcomeStatus::Rejected => "rejected",
4482        }
4483    }
4484
4485    fn outcome_resolution_name(resolution: CommitOutcomeResolution) -> &'static str {
4486        match resolution {
4487            CommitOutcomeResolution::Active => "active",
4488            CommitOutcomeResolution::ResolvedKeepServer => "resolved_keep_server",
4489            CommitOutcomeResolution::Superseded => "superseded",
4490            CommitOutcomeResolution::Dismissed => "dismissed",
4491        }
4492    }
4493
4494    fn parse_outcome_status(value: &str) -> Result<CommitOutcomeStatus, String> {
4495        match value {
4496            "applied" => Ok(CommitOutcomeStatus::Applied),
4497            "cached" => Ok(CommitOutcomeStatus::Cached),
4498            "conflict" => Ok(CommitOutcomeStatus::Conflict),
4499            "rejected" => Ok(CommitOutcomeStatus::Rejected),
4500            _ => Err(format!("invalid persisted commit outcome status {value:?}")),
4501        }
4502    }
4503
4504    fn parse_outcome_resolution(value: &str) -> Result<CommitOutcomeResolution, String> {
4505        match value {
4506            "active" => Ok(CommitOutcomeResolution::Active),
4507            "resolved_keep_server" => Ok(CommitOutcomeResolution::ResolvedKeepServer),
4508            "superseded" => Ok(CommitOutcomeResolution::Superseded),
4509            "dismissed" => Ok(CommitOutcomeResolution::Dismissed),
4510            _ => Err(format!(
4511                "invalid persisted commit outcome resolution {value:?}"
4512            )),
4513        }
4514    }
4515
4516    fn persist_commit_outcome(
4517        &self,
4518        client_commit_id: &str,
4519        status: CommitOutcomeStatus,
4520        results: &[CommitOperationOutcome],
4521        operations: Option<&[OutboxOp]>,
4522    ) -> Result<(), String> {
4523        let results_json = serde_json::to_string(results).map_err(|error| error.to_string())?;
4524        let operations_json = operations
4525            .map(|items| {
4526                serde_json::to_string(&items.iter().map(CommitOperation::from).collect::<Vec<_>>())
4527            })
4528            .transpose()
4529            .map_err(|error| error.to_string())?;
4530        self.conn
4531            .execute(
4532                "INSERT INTO _syncular_commit_outcomes (
4533                   client_commit_id, status, recorded_at_ms, results_json,
4534                   operations_json, resolution
4535                 ) VALUES (?1, ?2, ?3, ?4, ?5, 'active')",
4536                rusqlite::params![
4537                    client_commit_id,
4538                    Self::outcome_status_name(status),
4539                    self.clock_now_ms(),
4540                    results_json,
4541                    operations_json
4542                ],
4543            )
4544            .map(|_| ())
4545            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
4546    }
4547
4548    /// §7.2: the journal holds a stored cell only in its canonical shape. A cell
4549    /// the byte-envelope rule refuses cannot be journaled verbatim.
4550    fn stored_cell_is_representable(cell: &Value) -> bool {
4551        cell.is_null()
4552            || cell.is_string()
4553            || cell.is_boolean()
4554            || cell.is_number()
4555            || is_canonical_bytes_envelope(cell)
4556    }
4557
4558    fn stored_row_is_representable(value: &Value) -> bool {
4559        value
4560            .as_object()
4561            .is_some_and(|row| row.values().all(Self::stored_cell_is_representable))
4562    }
4563
4564    fn stored_operation_is_representable(value: &Value) -> bool {
4565        value.get("table").is_some_and(Value::is_string)
4566            && value.get("rowId").is_some_and(Value::is_string)
4567            && value.get("baseVersion").is_none_or(|v| {
4568                v.as_i64()
4569                    .is_some_and(|n| n.unsigned_abs() <= MAX_JS_SAFE_INTEGER)
4570            })
4571            && match value.get("op").and_then(Value::as_str) {
4572                Some("upsert") => {
4573                    value
4574                        .get("values")
4575                        .is_some_and(Self::stored_row_is_representable)
4576                        && value
4577                            .get("insert")
4578                            .is_none_or(|insert| *insert == Value::Bool(true))
4579                }
4580                Some("delete") => value.get("values").is_none() && value.get("insert").is_none(),
4581                _ => false,
4582            }
4583    }
4584
4585    /// §7.2/§7.4.4: a dropped commit keeps its operation envelope in the journal
4586    /// only when the journal reader accepts the serialized operation, which is the
4587    /// rule `outcome_from_row` applies to this same row later.
4588    fn outbox_op_is_journalable(operation: &OutboxOp) -> bool {
4589        serde_json::to_value(CommitOperation::from(operation))
4590            .is_ok_and(|value| Self::stored_operation_is_representable(&value))
4591    }
4592
4593    fn outcome_from_row(row: StoredCommitOutcomeRow) -> Result<CommitOutcome, String> {
4594        let StoredCommitOutcomeRow {
4595            sequence,
4596            client_commit_id,
4597            status,
4598            recorded_at_ms,
4599            results_json,
4600            operations_json,
4601            resolution,
4602            resolved_at_ms,
4603            replacement_client_commit_id,
4604        } = row;
4605        if sequence.unsigned_abs() > MAX_JS_SAFE_INTEGER
4606            || recorded_at_ms.unsigned_abs() > MAX_JS_SAFE_INTEGER
4607            || resolved_at_ms.is_some_and(|value| value.unsigned_abs() > MAX_JS_SAFE_INTEGER)
4608        {
4609            return Err("sync.local_corrupt: persisted commit outcome is invalid".to_owned());
4610        }
4611        let invalid = || "sync.local_corrupt: persisted commit outcome is invalid".to_owned();
4612        let stored: Vec<Value> = serde_json::from_str(&results_json).map_err(|_| invalid())?;
4613        for result in &stored {
4614            let nested = match result.get("status").and_then(Value::as_str) {
4615                Some("conflict") => result.get("conflict"),
4616                Some("error") => result.get("rejection"),
4617                _ => None,
4618            };
4619            if nested
4620                .and_then(|record| record.get("operation"))
4621                .is_some_and(|op| !Self::stored_operation_is_representable(op))
4622                || (result.get("status").and_then(Value::as_str) == Some("conflict")
4623                    && nested
4624                        .and_then(|record| record.get("serverRow"))
4625                        .is_some_and(|row| !Self::stored_row_is_representable(row)))
4626            {
4627                return Err(invalid());
4628            }
4629        }
4630        let results: Vec<CommitOperationOutcome> =
4631            serde_json::from_value(Value::Array(stored)).map_err(|_| invalid())?;
4632        let has_conflict = results
4633            .iter()
4634            .any(|result| matches!(result, CommitOperationOutcome::Conflict { .. }));
4635        let has_error = results
4636            .iter()
4637            .any(|result| matches!(result, CommitOperationOutcome::Error { .. }));
4638        let status = Self::parse_outcome_status(&status).map_err(|_| invalid())?;
4639        let redacted = results.is_empty() && operations_json.is_none();
4640        if match status {
4641            CommitOutcomeStatus::Applied | CommitOutcomeStatus::Cached => has_conflict || has_error,
4642            CommitOutcomeStatus::Conflict => !(has_conflict || redacted),
4643            CommitOutcomeStatus::Rejected => has_conflict || !(has_error || redacted),
4644        } {
4645            return Err(invalid());
4646        }
4647        let operations: Option<Vec<CommitOperation>> = operations_json
4648            .map(|raw| {
4649                let stored: Vec<Value> = serde_json::from_str(&raw).map_err(|_| invalid())?;
4650                if !stored.iter().all(Self::stored_operation_is_representable) {
4651                    return Err(invalid());
4652                }
4653                serde_json::from_value(Value::Array(stored)).map_err(|_| invalid())
4654            })
4655            .transpose()?;
4656        for result in &results {
4657            match result {
4658                CommitOperationOutcome::Conflict { conflict }
4659                    if conflict.server_version.unsigned_abs() > MAX_JS_SAFE_INTEGER =>
4660                {
4661                    return Err("sync.local_corrupt: persisted commit outcome is invalid".to_owned())
4662                }
4663                CommitOperationOutcome::Error { rejection } => {
4664                    if let Some(details) = &rejection.details {
4665                        let encoded = serde_json::to_string(details).map_err(|_| {
4666                            "sync.local_corrupt: persisted commit outcome is invalid".to_owned()
4667                        })?;
4668                        RejectionDetails::parse(&encoded).map_err(|_| {
4669                            "sync.local_corrupt: persisted commit outcome is invalid".to_owned()
4670                        })?;
4671                    }
4672                }
4673                _ => {}
4674            }
4675        }
4676        Ok(CommitOutcome {
4677            sequence,
4678            client_commit_id,
4679            status,
4680            recorded_at_ms,
4681            results,
4682            operations,
4683            retained_rows: None,
4684            resolution: Self::parse_outcome_resolution(&resolution)?,
4685            resolved_at_ms,
4686            replacement_client_commit_id,
4687        })
4688    }
4689
4690    /// Resolve declared unique keys through SQLite, including affinity and NULL semantics.
4691    fn unique_conflicts(
4692        &self,
4693        table: &TableSchema,
4694        values: &Map<String, Value>,
4695        base: bool,
4696    ) -> Result<Vec<RetainedUniqueConflict>, String> {
4697        let source = if base {
4698            base_table(&table.name)
4699        } else {
4700            visible_table(&table.name)
4701        };
4702        let mut conflicts = Vec::new();
4703        for index in table.indexes.iter().filter(|index| index.unique) {
4704            let mut params = vec![json_param_to_sql(
4705                values.get(&table.primary_key).ok_or("sync.local_corrupt")?,
4706            )?];
4707            for name in &index.columns {
4708                let column = table
4709                    .columns
4710                    .iter()
4711                    .find(|column| column.name == *name)
4712                    .ok_or("sync.local_corrupt")?;
4713                let cell = json_to_column_value(column, values.get(name))?;
4714                params.push(owned_sql_value(RowParam::Cell(&cell))?);
4715            }
4716            let sql = format!(
4717                "SELECT * FROM {source} WHERE {} != ? AND {}",
4718                quote_ident(&table.primary_key),
4719                index
4720                    .columns
4721                    .iter()
4722                    .map(|column| format!("{} = ?", quote_ident(column)))
4723                    .collect::<Vec<_>>()
4724                    .join(" AND ")
4725            );
4726            let mut statement = self
4727                .conn
4728                .prepare_cached(&sql)
4729                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4730            let records = statement
4731                .query_map(rusqlite::params_from_iter(params), |record| {
4732                    let values = table
4733                        .columns
4734                        .iter()
4735                        .enumerate()
4736                        .map(|(index, column)| {
4737                            let value = sql_ref_to_json(column, record.get_ref(index)?);
4738                            json_to_column_value(column, Some(&value)).map_err(|message| {
4739                                rusqlite::Error::ToSqlConversionFailure(message.into())
4740                            })
4741                        })
4742                        .collect::<Result<Row, _>>()?;
4743                    Ok((values, record.get::<_, i64>(table.columns.len())?))
4744                })
4745                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4746            for record in records {
4747                let (row, version) =
4748                    record.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4749                let row_id = render_row_id_json(Some(&column_value_to_json(&row[table.pk_index])))?;
4750                conflicts.push(RetainedUniqueConflict {
4751                    index: index.name.clone(),
4752                    columns: index.columns.clone(),
4753                    row_id,
4754                    server_row: table
4755                        .columns
4756                        .iter()
4757                        .zip(&row)
4758                        .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
4759                        .collect(),
4760                    server_version: version,
4761                });
4762            }
4763        }
4764        Ok(conflicts)
4765    }
4766
4767    fn attach_retained_rows(&self, mut outcome: CommitOutcome) -> Result<CommitOutcome, String> {
4768        if let Some((commit, initial)) = self
4769            .failed_commits
4770            .iter()
4771            .find(|(commit, _)| commit.client_commit_id == outcome.client_commit_id)
4772        {
4773            let mut rows = Vec::new();
4774            for operation in &commit.ops {
4775                // A table the current schema removed has no rows to retain.
4776                let Some(table) = self.schema.table(&operation.table) else {
4777                    continue;
4778                };
4779                let key =
4780                    serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
4781                        .map_err(|_| "sync.local_corrupt".to_owned())?;
4782                let base = self.stored_row(table, &operation.row_id, true)?;
4783                rows.push(RetainedCommitRow {
4784                    table: operation.table.clone(),
4785                    row_id: operation.row_id.clone(),
4786                    unique_conflicts: if operation.upsert {
4787                        self.unique_conflicts(
4788                            table,
4789                            initial
4790                                .get(&key)
4791                                .and_then(Value::as_object)
4792                                .or(operation.values.as_ref())
4793                                .ok_or("sync.local_corrupt")?,
4794                            true,
4795                        )?
4796                    } else {
4797                        Vec::new()
4798                    },
4799                    local_row: if operation.upsert {
4800                        initial
4801                            .get(&key)
4802                            .and_then(Value::as_object)
4803                            .cloned()
4804                            .or_else(|| operation.values.clone())
4805                    } else {
4806                        None
4807                    },
4808                    server_row: base.as_ref().map(|(values, _)| {
4809                        table
4810                            .columns
4811                            .iter()
4812                            .zip(values)
4813                            .map(|(column, value)| {
4814                                (column.name.clone(), column_value_to_json(value))
4815                            })
4816                            .collect()
4817                    }),
4818                    server_version: base.map(|(_, version)| version),
4819                });
4820            }
4821            outcome.retained_rows = Some(rows);
4822        }
4823        Ok(outcome)
4824    }
4825
4826    pub fn commit_outcome(&self, client_commit_id: &str) -> Result<Option<CommitOutcome>, String> {
4827        let row = self
4828            .conn
4829            .query_row(
4830                "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
4831                        resolution, resolved_at_ms, replacement_client_commit_id
4832                   FROM _syncular_commit_outcomes WHERE client_commit_id = ?1",
4833                rusqlite::params![client_commit_id],
4834                |row| {
4835                    Ok(StoredCommitOutcomeRow {
4836                        sequence: row.get(0)?,
4837                        client_commit_id: row.get(1)?,
4838                        status: row.get(2)?,
4839                        recorded_at_ms: row.get(3)?,
4840                        results_json: row.get(4)?,
4841                        operations_json: row.get(5)?,
4842                        resolution: row.get(6)?,
4843                        resolved_at_ms: row.get(7)?,
4844                        replacement_client_commit_id: row.get(8)?,
4845                    })
4846                },
4847            )
4848            .optional()
4849            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4850        row.map(|row| self.attach_retained_rows(Self::outcome_from_row(row)?))
4851            .transpose()
4852    }
4853
4854    /// Internal host step. The host must run its idle drive before finishing.
4855    #[doc(hidden)]
4856    pub fn prepare_current_base(
4857        &self,
4858        client_commit_id: &str,
4859        sequence: i64,
4860    ) -> Result<CurrentBasePreparation, CurrentBaseProof> {
4861        let outcome = self
4862            .commit_outcome(client_commit_id)
4863            .map_err(|_| CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed))?;
4864        let valid = outcome.as_ref().is_some_and(|outcome| {
4865            outcome.sequence == sequence
4866                && outcome.resolution == CommitOutcomeResolution::Active
4867                && matches!(
4868                    outcome.status,
4869                    CommitOutcomeStatus::Conflict | CommitOutcomeStatus::Rejected
4870                )
4871                && outcome
4872                    .operations
4873                    .as_ref()
4874                    .zip(outcome.retained_rows.as_ref())
4875                    .is_some_and(|(operations, rows)| {
4876                        !rows.is_empty() && operations.len() == rows.len()
4877                    })
4878        });
4879        if !valid {
4880            return Err(CurrentBaseProof::failure(
4881                CurrentBaseProofReason::OutcomeUnavailable,
4882            ));
4883        }
4884        if !self.transport_enabled {
4885            return Err(CurrentBaseProof::failure(CurrentBaseProofReason::Offline));
4886        }
4887        Ok(CurrentBasePreparation {
4888            client_commit_id: client_commit_id.to_owned(),
4889            sequence,
4890            units: load_units(&self.conn)
4891                .map_err(|_| CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed))?,
4892            transport_epoch: self.transport_epoch,
4893            registration_epoch: self.registration_epoch,
4894            security_epoch: self.security_epoch,
4895        })
4896    }
4897
4898    /// Internal host step after the completed post-rejection idle drive.
4899    #[doc(hidden)]
4900    pub fn finish_current_base(
4901        &mut self,
4902        prepared: CurrentBasePreparation,
4903        drive: SyncOutcome,
4904    ) -> CurrentBaseProof {
4905        if !self.transport_enabled || self.transport_epoch != prepared.transport_epoch {
4906            return CurrentBaseProof::failure(CurrentBaseProofReason::Offline);
4907        }
4908        if self.registration_epoch != prepared.registration_epoch {
4909            return CurrentBaseProof::failure(CurrentBaseProofReason::MissingCoverage);
4910        }
4911        if self.security_preflight || self.security_epoch != prepared.security_epoch {
4912            return CurrentBaseProof::failure(CurrentBaseProofReason::SyncFailed);
4913        }
4914        let report = match drive {
4915            SyncOutcome::Ok(report) => report,
4916            SyncOutcome::BudgetExhausted(_) => {
4917                return CurrentBaseProof::failure(CurrentBaseProofReason::BudgetExhausted)
4918            }
4919            _ => return CurrentBaseProof::failure(CurrentBaseProofReason::SyncFailed),
4920        };
4921        if report.schema_floor.is_some() {
4922            return CurrentBaseProof::failure(CurrentBaseProofReason::SchemaFloor);
4923        }
4924        if self
4925            .conn
4926            .execute_batch("SAVEPOINT syncular_base_proof")
4927            .is_err()
4928        {
4929            return CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed);
4930        }
4931        let proof = (|| {
4932            let snapshot = snapshot_read_connection(
4933                &self.conn,
4934                &SnapshotReadRequest {
4935                    units: prepared.units.iter().map(|unit| unit.id.clone()).collect(),
4936                    commit_ids: vec![prepared.client_commit_id.clone()],
4937                    ..Default::default()
4938                },
4939            )
4940            .map_err(|_| CurrentBaseProofReason::SnapshotFailed)?;
4941            let current = self
4942                .commit_outcome(&prepared.client_commit_id)
4943                .map_err(|_| CurrentBaseProofReason::SnapshotFailed)?;
4944            let unavailable = if report.revoked.is_empty() {
4945                CurrentBaseProofReason::OutcomeUnavailable
4946            } else {
4947                CurrentBaseProofReason::Revoked
4948            };
4949            let Some(outcome) = current else {
4950                return Err(unavailable);
4951            };
4952            if !matches!(
4953                snapshot.deliveries.first(),
4954                Some(CommitDelivery::Known { .. })
4955            ) || outcome.sequence != prepared.sequence
4956                || outcome.resolution != CommitOutcomeResolution::Active
4957                || !matches!(
4958                    outcome.status,
4959                    CommitOutcomeStatus::Conflict | CommitOutcomeStatus::Rejected
4960                )
4961            {
4962                return Err(unavailable);
4963            }
4964            let (Some(operations), Some(rows)) = (outcome.operations, outcome.retained_rows) else {
4965                return Err(unavailable);
4966            };
4967            if rows.is_empty() || operations.len() != rows.len() {
4968                return Err(unavailable);
4969            }
4970            let after =
4971                load_units(&self.conn).map_err(|_| CurrentBaseProofReason::SnapshotFailed)?;
4972            let complete: HashSet<&str> = snapshot
4973                .units
4974                .iter()
4975                .filter_map(|unit| match unit {
4976                    UnitCatchup::Known {
4977                        id,
4978                        bootstrap_complete: true,
4979                        ..
4980                    } => Some(id.as_str()),
4981                    _ => None,
4982                })
4983                .collect();
4984            // A unit id names one scope map (§4.1), so the same id before and
4985            // after the drive is the same registration.
4986            let candidates: Vec<&UnitRecord> = after
4987                .iter()
4988                .filter(|unit| {
4989                    complete.contains(unit.id.as_str())
4990                        && prepared.units.iter().any(|before| before.id == unit.id)
4991                })
4992                .collect();
4993            let covered = |table_name: &str, row: &Map<String, Value>| {
4994                let Some(table) = self.schema.table(table_name) else {
4995                    return false;
4996                };
4997                candidates.iter().any(|unit| {
4998                    covers(table, &unit.scopes)
4999                        && pair_scopes(&unit.scopes, table_name, unit.echo.as_deref()).is_some_and(
5000                            |scopes| {
5001                                scopes.iter().all(|(variable, allowed)| {
5002                                    table
5003                                        .scope_column(variable)
5004                                        .and_then(|column| match row.get(column) {
5005                                            Some(Value::String(value)) => Some(value.clone()),
5006                                            Some(Value::Bool(value)) => Some(value.to_string()),
5007                                            Some(Value::Number(value)) => {
5008                                                value.as_f64().map(|number| {
5009                                                    ryu_js::Buffer::new()
5010                                                        .format_finite(number)
5011                                                        .to_owned()
5012                                                })
5013                                            }
5014                                            _ => None,
5015                                        })
5016                                        .is_some_and(|value| {
5017                                            !value.is_empty() && allowed.contains(&value)
5018                                        })
5019                                })
5020                            },
5021                        )
5022                })
5023            };
5024            for row in &rows {
5025                let Some(base) = row.server_row.as_ref().or(row.local_row.as_ref()) else {
5026                    return Err(if report.revoked.is_empty() {
5027                        CurrentBaseProofReason::MissingCoverage
5028                    } else {
5029                        CurrentBaseProofReason::Revoked
5030                    });
5031                };
5032                if !covered(&row.table, base)
5033                    || row
5034                        .unique_conflicts
5035                        .iter()
5036                        .any(|conflict| !covered(&row.table, &conflict.server_row))
5037                {
5038                    return Err(if report.revoked.is_empty() {
5039                        CurrentBaseProofReason::MissingCoverage
5040                    } else {
5041                        CurrentBaseProofReason::Revoked
5042                    });
5043                }
5044            }
5045            Ok(rows)
5046        })();
5047        if self
5048            .conn
5049            .execute_batch("RELEASE syncular_base_proof")
5050            .is_err()
5051        {
5052            return CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed);
5053        }
5054        match proof {
5055            Ok(rows) => CurrentBaseProof {
5056                proven: true,
5057                rows: Some(rows),
5058                reason: None,
5059            },
5060            Err(reason) => CurrentBaseProof::failure(reason),
5061        }
5062    }
5063
5064    pub fn prove_current_base(
5065        &mut self,
5066        transport: &mut dyn Transport,
5067        client_commit_id: &str,
5068        sequence: i64,
5069        max_rounds: Option<u32>,
5070    ) -> CurrentBaseProof {
5071        let prepared = match self.prepare_current_base(client_commit_id, sequence) {
5072            Ok(prepared) => prepared,
5073            Err(proof) => return proof,
5074        };
5075        let drive = self.sync_until_idle(transport, max_rounds);
5076        self.finish_current_base(prepared, drive)
5077    }
5078
5079    pub fn commit_outcomes(&self, query: CommitOutcomeQuery) -> Result<Vec<CommitOutcome>, String> {
5080        if query.limit == Some(0) {
5081            return Err("sync.invalid_request: commit outcome limit must be positive".to_owned());
5082        }
5083        let mut sql = String::from(
5084            "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
5085                    resolution, resolved_at_ms, replacement_client_commit_id
5086               FROM _syncular_commit_outcomes",
5087        );
5088        if query.active_only {
5089            sql.push_str(" WHERE resolution = 'active' AND status IN ('conflict', 'rejected')");
5090        }
5091        sql.push_str(" ORDER BY seq DESC");
5092        if let Some(limit) = query.limit {
5093            sql.push_str(&format!(" LIMIT {limit}"));
5094        }
5095        let mut stmt = self
5096            .conn
5097            .prepare(&sql)
5098            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5099        let rows = stmt
5100            .query_map([], |row| {
5101                Ok(StoredCommitOutcomeRow {
5102                    sequence: row.get(0)?,
5103                    client_commit_id: row.get(1)?,
5104                    status: row.get(2)?,
5105                    recorded_at_ms: row.get(3)?,
5106                    results_json: row.get(4)?,
5107                    operations_json: row.get(5)?,
5108                    resolution: row.get(6)?,
5109                    resolved_at_ms: row.get(7)?,
5110                    replacement_client_commit_id: row.get(8)?,
5111                })
5112            })
5113            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5114        let mut outcomes = Vec::new();
5115        for row in rows {
5116            outcomes.push(self.attach_retained_rows(Self::outcome_from_row(
5117                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
5118            )?)?);
5119        }
5120        Ok(outcomes)
5121    }
5122
5123    fn prune_commit_outcomes(&self) -> Result<(), String> {
5124        #[cfg(test)]
5125        self.outcome_prune_count
5126            .set(self.outcome_prune_count.get() + 1);
5127        let max_entries = self.limits.outcome_retention_max_entries.unwrap_or(1_000);
5128        let count = self
5129            .conn
5130            .query_row(
5131                "SELECT COUNT(*) FROM _syncular_commit_outcomes",
5132                [],
5133                |row| row.get::<_, i64>(0),
5134            )
5135            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
5136            as usize;
5137        let excess = count.saturating_sub(max_entries);
5138        if excess == 0 {
5139            return Ok(());
5140        }
5141        let mut stmt = self
5142            .conn
5143            .prepare(
5144                "SELECT seq FROM _syncular_commit_outcomes
5145                  WHERE status IN ('applied', 'cached') OR resolution != 'active'
5146                  ORDER BY seq ASC LIMIT ?1",
5147            )
5148            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5149        let rows = stmt
5150            .query_map(rusqlite::params![excess as i64], |row| row.get::<_, i64>(0))
5151            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5152        let sequences = rows
5153            .collect::<Result<Vec<_>, _>>()
5154            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5155        drop(stmt);
5156        for sequence in sequences {
5157            self.conn
5158                .execute(
5159                    "DELETE FROM _syncular_commit_outcomes WHERE seq = ?1",
5160                    rusqlite::params![sequence],
5161                )
5162                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5163        }
5164        Ok(())
5165    }
5166
5167    // -- units (§4.1) ------------------------------------------------------------
5168
5169    /// Every registered unit in registration order (§4.1).
5170    pub fn units(&self) -> Result<Vec<UnitRecord>, String> {
5171        self.stored(load_units(&self.conn))
5172    }
5173
5174    pub fn unit(&self, id: &str) -> Result<Option<UnitRecord>, String> {
5175        self.stored(get_unit(&self.conn, id))
5176    }
5177
5178    /// Register one unit (§4.1) and return its id.
5179    pub fn subscribe(&mut self, scopes: UnitScopes) -> Result<String, String> {
5180        Ok(self.subscribe_many(vec![scopes])?.remove(0))
5181    }
5182
5183    /// Register several units in one local transaction and return their ids:
5184    /// one commit for all of them (on OPFS a commit costs milliseconds, so a
5185    /// cold start with hundreds of units pays it once). An error rolls back
5186    /// every registration of the call. Registering an active unit again
5187    /// changes nothing; a revoked unit re-arms (§3.3). A call that registers or
5188    /// re-arms a unit raises sync-needed with an interactive intent.
5189    pub fn subscribe_many(&mut self, units: Vec<UnitScopes>) -> Result<Vec<String>, String> {
5190        self.begin_observation("syncular_subscribe")?;
5191        let mut changed = false;
5192        let registered = (|| {
5193            let mut ids = Vec::with_capacity(units.len());
5194            for scopes in units {
5195                let (id, held) = self.hold_unit(scopes, true)?;
5196                changed |= held;
5197                ids.push(id);
5198            }
5199            self.conn
5200                .execute_batch("RELEASE syncular_subscribe")
5201                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5202            Ok(ids)
5203        })();
5204        let ids = match registered {
5205            Ok(ids) => ids,
5206            Err(error) => {
5207                self.rollback_observation("syncular_subscribe");
5208                return Err(error);
5209            }
5210        };
5211        if changed {
5212            self.registration_epoch = self.registration_epoch.wrapping_add(1);
5213            self.set_sync_needed(true, true);
5214        }
5215        Ok(ids)
5216    }
5217
5218    /// Release the explicit reference to a unit. The unit leaves the
5219    /// registration list when no window base holds it (§4.1); its rows stay.
5220    pub fn unsubscribe(&mut self, id: &str) -> Result<(), String> {
5221        let Some(unit) = self.stored(get_unit(&self.conn, id))? else {
5222            return Ok(());
5223        };
5224        if !unit.explicit {
5225            return Ok(());
5226        }
5227        if !self.stored(window_refs(&self.conn, id))?.is_empty() {
5228            return self.stored(save_unit(
5229                &self.conn,
5230                &UnitRecord {
5231                    explicit: false,
5232                    ..unit
5233                },
5234            ));
5235        }
5236        self.conn
5237            .execute("DELETE FROM _syncular_units WHERE id = ?1", [id])
5238            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5239        self.registration_epoch = self.registration_epoch.wrapping_add(1);
5240        if unit.status == UnitStatus::Active {
5241            self.set_sync_needed(true, true);
5242        }
5243        Ok(())
5244    }
5245
5246    /// Take one reference on a unit (§4.1): validate the scope map, register a
5247    /// new unit catching up with cursor `-1`, re-arm a revoked one (§3.3).
5248    /// Returns the id and whether the registration list changed.
5249    fn hold_unit(&mut self, scopes: UnitScopes, explicit: bool) -> Result<(String, bool), String> {
5250        if scopes.is_empty()
5251            || scopes.values().any(|value| value == "*")
5252            || !self
5253                .schema
5254                .tables
5255                .iter()
5256                .any(|table| covers(table, &scopes))
5257        {
5258            return Err("sync.invalid_subscription: a unit names one non-wildcard value per variable and covers a table".to_owned());
5259        }
5260        let id = ssp2::registration::unit_id(&scope_pairs(&scopes));
5261        let existing = self.stored(get_unit(&self.conn, &id))?;
5262        if let Some(existing) = existing
5263            .as_ref()
5264            .filter(|unit| unit.status != UnitStatus::Revoked)
5265        {
5266            if explicit && !existing.explicit {
5267                self.stored(save_unit(
5268                    &self.conn,
5269                    &UnitRecord {
5270                        explicit,
5271                        ..existing.clone()
5272                    },
5273                ))?;
5274            }
5275            return Ok((id, false));
5276        }
5277        self.delete_pending_evict(&id)?;
5278        self.stored(save_unit(
5279            &self.conn,
5280            &UnitRecord {
5281                id: id.clone(),
5282                scopes,
5283                explicit: explicit || existing.is_some_and(|unit| unit.explicit),
5284                status: UnitStatus::Active,
5285                reason_code: None,
5286                cursor: Some(-1),
5287                bootstrap_state: None,
5288                seeded: Vec::new(),
5289                echo: None,
5290            },
5291        ))?;
5292        Ok((id, true))
5293    }
5294
5295    // -- windows (§4.8) -----------------------------------------------------------
5296
5297    /// §4.8: set the windowed-in values of a base, one unit per value. Added
5298    /// values take a reference on their unit (a new unit bootstraps on the
5299    /// next sync); removed values release theirs, and a unit no reference
5300    /// holds leaves the registration list fused with its first eviction chunk
5301    /// (E1–E4). Idempotent; re-entry cancels a deferred eviction.
5302    pub fn set_window(
5303        &mut self,
5304        base: &WindowBase,
5305        units: &[String],
5306    ) -> Result<CommandEffects, String> {
5307        let base_key = window_base_key(base);
5308        let live = self.load_window_units(&base_key)?;
5309        let mut changed = false;
5310        for value in units {
5311            if live.iter().any(|(held, _)| held == value) {
5312                continue;
5313            }
5314            let rollback = self.capture();
5315            self.begin_observation("syncular_window")?;
5316            let widened = (|| {
5317                let (id, registered) = self.hold_unit(window_unit_scopes(base, value), false)?;
5318                self.sync_needed |= registered;
5319                self.conn
5320                    .execute(
5321                        "INSERT OR REPLACE INTO _syncular_windows(base, unit, sub_id) VALUES (?1, ?2, ?3)",
5322                        rusqlite::params![base_key, value, id],
5323                    )
5324                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5325                let mut batch = ChangeAccumulator::default();
5326                batch.window(&base_key, value);
5327                batch.status = true;
5328                self.finish_observation("syncular_window", batch)
5329            })();
5330            if let Err(error) = widened {
5331                self.rollback_observation("syncular_window");
5332                self.restore(rollback);
5333                return Err(error);
5334            }
5335            changed = true;
5336            self.registration_epoch = self.registration_epoch.wrapping_add(1);
5337        }
5338        for (value, id) in live {
5339            if units.contains(&value) {
5340                continue;
5341            }
5342            self.rebuild_overlay_if_dirty()?;
5343            let rollback = self.capture();
5344            self.begin_observation("syncular_window")?;
5345            let remaining = self.release_window_unit(&base_key, &value, &id);
5346            let remaining = match remaining {
5347                Ok(remaining) => remaining,
5348                Err(error) => {
5349                    self.rollback_observation("syncular_window");
5350                    self.restore(rollback);
5351                    return Err(error);
5352                }
5353            };
5354            changed = true;
5355            self.registration_epoch = self.registration_epoch.wrapping_add(1);
5356            if remaining {
5357                self.drain_pending_evictions()?;
5358            }
5359        }
5360        Ok(if changed && self.transport_enabled {
5361            CommandEffects::interactive()
5362        } else {
5363            CommandEffects::none()
5364        })
5365    }
5366
5367    /// §4.8 completeness oracle: the windowed-in values of a base and those
5368    /// whose unit is not complete. A value is complete once its unit is active
5369    /// and no longer bootstrapping; registration alone is not completeness.
5370    pub fn window_state(&self, base: &WindowBase) -> Result<WindowState, String> {
5371        persisted_window_state(&self.conn, base).map_err(|failure| failure.message)
5372    }
5373
5374    /// Registry rows for a base, ordered by value: (value, unit id).
5375    fn load_window_units(&self, base_key: &str) -> Result<Vec<(String, String)>, String> {
5376        let mut statement = self
5377            .conn
5378            .prepare_cached(
5379                "SELECT unit, sub_id FROM _syncular_windows WHERE base = ?1 ORDER BY unit ASC",
5380            )
5381            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5382        let rows = statement
5383            .query_map([base_key], |row| Ok((row.get(0)?, row.get(1)?)))
5384            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5385        rows.collect::<Result<_, _>>()
5386            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5387    }
5388
5389    /// Every window registry row: (base key, value).
5390    fn load_window_registry(&self) -> Result<Vec<(String, String)>, String> {
5391        let mut statement = self
5392            .conn
5393            .prepare_cached("SELECT base, unit FROM _syncular_windows ORDER BY base, unit")
5394            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5395        let rows = statement
5396            .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))
5397            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5398        rows.collect::<Result<_, _>>()
5399            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5400    }
5401
5402    /// §4.8: release a window value's reference inside the caller's
5403    /// observation. When no reference holds the unit, unregister it fused with
5404    /// its first eviction chunk (E3): delete the rows of every covered table
5405    /// matching the persisted echo, except rows pinned by the outbox (E1) or
5406    /// owned by another unit (§3.3), and park the per-table scopes of what
5407    /// remains in the pending-eviction registry. Fails closed without a local
5408    /// scope-column mapping. Returns whether unpinned rows remain.
5409    fn release_window_unit(
5410        &mut self,
5411        base_key: &str,
5412        value: &str,
5413        id: &str,
5414    ) -> Result<bool, String> {
5415        self.conn
5416            .execute(
5417                "DELETE FROM _syncular_windows WHERE base = ?1 AND unit = ?2",
5418                rusqlite::params![base_key, value],
5419            )
5420            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5421        let mut batch = ChangeAccumulator::default();
5422        batch.window(base_key, value);
5423        batch.status = true;
5424        let unit = self.stored(get_unit(&self.conn, id))?;
5425        let mut remaining = false;
5426        if let Some(unit) = unit.filter(|unit| !unit.explicit) {
5427            if self.stored(window_refs(&self.conn, id))?.is_empty() {
5428                let units = self.stored(load_units(&self.conn))?;
5429                self.conn
5430                    .execute("DELETE FROM _syncular_units WHERE id = ?1", [id])
5431                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5432                self.sync_needed = true;
5433                for table in self.schema.tables.clone() {
5434                    let Some(effective) = unit
5435                        .echo
5436                        .as_deref()
5437                        .filter(|_| covers(&table, &unit.scopes))
5438                        .and_then(|echo| pair_scopes(&unit.scopes, &table.name, Some(echo)))
5439                    else {
5440                        continue;
5441                    };
5442                    let pinned = self.pinned_row_ids(&table.name);
5443                    let owners = unit_owners(&units, &table, id);
5444                    let (more, deferred) =
5445                        self.evict_scope_rows(&table.name, &effective, &owners, &pinned)?;
5446                    remaining |= more;
5447                    if deferred {
5448                        self.save_pending_evict(id, &table.name, &effective)?;
5449                    }
5450                    self.record_scope_map(&mut batch, &table.name, &effective);
5451                }
5452            }
5453        }
5454        self.finish_observation("syncular_window", batch)?;
5455        Ok(remaining)
5456    }
5457
5458    /// §4.8 E1: delete base rows owned by the effective scopes
5459    /// ([`local_scope_predicate`]) EXCEPT pinned primary keys. Returns (more
5460    /// unpinned work, any scoped rows remain). A missing scope-column mapping
5461    /// fails without deleting rows.
5462    fn evict_scope_rows(
5463        &self,
5464        table_name: &str,
5465        effective: &[(String, Vec<String>)],
5466        owners: &[Vec<(String, Vec<String>)>],
5467        pinned: &std::collections::HashSet<String>,
5468    ) -> Result<(bool, bool), String> {
5469        let table = self.schema.table(table_name).ok_or("sync.unknown_table")?;
5470        let Some((scope, mut params)) = local_scope_predicate(table, effective, owners)
5471            .map_err(|()| "sync.scope_revoked".to_owned())?
5472        else {
5473            return Ok((false, false));
5474        };
5475        let scope_params = params.len();
5476        let pk = quote_ident(&table.primary_key);
5477        // Pinned ids are rowId text; the text form matches every key type.
5478        let predicate = if pinned.is_empty() {
5479            scope.clone()
5480        } else {
5481            params.extend(pinned.iter().cloned().map(SqlValue::Text));
5482            let holes = vec!["?"; pinned.len()].join(", ");
5483            format!("{scope} AND CAST({pk} AS TEXT) NOT IN ({holes})")
5484        };
5485        let base = base_table(table_name);
5486        let visible = visible_table(table_name);
5487        let mut ids: Vec<SqlValue> = Vec::new();
5488        for source in [&base, &visible] {
5489            let mut stmt = self
5490                .conn
5491                .prepare_cached(&format!(
5492                    "SELECT {pk} FROM {source} WHERE {predicate} LIMIT 1024"
5493                ))
5494                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5495            ids = stmt
5496                .query_map(rusqlite::params_from_iter(&params), |row| row.get(0))
5497                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
5498                .collect::<Result<_, _>>()
5499                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5500            if !ids.is_empty() {
5501                break;
5502            }
5503        }
5504        if !ids.is_empty() {
5505            for id in &ids {
5506                self.conn
5507                    .execute(
5508                        "DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2",
5509                        rusqlite::params![table_name, id],
5510                    )
5511                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5512            }
5513            let holes = vec!["?"; ids.len()].join(", ");
5514            for source in [&base, &visible] {
5515                self.conn
5516                    .execute(
5517                        &format!("DELETE FROM {source} WHERE {pk} IN ({holes})"),
5518                        rusqlite::params_from_iter(&ids),
5519                    )
5520                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5521            }
5522        }
5523        let mut remaining = false;
5524        let mut deferred = false;
5525        for source in [&base, &visible] {
5526            remaining |= self
5527                .conn
5528                .query_row(
5529                    &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {predicate})"),
5530                    rusqlite::params_from_iter(&params),
5531                    |row| row.get::<_, bool>(0),
5532                )
5533                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5534            deferred |= self
5535                .conn
5536                .query_row(
5537                    &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {scope})"),
5538                    rusqlite::params_from_iter(&params[..scope_params]),
5539                    |row| row.get::<_, bool>(0),
5540                )
5541                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5542        }
5543        Ok((remaining, deferred))
5544    }
5545
5546    /// §4.8 E1: continue deferred evictions after the outbox drains or between
5547    /// chunks, retaining outbox pins and their durable marker. A unit that
5548    /// re-entered the window has no pending entry left.
5549    fn drain_pending_evictions(&mut self) -> Result<(), String> {
5550        let pending = self.load_pending_evictions()?;
5551        if pending.is_empty() {
5552            return Ok(());
5553        }
5554        self.rebuild_overlay_if_dirty()?;
5555        for (unit_id, table_name, effective) in pending {
5556            let Some(table) = self.schema.table(&table_name).cloned() else {
5557                self.delete_pending_evict_table(&unit_id, &table_name)?;
5558                continue;
5559            };
5560            loop {
5561                std::thread::yield_now();
5562                self.begin_observation("syncular_evict")?;
5563                let chunk = (|| {
5564                    let pinned = self.pinned_row_ids(&table_name);
5565                    let owners =
5566                        unit_owners(&self.stored(load_units(&self.conn))?, &table, &unit_id);
5567                    let (remaining, deferred) =
5568                        self.evict_scope_rows(&table_name, &effective, &owners, &pinned)?;
5569                    if !deferred {
5570                        self.delete_pending_evict_table(&unit_id, &table_name)?;
5571                    }
5572                    let mut batch = ChangeAccumulator::default();
5573                    self.record_scope_map(&mut batch, &table_name, &effective);
5574                    self.finish_observation("syncular_evict", batch)?;
5575                    Ok(remaining)
5576                })();
5577                match chunk {
5578                    Ok(true) => {}
5579                    Ok(false) => break,
5580                    Err(error) => {
5581                        self.rollback_observation("syncular_evict");
5582                        return Err(error);
5583                    }
5584                }
5585            }
5586        }
5587        Ok(())
5588    }
5589
5590    /// §4.8 E1: primary keys of `table` referenced by a pending outbox
5591    /// commit — rows that MUST NOT be evicted until the commit drains.
5592    fn pinned_row_ids(&self, table: &str) -> std::collections::HashSet<String> {
5593        let mut pinned = std::collections::HashSet::new();
5594        for commit in self
5595            .outbox
5596            .iter()
5597            .map(|commit| &**commit)
5598            .chain(self.failed_commits.iter().map(|(commit, _)| commit))
5599        {
5600            for op in &commit.ops {
5601                if op.table == table {
5602                    pinned.insert(op.row_id.clone());
5603                }
5604            }
5605        }
5606        pinned
5607    }
5608
5609    fn save_pending_evict(
5610        &self,
5611        unit_id: &str,
5612        table: &str,
5613        effective: &[(String, Vec<String>)],
5614    ) -> Result<(), String> {
5615        self.conn.execute(
5616            "INSERT OR REPLACE INTO _syncular_pending_evictions(unit_id, tbl, effective_scopes) VALUES (?1, ?2, ?3)",
5617            rusqlite::params![unit_id, table, scope_map_to_json(effective).to_string()],
5618        ).map(|_| ()).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5619    }
5620
5621    /// Re-entry cancels every pending eviction of the unit.
5622    fn delete_pending_evict(&self, unit_id: &str) -> Result<(), String> {
5623        self.conn
5624            .execute(
5625                "DELETE FROM _syncular_pending_evictions WHERE unit_id = ?1",
5626                [unit_id],
5627            )
5628            .map(|_| ())
5629            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5630    }
5631
5632    fn delete_pending_evict_table(&self, unit_id: &str, table: &str) -> Result<(), String> {
5633        self.conn
5634            .execute(
5635                "DELETE FROM _syncular_pending_evictions WHERE unit_id = ?1 AND tbl = ?2",
5636                [unit_id, table],
5637            )
5638            .map(|_| ())
5639            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5640    }
5641
5642    fn load_pending_evictions(&self) -> Result<Vec<PendingEvict>, String> {
5643        let mut stmt = self
5644            .conn
5645            .prepare("SELECT unit_id, tbl, effective_scopes FROM _syncular_pending_evictions")
5646            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5647        let rows = stmt
5648            .query_map([], |row| {
5649                Ok((
5650                    row.get::<_, String>(0)?,
5651                    row.get::<_, String>(1)?,
5652                    row.get::<_, String>(2)?,
5653                ))
5654            })
5655            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5656        rows.map(|row| {
5657            let (unit_id, table, json) =
5658                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5659            let value: Value = serde_json::from_str(&json)
5660                .map_err(|_| "sync.local_corrupt: invalid pending eviction scopes".to_owned())?;
5661            let effective = json_to_scope_map(&value)
5662                .map_err(|_| "sync.local_corrupt: invalid pending eviction scopes".to_owned())?;
5663            Ok((unit_id, table, effective))
5664        })
5665        .collect()
5666    }
5667
5668    /// Select failed-intent retention; reload durable retained aggregates.
5669    pub fn set_retain_failed_commits(&mut self, enabled: bool) -> Result<(), String> {
5670        self.retain_failed_commits = enabled;
5671        for (commit, _) in self.failed_commits.iter() {
5672            for operation in &commit.ops {
5673                self.overlay_dirty.table(&operation.table);
5674            }
5675        }
5676        self.failed_commits = self.load_failed_commits()?.into();
5677        for (commit, _) in self.failed_commits.iter() {
5678            for operation in &commit.ops {
5679                self.overlay_dirty.table(&operation.table);
5680            }
5681        }
5682        Ok(())
5683    }
5684
5685    fn load_failed_commits(&self) -> Result<Vec<FailedCommit>, String> {
5686        let mut statement = self.conn.prepare("SELECT client_commit_id, operations_json, initial_json FROM _syncular_failed_commits ORDER BY seq")
5687            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5688        let rows = statement
5689            .query_map([], |row| {
5690                Ok((
5691                    row.get::<_, String>(0)?,
5692                    row.get::<_, String>(1)?,
5693                    row.get::<_, String>(2)?,
5694                ))
5695            })
5696            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5697        rows.map(|row| {
5698            let (id, operations, initial) =
5699                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5700            let operations: Vec<CommitOperation> =
5701                serde_json::from_str(&operations).map_err(|_| "sync.local_corrupt".to_owned())?;
5702            let initial =
5703                serde_json::from_str(&initial).map_err(|_| "sync.local_corrupt".to_owned())?;
5704            Ok((
5705                OutboxCommit {
5706                    client_commit_id: id,
5707                    ops: operations
5708                        .into_iter()
5709                        .map(|op| OutboxOp {
5710                            upsert: op.op == "upsert",
5711                            table: op.table,
5712                            row_id: op.row_id,
5713                            base_version: op.base_version,
5714                            values: op.values,
5715                            insert: op.insert,
5716                        })
5717                        .collect(),
5718                },
5719                initial,
5720            ))
5721        })
5722        .collect()
5723    }
5724
5725    fn retain_failed_commit(&mut self, id: &str, operations: &[OutboxOp]) -> Result<(), String> {
5726        let mut initial = Map::new();
5727        for operation in operations {
5728            // A table the current schema removed has no rows to retain.
5729            let Some(table) = self.schema.table(&operation.table) else {
5730                continue;
5731            };
5732            let initial_row = match self.stored_row(table, &operation.row_id, true)? {
5733                Some(row) => Some(row),
5734                None => self.visible_row(table, &operation.row_id)?,
5735            };
5736            if let Some((values, _)) = initial_row {
5737                let mut intended: Map<String, Value> = table
5738                    .columns
5739                    .iter()
5740                    .zip(values.iter())
5741                    .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
5742                    .collect();
5743                if let Some(patch) = &operation.values {
5744                    intended.extend(patch.clone());
5745                }
5746                initial.insert(
5747                    serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
5748                        .map_err(|_| "sync.local_corrupt".to_owned())?,
5749                    Value::Object(intended),
5750                );
5751            }
5752        }
5753        self.conn.execute("INSERT INTO _syncular_failed_commits(client_commit_id, operations_json, initial_json) VALUES (?1, ?2, ?3)", rusqlite::params![id,
5754            serde_json::to_string(&operations.iter().map(CommitOperation::from).collect::<Vec<_>>()).map_err(|_| "sync.local_corrupt".to_owned())?,
5755            serde_json::to_string(&initial).map_err(|_| "sync.local_corrupt".to_owned())?])
5756            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5757        self.failed_commits.push((
5758            OutboxCommit {
5759                client_commit_id: id.to_owned(),
5760                ops: operations.to_vec(),
5761            },
5762            initial,
5763        ));
5764        Ok(())
5765    }
5766
5767    fn drop_failed_commits(
5768        &mut self,
5769        matches: impl Fn(&str, &Map<String, Value>) -> bool,
5770    ) -> Result<Vec<OutboxCommit>, String> {
5771        let mut doomed = self.drop_acknowledged_rows(&matches)?;
5772        for (commit, initial) in self.failed_commits.iter() {
5773            for operation in &commit.ops {
5774                let key =
5775                    serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
5776                        .map_err(|_| "sync.local_corrupt".to_owned())?;
5777                if initial
5778                    .get(&key)
5779                    .and_then(Value::as_object)
5780                    .is_some_and(|values| matches(&operation.table, values))
5781                {
5782                    doomed.push(commit.clone());
5783                    break;
5784                }
5785            }
5786        }
5787        for commit in &doomed {
5788            self.conn
5789                .execute(
5790                    "DELETE FROM _syncular_failed_commits WHERE client_commit_id = ?1",
5791                    rusqlite::params![commit.client_commit_id],
5792                )
5793                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5794        }
5795        for commit in &doomed {
5796            self.conn.execute("UPDATE _syncular_commit_outcomes SET operations_json = NULL, results_json = '[]' WHERE client_commit_id = ?1", rusqlite::params![commit.client_commit_id]).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5797        }
5798        self.conflicts.retain(|record| {
5799            !doomed
5800                .iter()
5801                .any(|commit| commit.client_commit_id == record.client_commit_id)
5802        });
5803        self.rejections.retain(|record| {
5804            !doomed
5805                .iter()
5806                .any(|commit| commit.client_commit_id == record.client_commit_id)
5807        });
5808        self.failed_commits.retain(|(commit, _)| {
5809            !doomed
5810                .iter()
5811                .any(|dropped| dropped.client_commit_id == commit.client_commit_id)
5812        });
5813        for commit in &doomed {
5814            for operation in &commit.ops {
5815                self.overlay_dirty.table(&operation.table);
5816            }
5817        }
5818        Ok(doomed)
5819    }
5820
5821    /// Record one commit of local mutations. The returned commit ID is the
5822    /// §2.3 idempotency key. Failures are structured so callers read
5823    /// `code`/`message`/`details` instead of parsing a string prefix.
5824    pub fn mutate(&mut self, mutations: Vec<Mutation>) -> Result<String, ClientError> {
5825        self.mutate_with_commit_id(mutations, None)
5826    }
5827
5828    /// [`Self::mutate`] under a caller-supplied `clientCommitId` (§7.1), so an
5829    /// app that loses the reply knows its commit. The ID is non-empty (§6.1);
5830    /// one this replica still holds fails with `client.commit_id_reused`.
5831    pub fn mutate_with_commit_id(
5832        &mut self,
5833        mutations: Vec<Mutation>,
5834        client_commit_id: Option<String>,
5835    ) -> Result<String, ClientError> {
5836        let prior = self.storage_failure.borrow_mut().take();
5837        if let Some(id) = &client_commit_id {
5838            let held = if id.is_empty() {
5839                Err("sync.invalid_request: clientCommitId is empty".to_owned())
5840            } else {
5841                self.conn
5842                    .query_row(
5843                        "SELECT EXISTS(SELECT 1 FROM _syncular_outbox WHERE commit_id=?1 UNION ALL SELECT 1 FROM _syncular_commit_outcomes WHERE client_commit_id=?1)",
5844                        [id],
5845                        |row| row.get(0),
5846                    )
5847                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5848            };
5849            match held {
5850                Ok(false) => {}
5851                Ok(true) => {
5852                    return self.finish_authoring(
5853                        prior,
5854                        Err(ClientError {
5855                            code: "client.commit_id_reused".to_owned(),
5856                            message: "clientCommitId is held".to_owned(),
5857                            details: Some(json!({ "clientCommitId": id })),
5858                            retryable: false,
5859                        }),
5860                    )
5861                }
5862                Err(message) => return self.finish_authoring(prior, Err(message.into())),
5863            }
5864        }
5865        let outcome = (|| -> Result<String, ClientError> {
5866            if mutations.is_empty() {
5867                return Err(
5868                    "sync.invalid_request: a commit must contain at least one operation (§6.1)"
5869                        .to_owned()
5870                        .into(),
5871                );
5872            }
5873            let mut ops = Vec::with_capacity(mutations.len());
5874            for mutation in mutations {
5875                match mutation {
5876                    Mutation::Patch {
5877                        table,
5878                        values,
5879                        base_version,
5880                    } => {
5881                        let schema_table = self
5882                            .schema
5883                            .table(&table)
5884                            .ok_or_else(|| "sync.unknown_table".to_owned())?;
5885                        let values = normalize_values_casing(schema_table, values)?;
5886                        let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
5887                        ops.push(self.prepare_patch(&table, &row_id, values, base_version)?);
5888                    }
5889                    Mutation::Upsert {
5890                        table,
5891                        values,
5892                        base_version,
5893                    } => {
5894                        let schema_table = self.schema.table(&table).ok_or_else(|| {
5895                            format!("sync.unknown_table: unknown table {table:?}")
5896                        })?;
5897                        // §5: value keys are accepted in snake_case AND the
5898                        // generated row types' camelCase; normalize to SQL truth
5899                        // before the pk lookup / codec see them.
5900                        let values = normalize_values_casing(schema_table, values)?;
5901                        let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
5902                        // §6.7 `mutate` marks every column present: fill missing
5903                        // nullable columns with NULL before the payload is built.
5904                        let values = full_row_values(schema_table, values)?;
5905                        self.refuse_sealed_row(schema_table, &row_id)?;
5906                        // §5.11: validate the payload encodes with the current
5907                        // codec. A key-selection or unknown-key failure is a
5908                        // durable rejection at the push seam (§10.3), never an
5909                        // author-time error. Full-row `mutate` presents every
5910                        // column, so no stored fallback is needed.
5911                        self.validate_author_encode(schema_table, &row_id, &values)?;
5912                        // §7.1 insert intent: the row is absent from the
5913                        // visible replica at call time, before this batch
5914                        // applies anything.
5915                        let insert = !self.visible_row_exists(schema_table, &row_id)?;
5916                        ops.push(OutboxOp {
5917                            upsert: true,
5918                            table,
5919                            row_id,
5920                            base_version,
5921                            values: Some(values),
5922                            insert,
5923                        });
5924                    }
5925                    Mutation::Delete {
5926                        table,
5927                        row_id,
5928                        base_version,
5929                    } => {
5930                        let schema_table = self.schema.table(&table).ok_or_else(|| {
5931                            format!("sync.unknown_table: unknown table {table:?}")
5932                        })?;
5933                        self.refuse_sealed_row(schema_table, &row_id)?;
5934                        // §2.4: the server rejects a rowId holding U+0000.
5935                        if row_id.contains('\0') {
5936                            return Err("sync.invalid_request: rowId holds U+0000 (§2.4)"
5937                                .to_owned()
5938                                .into());
5939                        }
5940                        ops.push(OutboxOp {
5941                            upsert: false,
5942                            table,
5943                            row_id,
5944                            base_version,
5945                            values: None,
5946                            insert: false,
5947                        });
5948                    }
5949                }
5950            }
5951            Ok(self.record_outbox_commit(ops, client_commit_id)?)
5952        })();
5953        self.finish_authoring(prior, outcome)
5954    }
5955
5956    /// Scope the retained `storage_failure` diagnostic to this call so an
5957    /// earlier operation's failure cannot classify a new one, and surface the
5958    /// call's own classified failure with its details. A fresh classified
5959    /// failure replaces the retained diagnostic; otherwise the earlier one is
5960    /// kept for the next sync.
5961    fn finish_authoring(
5962        &self,
5963        prior: Option<QueryReadFailure>,
5964        outcome: Result<String, ClientError>,
5965    ) -> Result<String, ClientError> {
5966        let fresh = self.storage_failure.borrow_mut().take();
5967        *self.storage_failure.borrow_mut() = fresh.clone().or(prior);
5968        outcome.map_err(|error| fresh.map(ClientError::from).unwrap_or(error))
5969    }
5970
5971    fn record_outbox_commit(
5972        &mut self,
5973        ops: Vec<OutboxOp>,
5974        client_commit_id: Option<String>,
5975    ) -> Result<String, String> {
5976        let commit = OutboxCommit {
5977            client_commit_id: client_commit_id.unwrap_or_else(|| uuid::Uuid::new_v4().to_string()),
5978            ops,
5979        };
5980        self.begin_observation("syncular_mutation")?;
5981        let mut batch = ChangeAccumulator::default();
5982        for op in &commit.ops {
5983            let mut precise = self.record_row_scopes(&mut batch, &op.table, &op.row_id, false);
5984            if let Some(values) = &op.values {
5985                if let Some(table) = self.schema.table(&op.table) {
5986                    for scope in &table.scope_variables {
5987                        if let Some(Value::String(value)) = values.get(&scope.column) {
5988                            batch.scope(&op.table, format!("{}:{value}", scope.prefix));
5989                            precise = true;
5990                        }
5991                    }
5992                }
5993            }
5994            if !precise {
5995                batch.table(&op.table);
5996            }
5997        }
5998        if let Err(error) = self.persist_outbox_insert(&commit) {
5999            self.rollback_observation("syncular_mutation");
6000            return Err(error);
6001        }
6002        // A clean overlay contains the FIFO fold of every previous commit, so
6003        // the new commit applies over it. §7.1: an operation that does not
6004        // apply (a secondary unique collision) fails the whole commit; the
6005        // savepoint rollback removes the outbox entry and every visible write.
6006        // The append undoes itself: a rollback capture would copy the outbox.
6007        let was_dirty = self.overlay_dirty.snapshot();
6008        let id = commit.client_commit_id.clone();
6009        let appended = self.rebuild_overlay_if_dirty().and_then(|()| {
6010            commit
6011                .ops
6012                .iter()
6013                .try_for_each(|op| self.apply_outbox_op(op))?;
6014            self.outbox.push(Arc::new(commit));
6015            batch.status = true;
6016            self.finish_observation("syncular_mutation", batch)
6017                .inspect_err(|_| drop(self.outbox.pop()))
6018        });
6019        if let Err(error) = appended {
6020            self.rollback_observation("syncular_mutation");
6021            self.overlay_dirty.restore(was_dirty);
6022            return Err(error);
6023        }
6024        Ok(id)
6025    }
6026
6027    /// Record one §6.1 sparse upsert: the primary key plus the supplied
6028    /// non-scope columns are present; every other column is absent and stays
6029    /// untouched on the server and in the local overlay. An absent local base
6030    /// refuses the whole batch with sync.row_missing before enqueueing (§7.1).
6031    /// Failures are structured so callers read `code`/`message`/`details`.
6032    pub fn patch(
6033        &mut self,
6034        table: &str,
6035        row_id: &str,
6036        partial: Map<String, Value>,
6037        base_version: Option<i64>,
6038    ) -> Result<String, ClientError> {
6039        let prior = self.storage_failure.borrow_mut().take();
6040        let outcome = self.perform_patch(table, row_id, partial, base_version);
6041        self.finish_authoring(prior, outcome)
6042    }
6043
6044    /// The unscoped patch body. Public authoring entry points scope
6045    /// `storage_failure` around the whole operation, including any pre-read.
6046    fn perform_patch(
6047        &mut self,
6048        table: &str,
6049        row_id: &str,
6050        partial: Map<String, Value>,
6051        base_version: Option<i64>,
6052    ) -> Result<String, ClientError> {
6053        let operation = self.prepare_patch(table, row_id, partial, base_version)?;
6054        Ok(self.record_outbox_commit(vec![operation], None)?)
6055    }
6056
6057    fn prepare_patch(
6058        &self,
6059        table: &str,
6060        row_id: &str,
6061        partial: Map<String, Value>,
6062        base_version: Option<i64>,
6063    ) -> Result<OutboxOp, ClientError> {
6064        let schema_table = self
6065            .schema
6066            .table(table)
6067            .ok_or_else(|| format!("sync.unknown_table: unknown table {table:?}"))?;
6068        let partial = normalize_values_casing(schema_table, partial)?;
6069        if let Some(pk) = partial.get(&schema_table.primary_key) {
6070            if render_row_id_json(Some(pk)).ok().as_deref() != Some(row_id) {
6071                return Err(invalid_authoring(
6072                    "patch cannot change the primary key",
6073                    json!({ "table": table }),
6074                ));
6075            }
6076        }
6077        if !self.visible_row_exists(schema_table, row_id)? {
6078            return Err("sync.row_missing: a sparse patch requires a local row"
6079                .to_owned()
6080                .into());
6081        }
6082        self.refuse_sealed_row(schema_table, row_id)?;
6083        // §3.4 rule 5 / §6.2: scope columns are immutable on update. The
6084        // server accepts a present scope column whose value equals the
6085        // stored row and applies it as a no-op, so a decoded envelope
6086        // round-trips. The local row is the only value the client can prove
6087        // equality against; when the row is absent locally the client cannot
6088        // prove equality against the server's stored row and fails closed.
6089        // A primary key that is also a scope column needs no stored-row
6090        // read: the primary key in a sparse payload is by construction the
6091        // row id being patched (§6.1), so its value is proven equal already.
6092        for scope in &schema_table.scope_variables {
6093            if !partial.contains_key(&scope.column) || scope.column == schema_table.primary_key {
6094                continue;
6095            }
6096            let Some(column) = schema_table
6097                .columns
6098                .iter()
6099                .find(|column| column.name == scope.column)
6100            else {
6101                continue;
6102            };
6103            let stored = self
6104                .conn
6105                .query_row(
6106                    &format!(
6107                        "SELECT {} FROM {} WHERE {}",
6108                        quote_ident(&scope.column),
6109                        visible_table(&schema_table.name),
6110                        row_id_predicate(schema_table)
6111                    ),
6112                    rusqlite::params![row_id],
6113                    |row| Ok(sql_ref_to_json(column, row.get_ref(0)?)),
6114                )
6115                .optional()
6116                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6117            // Decode the supplied value through the same column-type path the
6118            // rest of the patch encoding uses, so a value that only equals
6119            // the stored one in another number representation still compares
6120            // equal.
6121            let incoming = json_to_column_value(column, partial.get(&scope.column))
6122                .ok()
6123                .map(|value| column_value_to_json(&value));
6124            // A stored blob never equals a patch value, matching the server's
6125            // scope-column comparison (§3.4): bytes are unstorable as scope
6126            // values.
6127            let equal = stored.as_ref().is_some_and(|stored| {
6128                stored.get("$bytes").is_none() && incoming.as_ref() == Some(stored)
6129            });
6130            if !equal {
6131                return Err(invalid_authoring(
6132                    "patch cannot write a scope column (§3.4)",
6133                    json!({ "table": table, "column": scope.column }),
6134                ));
6135            }
6136        }
6137        let mut values = partial;
6138        // Presence-set semantics: a proven-equal scope column is dropped, so
6139        // the patch leaves the stored value untouched (§6.2).
6140        for scope in &schema_table.scope_variables {
6141            values.remove(&scope.column);
6142        }
6143        let pk_value = match schema_table.columns[schema_table.pk_index].ty {
6144            // §2.4: the rowId of an integer key is its canonical decimal form.
6145            ColumnType::Integer => match row_id.parse::<i64>() {
6146                Ok(v)
6147                    if v.to_string() == row_id
6148                        && (-ssp2::primitives::I64_SAFE_MAX..=ssp2::primitives::I64_SAFE_MAX)
6149                            .contains(&v) =>
6150                {
6151                    Value::from(v)
6152                }
6153                _ => {
6154                    return Err(format!(
6155                    "sync.invalid_request: table {table:?}: invalid integer primary key {row_id:?}"
6156                )
6157                    .into())
6158                }
6159            },
6160            ColumnType::Boolean => match row_id {
6161                "true" => Value::Bool(true),
6162                "false" => Value::Bool(false),
6163                _ => {
6164                    return Err(format!(
6165                    "sync.invalid_request: table {table:?}: invalid boolean primary key {row_id:?}"
6166                )
6167                    .into())
6168                }
6169            },
6170            _ => Value::from(row_id.to_owned()),
6171        };
6172        values.insert(schema_table.primary_key.clone(), pk_value);
6173        // §6.1: a patch cannot null a non-nullable column.
6174        if let Some(column) = schema_table
6175            .columns
6176            .iter()
6177            .find(|column| !column.nullable && values.get(&column.name) == Some(&Value::Null))
6178        {
6179            return Err(invalid_authoring(
6180                "a non-nullable column is null (§6.1)",
6181                json!({ "table": table, "column": column.name }),
6182            ));
6183        }
6184        // §5.11: structural validation only; an absent key-id selector is
6185        // resolved at the push seam from the stored local row, and a
6186        // key-selection or unknown-key failure becomes a durable
6187        // `client.encrypt_failed` rejection there (§10.3), never an
6188        // author-time error.
6189        self.validate_author_encode(schema_table, row_id, &values)?;
6190        Ok(OutboxOp {
6191            upsert: true,
6192            table: table.to_owned(),
6193            row_id: row_id.to_owned(),
6194            base_version,
6195            values: Some(values),
6196            insert: false,
6197        })
6198    }
6199
6200    /// §5.11 key-id fallback: the stored local row's selector value as a
6201    /// positional row (only the selector slot filled). Returns None when no
6202    /// selector is configured, the row is locally absent, or the stored
6203    /// value is not a usable key id. A failed storage read is an error.
6204    fn stored_key_fallback(
6205        &self,
6206        table: &TableSchema,
6207        row_id: &str,
6208    ) -> Result<Option<Row>, String> {
6209        let Some(selector) = self.encryption.key_id_columns.get(&table.name) else {
6210            return Ok(None);
6211        };
6212        let Some(index) = table
6213            .columns
6214            .iter()
6215            .position(|column| &column.name == selector)
6216        else {
6217            return Ok(None);
6218        };
6219        let sql = format!(
6220            "SELECT {} FROM {} WHERE {}",
6221            quote_ident(selector),
6222            visible_table(&table.name),
6223            row_id_predicate(table)
6224        );
6225        let key = self
6226            .conn
6227            .query_row(&sql, rusqlite::params![row_id], |row| {
6228                row.get::<_, Option<String>>(0)
6229            })
6230            .optional()
6231            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
6232            .flatten()
6233            .filter(|key| !key.is_empty());
6234        let Some(key) = key else {
6235            return Ok(None);
6236        };
6237        let mut fallback: Row = vec![None; table.columns.len()];
6238        fallback[index] = Some(ColumnValue::String(key));
6239        Ok(Some(fallback))
6240    }
6241
6242    /// §5.11/§10.3 author-time encode check: a structural failure (a
6243    /// non-nullable NULL, a value the codec rejects) is an author-time
6244    /// error; a key-selection or unknown-key failure is deferred to the push
6245    /// seam, where it becomes a durable `client.encrypt_failed` rejection.
6246    fn validate_author_encode(
6247        &self,
6248        table: &TableSchema,
6249        row_id: &str,
6250        values: &Map<String, Value>,
6251    ) -> Result<(), String> {
6252        match encode_sparse_row_json(table, row_id, values, &self.encryption, None) {
6253            Err(error) if error.starts_with(ENCRYPT_FAILED_CODE) => Ok(()),
6254            result => result.map(|_| ()),
6255        }
6256    }
6257
6258    /// §5.11/§10.3: drop every pending commit whose push encode fails with
6259    /// `client.encrypt_failed` (an unresolvable key id or an unknown key) and
6260    /// raise one durable rejection per commit. `sync()` never aborts for an
6261    /// encode failure. Mirrors §7.4.4's `drop_incompatible_outbox`, including
6262    /// undoing the dropped commit's purely-optimistic rows.
6263    /// §7.1/§7.4.4: drop pending commits whose values the current codec refuses
6264    /// (an envelope or a type an earlier version accepted). `with_encryption`
6265    /// additionally drops commits whose §5.11 encryption cannot resolve a key;
6266    /// the startup boundary leaves those to the send seam, where hosts configure
6267    /// keys after open.
6268    /// Recovery runs as one observation. An enclosing observation (the guarded
6269    /// schema reset) passes its `batch`, so the rejection, outcome, and table
6270    /// domains join that change event instead of publishing a second one.
6271    fn drop_unencodable_outbox(
6272        &mut self,
6273        with_encryption: bool,
6274        batch: Option<&mut ChangeAccumulator>,
6275    ) -> Result<bool, String> {
6276        let mut failures = Vec::new();
6277        for commit in self.outbox.iter() {
6278            for op in &commit.ops {
6279                let Some(table) = self.schema.table(&op.table).filter(|_| op.upsert) else {
6280                    continue;
6281                };
6282                // An insert lacking a non-nullable column is left to
6283                // `drop_incompatible_outbox` (§7.4.4).
6284                let Some(values) = op.presented_values(table) else {
6285                    continue;
6286                };
6287                let fallback = self.stored_key_fallback(table, &op.row_id)?;
6288                let failure = match encode_sparse_row_json(
6289                    table,
6290                    &op.row_id,
6291                    &values,
6292                    &self.encryption,
6293                    fallback.as_ref(),
6294                ) {
6295                    Err(error) if error.starts_with(ENCRYPT_FAILED_CODE) => {
6296                        with_encryption.then_some((error, ENCRYPT_FAILED_CODE))
6297                    }
6298                    // §7.1/§7.4.4: a value the current codec refuses was
6299                    // persisted by a version that accepted it; it leaves
6300                    // through the incompatible-outbox path, so the FIFO
6301                    // queue never wedges.
6302                    Err(error) if error.starts_with(INVALID_REQUEST_CODE) => Some((
6303                        LEGACY_VALUE_REFUSAL_MESSAGE.to_owned(),
6304                        OUTBOX_INCOMPATIBLE_CODE,
6305                    )),
6306                    _ => None,
6307                };
6308                if let Some((error, code)) = failure {
6309                    failures.push((commit.clone(), error, code));
6310                    break;
6311                }
6312            }
6313        }
6314        if failures.is_empty() {
6315            return Ok(false);
6316        }
6317        // §7.4.4: one observation covers the durable outcome writes, the outbox
6318        // removal, the in-memory drop, and the overlay rebuild, so a storage fault
6319        // leaves the replica and the client unchanged, a successful recovery
6320        // publishes one batch with the rejection, outcome, and table domains, and
6321        // an ordinary open with nothing to recover publishes nothing.
6322        let rollback = self.capture();
6323        let owns_observation = batch.is_none();
6324        let owns_transaction = self.conn.is_autocommit();
6325        let mut domains = ChangeAccumulator::default();
6326        if owns_observation {
6327            self.begin_observation("syncular_value_recovery")?;
6328        }
6329        let dropped = (|| -> Result<(), String> {
6330            for (commit, message, code) in failures {
6331                if self.retain_failed_commits && code == ENCRYPT_FAILED_CODE {
6332                    self.retain_failed_commit(&commit.client_commit_id, &commit.ops)?;
6333                }
6334                let representable = commit.ops.iter().all(Self::outbox_op_is_journalable);
6335                let rejection = RejectionRecord {
6336                    client_commit_id: commit.client_commit_id.clone(),
6337                    op_index: 0,
6338                    code: code.to_owned(),
6339                    message,
6340                    retryable: false,
6341                    details: (code == OUTBOX_INCOMPATIBLE_CODE).then(|| RejectionDetails {
6342                        field_paths: None,
6343                        reason: Some(INVALID_STORED_VALUES_REASON.to_owned()),
6344                        required_action: None,
6345                        references: None,
6346                    }),
6347                    operation: representable
6348                        .then(|| commit.ops.first())
6349                        .flatten()
6350                        .map(CommitOperation::from),
6351                };
6352                self.persist_commit_outcome(
6353                    &commit.client_commit_id,
6354                    CommitOutcomeStatus::Rejected,
6355                    &[CommitOperationOutcome::Error {
6356                        rejection: rejection.clone(),
6357                    }],
6358                    representable.then_some(commit.ops.as_slice()),
6359                )?;
6360                self.delete_outbox_persisted(&commit.client_commit_id)?;
6361                self.outbox
6362                    .retain(|candidate| candidate.client_commit_id != commit.client_commit_id);
6363                self.rejections.push(rejection);
6364                for operation in &commit.ops {
6365                    domains.table(operation.table.as_str());
6366                }
6367            }
6368            domains.status = true;
6369            domains.rejections = true;
6370            domains.outcomes = true;
6371            self.prune_commit_outcomes()?;
6372            // The visible projection is re-derived from the surviving base + outbox.
6373            self.overlay_dirty.set(true);
6374            self.rebuild_overlay()?;
6375            Ok(())
6376        })();
6377        let result = match batch {
6378            Some(batch) => {
6379                if dropped.is_ok() {
6380                    for table in domains.tables.keys() {
6381                        batch.table(table.as_str());
6382                    }
6383                    batch.status |= domains.status;
6384                    batch.rejections |= domains.rejections;
6385                    batch.outcomes |= domains.outcomes;
6386                }
6387                dropped
6388            }
6389            None => {
6390                dropped.and_then(|()| self.finish_observation("syncular_value_recovery", domains))
6391            }
6392        };
6393        if let Err(error) = result {
6394            if owns_observation {
6395                if owns_transaction {
6396                    if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
6397                        let mut failure = self.storage_failure.borrow_mut();
6398                        let failure =
6399                            failure.get_or_insert_with(|| QueryReadFailure::from(error.clone()));
6400                        failure.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
6401                    }
6402                } else {
6403                    self.rollback_observation("syncular_value_recovery");
6404                }
6405            }
6406            self.restore(rollback);
6407            return Err(error);
6408        }
6409        Ok(true)
6410    }
6411
6412    pub fn pending_commit_ids(&self) -> Vec<String> {
6413        self.outbox
6414            .iter()
6415            .map(|c| c.client_commit_id.clone())
6416            .collect()
6417    }
6418
6419    /// §6.1: the sparse push payload bytes of every pending upsert operation,
6420    /// FIFO — exactly what the next push round would put on the wire. Used by
6421    /// conformance B.21(h) to pin cross-core byte identity.
6422    pub fn pending_payloads(&self) -> Result<Vec<Vec<u8>>, String> {
6423        let mut payloads = Vec::new();
6424        for commit in self.outbox.iter() {
6425            for op in &commit.ops {
6426                let Some(table) = self.schema.table(&op.table).filter(|_| op.upsert) else {
6427                    continue;
6428                };
6429                let Some(values) = op.presented_values(table) else {
6430                    continue;
6431                };
6432                let fallback = self.stored_key_fallback(table, &op.row_id)?;
6433                if let Ok(payload) = encode_sparse_row_json(
6434                    table,
6435                    &op.row_id,
6436                    &values,
6437                    &self.encryption,
6438                    fallback.as_ref(),
6439                ) {
6440                    payloads.push(payload);
6441                }
6442            }
6443        }
6444        Ok(payloads)
6445    }
6446
6447    pub fn conflicts(&self) -> &[ConflictRecord] {
6448        &self.conflicts
6449    }
6450
6451    pub fn rejections(&self) -> &[RejectionRecord] {
6452        &self.rejections
6453    }
6454
6455    pub fn resolve_commit_outcome(
6456        &mut self,
6457        input: ResolveCommitOutcomeInput,
6458    ) -> Result<CommitOutcome, String> {
6459        let current = self
6460            .commit_outcome(&input.client_commit_id)?
6461            .ok_or_else(|| {
6462                format!(
6463                    "sync.outcome_not_found: no durable outcome exists for {:?}",
6464                    input.client_commit_id
6465                )
6466            })?;
6467        if current.resolution != CommitOutcomeResolution::Active {
6468            return Ok(current);
6469        }
6470        if input.resolution == CommitOutcomeResolution::Active {
6471            return Err("sync.invalid_request: resolution must leave active state".to_owned());
6472        }
6473        match input.resolution {
6474            CommitOutcomeResolution::Superseded => {
6475                let replacement = input
6476                    .replacement_client_commit_id
6477                    .as_deref()
6478                    .filter(|value| !value.is_empty() && *value != input.client_commit_id)
6479                    .ok_or_else(|| {
6480                        "sync.invalid_request: superseded outcomes require a distinct replacementClientCommitId"
6481                            .to_owned()
6482                    })?;
6483                let _ = replacement;
6484            }
6485            _ if input.replacement_client_commit_id.is_some() => {
6486                return Err(
6487                    "sync.invalid_request: replacementClientCommitId is valid only for superseded outcomes"
6488                        .to_owned(),
6489                );
6490            }
6491            _ => {}
6492        }
6493        let allowed = match current.status {
6494            CommitOutcomeStatus::Conflict => matches!(
6495                input.resolution,
6496                CommitOutcomeResolution::ResolvedKeepServer | CommitOutcomeResolution::Superseded
6497            ),
6498            CommitOutcomeStatus::Rejected => {
6499                matches!(
6500                    input.resolution,
6501                    CommitOutcomeResolution::Superseded
6502                        | CommitOutcomeResolution::ResolvedKeepServer
6503                )
6504            }
6505            CommitOutcomeStatus::Applied | CommitOutcomeStatus::Cached => {
6506                input.resolution == CommitOutcomeResolution::Dismissed
6507            }
6508        };
6509        if !allowed {
6510            return Err(format!(
6511                "sync.invalid_request: resolution {:?} is invalid for {:?} outcome",
6512                input.resolution, current.status
6513            ));
6514        }
6515
6516        let rollback = self.capture();
6517        self.begin_observation("syncular_outcome_resolution")?;
6518        let result = (|| {
6519            self.conn
6520                .execute(
6521                    "UPDATE _syncular_commit_outcomes
6522                        SET resolution = ?1, resolved_at_ms = ?2,
6523                            replacement_client_commit_id = ?3
6524                      WHERE client_commit_id = ?4 AND resolution = 'active'",
6525                    rusqlite::params![
6526                        Self::outcome_resolution_name(input.resolution),
6527                        self.clock_now_ms(),
6528                        input.replacement_client_commit_id,
6529                        input.client_commit_id
6530                    ],
6531                )
6532                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6533            let resolved = self
6534                .commit_outcome(&current.client_commit_id)?
6535                .ok_or_else(|| "sync.outcome_not_found: outcome disappeared".to_owned())?;
6536            self.conn
6537                .execute(
6538                    "DELETE FROM _syncular_failed_commits WHERE client_commit_id = ?1",
6539                    rusqlite::params![input.client_commit_id],
6540                )
6541                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6542            for (commit, _) in self.failed_commits.iter() {
6543                if commit.client_commit_id == input.client_commit_id {
6544                    for operation in &commit.ops {
6545                        self.overlay_dirty.table(&operation.table);
6546                    }
6547                }
6548            }
6549            self.failed_commits
6550                .retain(|(commit, _)| commit.client_commit_id != input.client_commit_id);
6551            self.prune_commit_outcomes()?;
6552            Ok(resolved)
6553        })();
6554        let resolved = match result {
6555            Ok(outcome) => outcome,
6556            Err(error) => {
6557                self.rollback_observation("syncular_outcome_resolution");
6558                self.restore(rollback);
6559                return Err(error);
6560            }
6561        };
6562        self.conflicts
6563            .retain(|record| record.client_commit_id != current.client_commit_id);
6564        self.rejections
6565            .retain(|record| record.client_commit_id != current.client_commit_id);
6566        let mut batch = ChangeAccumulator {
6567            conflicts: current.status == CommitOutcomeStatus::Conflict,
6568            rejections: current.status == CommitOutcomeStatus::Rejected,
6569            outcomes: true,
6570            ..ChangeAccumulator::default()
6571        };
6572        if let Some(operations) = &current.operations {
6573            for operation in operations {
6574                batch.table(&operation.table);
6575            }
6576        }
6577        if let Err(error) = self
6578            .rebuild_overlay_if_dirty()
6579            .and_then(|()| self.finish_observation("syncular_outcome_resolution", batch))
6580        {
6581            self.rollback_observation("syncular_outcome_resolution");
6582            self.restore(rollback);
6583            return Err(error);
6584        }
6585        Ok(resolved)
6586    }
6587
6588    pub fn schema_floor(&self) -> Option<&SchemaFloor> {
6589        self.schema_floor.as_ref()
6590    }
6591
6592    /// §7.3.5: the client's opaque auth-lease state, if any.
6593    pub fn lease_state(&self) -> Option<&LeaseState> {
6594        self.lease_state.as_ref()
6595    }
6596
6597    /// §7.3.5: record a request-level lease error (stop-and-surface). Only
6598    /// the two lease codes set it; other errors leave leaseState untouched.
6599    fn record_lease_error(&mut self, code: &str) -> Result<(), String> {
6600        if code != "sync.auth_lease_required" && code != "sync.auth_lease_revoked" {
6601            return Ok(());
6602        }
6603        let mut next = self.lease_state.clone().unwrap_or_default();
6604        next.error_code = Some(code.to_owned());
6605        self.set_lease_state(Some(next))
6606    }
6607
6608    fn set_lease_state(&mut self, next: Option<LeaseState>) -> Result<(), String> {
6609        if self.lease_state == next {
6610            return Ok(());
6611        }
6612        self.begin_observation("syncular_lease")?;
6613        let rollback = self.capture();
6614        self.lease_state = next;
6615        let persisted = match &self.lease_state {
6616            Some(lease) => serde_json::to_string(lease)
6617                .map_err(|error| error.to_string())
6618                .and_then(|json| self.set_meta(LEASE_STATE_KEY, &json)),
6619            None => self.delete_meta(LEASE_STATE_KEY),
6620        };
6621        let batch = ChangeAccumulator {
6622            status: true,
6623            ..ChangeAccumulator::default()
6624        };
6625        if let Err(error) =
6626            persisted.and_then(|()| self.finish_observation("syncular_lease", batch))
6627        {
6628            self.rollback_observation("syncular_lease");
6629            self.restore(rollback);
6630            return Err(error);
6631        }
6632        Ok(())
6633    }
6634
6635    fn set_schema_floor(&mut self, next: Option<SchemaFloor>) -> Result<(), String> {
6636        if self.schema_floor == next {
6637            return Ok(());
6638        }
6639        self.begin_observation("syncular_schema_floor")?;
6640        let rollback = self.capture();
6641        self.stopped = next.is_some();
6642        self.schema_floor = next;
6643        let persisted = match &self.schema_floor {
6644            Some(floor) => serde_json::to_string(floor)
6645                .map_err(|error| error.to_string())
6646                .and_then(|json| self.set_meta(SCHEMA_FLOOR_KEY, &json)),
6647            None => self.delete_meta(SCHEMA_FLOOR_KEY),
6648        };
6649        let batch = ChangeAccumulator {
6650            status: true,
6651            ..ChangeAccumulator::default()
6652        };
6653        if let Err(error) =
6654            persisted.and_then(|()| self.finish_observation("syncular_schema_floor", batch))
6655        {
6656            self.rollback_observation("syncular_schema_floor");
6657            self.restore(rollback);
6658            return Err(error);
6659        }
6660        Ok(())
6661    }
6662
6663    fn set_upgrading(&mut self, value: bool) {
6664        if self.upgrading == value {
6665            return;
6666        }
6667        if self.begin_observation("syncular_upgrading").is_err() {
6668            return;
6669        }
6670        self.upgrading = value;
6671        let batch = ChangeAccumulator {
6672            status: true,
6673            ..ChangeAccumulator::default()
6674        };
6675        if self
6676            .finish_observation("syncular_upgrading", batch)
6677            .is_err()
6678        {
6679            self.rollback_observation("syncular_upgrading");
6680        }
6681    }
6682
6683    pub fn sync_needed(&self) -> bool {
6684        self.sync_needed
6685    }
6686
6687    pub fn read_rows(&self, table: &str) -> Result<Vec<RowState>, String> {
6688        let schema_table = self
6689            .schema
6690            .table(table)
6691            .ok_or_else(|| format!("unknown table {table:?}"))?;
6692        let sql = format!(
6693            "SELECT * FROM {} ORDER BY {} ASC",
6694            visible_table(table),
6695            quote_ident(&schema_table.primary_key)
6696        );
6697        let mut stmt = self
6698            .conn
6699            .prepare(&sql)
6700            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6701        let mut rows = stmt
6702            .query([])
6703            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6704        let mut out = Vec::new();
6705        while let Some(row) = rows
6706            .next()
6707            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
6708        {
6709            let mut values = Map::new();
6710            for (i, column) in schema_table.columns.iter().enumerate() {
6711                let value = row
6712                    .get_ref(i)
6713                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6714                values.insert(column.name.clone(), sql_ref_to_json(column, value));
6715            }
6716            let version: i64 = row
6717                .get(schema_table.columns.len())
6718                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6719            let row_id = match values.get(&schema_table.primary_key) {
6720                Some(Value::String(s)) => s.clone(),
6721                Some(Value::Number(n)) => n.to_string(),
6722                Some(Value::Bool(b)) => b.to_string(),
6723                other => format!("{}", other.cloned().unwrap_or(Value::Null)),
6724            };
6725            let sealed_key = if schema_table.has_encrypted_columns() {
6726                row.get(schema_table.columns.len() + 1)
6727                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
6728            } else {
6729                None
6730            };
6731            out.push(RowState {
6732                row_id,
6733                version,
6734                values,
6735                sealed_key,
6736            });
6737        }
6738        Ok(out)
6739    }
6740
6741    // -- §5.10.5 native CRDT (the `crdt-yjs` feature) --------------------------
6742    //
6743    // The Rust face of the §5.10.4 client model: a local crdt edit loads the
6744    // current stored (server-merged ⊕ pending-overlay) column bytes, applies
6745    // the op with `yrs`, re-encodes the whole doc state, and records a
6746    // baseVersion-less sparse upsert carrying only the primary key and the
6747    // crdt column (§5.10.3, §6.6). No local merge — merging is server-side;
6748    // the overlay re-materializes the edit immediately (optimistic apply,
6749    // §7.1) and the server-merged bytes arrive on the next pull, idempotently.
6750    // Byte-compatible with `@syncular/crdt-yjs`.
6751
6752    /// The current stored value of a `crdt` column for one row — the visible
6753    /// (optimistic) bytes, or `None` when the row is absent or the column is
6754    /// NULL (the empty document, §5.10.1). Errors if the column is not a
6755    /// `crdt` column (guards the app against a typo'd column name).
6756    #[cfg(feature = "crdt-yjs")]
6757    fn crdt_column_bytes(
6758        &self,
6759        table: &str,
6760        row_id: &str,
6761        column: &str,
6762    ) -> Result<Option<Vec<u8>>, String> {
6763        let schema_table = self
6764            .schema
6765            .table(table)
6766            .ok_or_else(|| format!("unknown table {table:?}"))?;
6767        let col = schema_table
6768            .columns
6769            .iter()
6770            .find(|c| c.name == column)
6771            .ok_or_else(|| format!("table {table:?} has no column {column:?}"))?;
6772        if col.ty != ColumnType::Crdt {
6773            return Err(format!("column {column:?} is not a crdt column (§5.10.1)"));
6774        }
6775        let sql = format!(
6776            "SELECT {} FROM {} WHERE {}",
6777            quote_ident(column),
6778            visible_table(table),
6779            row_id_predicate(schema_table)
6780        );
6781        let bytes: Option<Vec<u8>> = self
6782            .conn
6783            .prepare_cached(&sql)
6784            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
6785            .query_row(rusqlite::params![row_id], |row| {
6786                row.get::<_, Option<Vec<u8>>>(0)
6787            })
6788            .map_err(|e| match e {
6789                rusqlite::Error::QueryReturnedNoRows => "no such row".to_owned(),
6790                other => Self::sqlite_failure(&self.storage_failure, other),
6791            })?;
6792        Ok(bytes)
6793    }
6794
6795    /// §5.10.4 materialize: the collaborative text of a `crdt` column, decoded
6796    /// from the stored bytes with `yrs` — `YjsColumn.text(name).toString()`.
6797    /// An absent row / NULL column is the empty document (empty string).
6798    #[cfg(feature = "crdt-yjs")]
6799    pub fn crdt_text(
6800        &self,
6801        table: &str,
6802        row_id: &str,
6803        column: &str,
6804        name: &str,
6805    ) -> Result<String, String> {
6806        let bytes = self
6807            .crdt_column_bytes(table, row_id, column)?
6808            .unwrap_or_default();
6809        crate::crdt::text(&bytes, name)
6810    }
6811
6812    /// §5.10.4 push-an-update: apply a text insert to a `crdt` column and
6813    /// record the resulting crdt-only sparse operation. Returns the enqueued
6814    /// `clientCommitId`.
6815    #[cfg(feature = "crdt-yjs")]
6816    pub fn crdt_insert_text(
6817        &mut self,
6818        table: &str,
6819        row_id: &str,
6820        column: &str,
6821        name: &str,
6822        index: u32,
6823        value: &str,
6824    ) -> Result<String, ClientError> {
6825        let prior = self.storage_failure.borrow_mut().take();
6826        let outcome = (|| -> Result<String, ClientError> {
6827            let current = self
6828                .crdt_column_bytes(table, row_id, column)?
6829                .unwrap_or_default();
6830            let update = crate::crdt::insert_text(&current, name, index, value)?;
6831            self.perform_crdt_push_update(table, row_id, column, &update)
6832        })();
6833        self.finish_authoring(prior, outcome)
6834    }
6835
6836    /// §5.10.4 push-an-update: apply a text delete to a `crdt` column and
6837    /// record the resulting crdt-only sparse operation. Returns the enqueued
6838    /// `clientCommitId`.
6839    #[cfg(feature = "crdt-yjs")]
6840    pub fn crdt_delete_text(
6841        &mut self,
6842        table: &str,
6843        row_id: &str,
6844        column: &str,
6845        name: &str,
6846        index: u32,
6847        len: u32,
6848    ) -> Result<String, ClientError> {
6849        let prior = self.storage_failure.borrow_mut().take();
6850        let outcome = (|| -> Result<String, ClientError> {
6851            let current = self
6852                .crdt_column_bytes(table, row_id, column)?
6853                .unwrap_or_default();
6854            let update = crate::crdt::delete_text(&current, name, index, len)?;
6855            self.perform_crdt_push_update(table, row_id, column, &update)
6856        })();
6857        self.finish_authoring(prior, outcome)
6858    }
6859
6860    /// §5.10.4 generic escape hatch: apply an arbitrary Yjs update onto a
6861    /// `crdt` column's current state and record the resulting crdt-only sparse
6862    /// operation. The app authored the update with its own `yrs` model.
6863    /// Returns the enqueued `clientCommitId`.
6864    #[cfg(feature = "crdt-yjs")]
6865    pub fn crdt_apply_update(
6866        &mut self,
6867        table: &str,
6868        row_id: &str,
6869        column: &str,
6870        update: &[u8],
6871    ) -> Result<String, ClientError> {
6872        let prior = self.storage_failure.borrow_mut().take();
6873        let outcome = (|| -> Result<String, ClientError> {
6874            let current = self
6875                .crdt_column_bytes(table, row_id, column)?
6876                .unwrap_or_default();
6877            let next = crate::crdt::apply_update(&current, update)?;
6878            self.perform_crdt_push_update(table, row_id, column, &next)
6879        })();
6880        self.finish_authoring(prior, outcome)
6881    }
6882
6883    /// Shared tail of the crdt edit methods: record a §6.1 sparse crdt-only
6884    /// upsert — the primary key plus the crdt column, no `baseVersion` — and
6885    /// enqueue it. The server merges the crdt column and leaves every other
6886    /// column untouched (§5.10.3), so a baseVersion-less crdt edit cannot
6887    /// clobber a concurrent edit to another column. A locally absent row
6888    /// records the same partial operation; the server answers per §6.2.
6889    #[cfg(feature = "crdt-yjs")]
6890    fn perform_crdt_push_update(
6891        &mut self,
6892        table: &str,
6893        row_id: &str,
6894        column: &str,
6895        crdt_bytes: &[u8],
6896    ) -> Result<String, ClientError> {
6897        let mut bytes_obj = Map::new();
6898        bytes_obj.insert("$bytes".to_owned(), Value::from(bytes_to_hex(crdt_bytes)));
6899        let mut partial = Map::new();
6900        partial.insert(column.to_owned(), Value::Object(bytes_obj));
6901        self.perform_patch(table, row_id, partial, None)
6902    }
6903
6904    /// Run an arbitrary read-only SQL query against the local database and
6905    /// return each row as a `column-name → JSON value` map. This is the seam
6906    /// the React `useSyncQuery` live-query API needs (it takes app-authored
6907    /// SQL over the visible tables/views, not a fixed table read like
6908    /// [`read_rows`]).
6909    ///
6910    /// Bound `params` are the driver value forms: JSON strings/numbers/bools/
6911    /// null bind directly; a `{"$bytes": hex}` object binds as a BLOB — the
6912    /// same envelope the command surface uses everywhere else. Output BLOB
6913    /// columns come back as `{"$bytes": hex}` to round-trip cleanly.
6914    ///
6915    /// The result column typing is dynamic (SQLite's stored affinity), because
6916    /// arbitrary SQL can alias, join, and compute — there is no schema column
6917    /// to consult per output cell, unlike [`read_rows`].
6918    pub fn query(&self, sql: &str, params: &[QueryValue]) -> Result<Vec<QueryRow>, String> {
6919        query_connection(&self.conn, sql, params).map_err(|failure| failure.to_string())
6920    }
6921
6922    /// Repository benchmark and conformance access, present only with `bench-internals`.
6923    #[cfg(feature = "bench-internals")]
6924    #[doc(hidden)]
6925    pub fn benchmark_connection(&mut self) -> &mut Connection {
6926        &mut self.conn
6927    }
6928
6929    /// Private benchmark intervals; absent from ordinary builds and diagnostics.
6930    #[cfg(feature = "bench-internals")]
6931    #[doc(hidden)]
6932    pub fn benchmark_phases(
6933        &self,
6934        enabled: Option<bool>,
6935        reset: bool,
6936    ) -> Result<Option<Value>, String> {
6937        self.benchmark_phases.configure(enabled, reset)?;
6938        Ok(self.benchmark_phases.snapshot())
6939    }
6940
6941    /// One serialized, read-only authority observation, including durable coverage.
6942    pub fn authority_snapshot(&self) -> Result<Value, String> {
6943        validate_authority_reads(&self.schema, &self.limits.authority_reads)?;
6944        if self.limits.authority_reads.is_empty() {
6945            return Err("client.authority_read_forbidden: no authority reads were declared at client creation".into());
6946        }
6947        self.conn
6948            .execute_batch("SAVEPOINT authority_snapshot")
6949            .map_err(|e| e.to_string())?;
6950        let result = (|| {
6951            let revision = self
6952                .get_meta(LOCAL_REVISION_KEY)?
6953                .ok_or("sync.local_corrupt: authority revision is missing")?;
6954            let parsed_revision = revision
6955                .parse::<u64>()
6956                .map_err(|_| "sync.local_corrupt: authority revision is invalid")?;
6957            if parsed_revision.to_string() != revision {
6958                return Err("sync.local_corrupt: authority revision is invalid".into());
6959            }
6960            let mut tables = Vec::new();
6961            for read in &self.limits.authority_reads {
6962                let table = self
6963                    .schema
6964                    .table(&read.table)
6965                    .ok_or("client.authority_read_forbidden: authority table is not declared")?;
6966                let scalar = |column: &str, value: &Value| -> Result<String, String> {
6967                    let ty = table
6968                        .columns
6969                        .iter()
6970                        .find(|c| c.name == column)
6971                        .expect("validated authority column")
6972                        .ty;
6973                    if ty == ColumnType::Boolean {
6974                        if let Some(value) = value.as_bool() {
6975                            return Ok(value.to_string());
6976                        }
6977                        return match value.as_i64() {
6978                            Some(0) => Ok("false".into()),
6979                            Some(1) => Ok("true".into()),
6980                            _ => Err("sync.local_corrupt: authority boolean is invalid".into()),
6981                        };
6982                    }
6983                    match value {
6984                        Value::String(s) => Ok(s.clone()),
6985                        Value::Number(n) => Ok(n.to_string()),
6986                        Value::Object(o) if ty == ColumnType::Integer && o.len() == 1 => o
6987                            .get("$bigint")
6988                            .and_then(Value::as_str)
6989                            .map(str::to_owned)
6990                            .ok_or("sync.local_corrupt: authority integer is invalid".into()),
6991                        _ => {
6992                            Err("sync.local_corrupt: authority scalar is missing or invalid".into())
6993                        }
6994                    }
6995                };
6996                let matches = |row: &QueryRow| -> Result<bool, String> {
6997                    for (variable, values) in &read.scopes {
6998                        let column = table
6999                            .scope_column(variable)
7000                            .expect("validated authority scope");
7001                        let value = row
7002                            .get(column)
7003                            .ok_or("sync.local_corrupt: authority scope column is missing")?;
7004                        if value.is_null() || !values.contains(&scalar(column, value)?) {
7005                            return Ok(false);
7006                        }
7007                    }
7008                    Ok(true)
7009                };
7010                let selected = read
7011                    .columns
7012                    .iter()
7013                    .map(|c| quote_ident(c))
7014                    .collect::<Vec<_>>()
7015                    .join(",");
7016                let mut intent = BTreeSet::new();
7017                let mut relevant = BTreeSet::new();
7018                for row in query_connection(&self.conn,
7019                    "SELECT id,values_json FROM (SELECT json_extract(value,'$.table') AS tbl,json_extract(value,'$.rowId') AS id,json_extract(value,'$.values') AS values_json FROM _syncular_outbox,json_each(ops_json) UNION ALL SELECT json_extract(value,'$.table'),json_extract(value,'$.rowId'),json_extract(value,'$.values') FROM _syncular_failed_commits,json_each(operations_json) UNION ALL SELECT tbl,id,intent_json FROM _syncular_acked_rows) WHERE tbl=?",
7020                    &[Value::String(read.table.clone())]).map_err(|e| e.to_string())? {
7021                    let id = row.get("id").and_then(Value::as_str).ok_or("sync.local_corrupt: authority intent id is invalid")?.to_owned();
7022                    intent.insert(id.clone());
7023                    if let Some(raw) = row.get("values_json").and_then(Value::as_str) {
7024                        let values: QueryRow = serde_json::from_str(raw).map_err(|_| "sync.local_corrupt: authority intent values are invalid")?;
7025                        if read.scopes.keys().all(|v| values.contains_key(table.scope_column(v).expect("validated scope"))) && matches(&values)? {relevant.insert(id);}
7026                    }
7027                }
7028                let mut rows = BTreeMap::new();
7029                for full in [base_table(&read.table), visible_table(&read.table)] {
7030                    for mut values in query_connection(
7031                        &self.conn,
7032                        &format!(
7033                            "SELECT {selected},_syncular_version AS authority_version FROM {full}"
7034                        ),
7035                        &[],
7036                    )
7037                    .map_err(|e| e.to_string())?
7038                    {
7039                        if !matches(&values)? {
7040                            continue;
7041                        }
7042                        let id = scalar(
7043                            &table.primary_key,
7044                            values
7045                                .get(&table.primary_key)
7046                                .expect("selected primary key"),
7047                        )?;
7048                        relevant.insert(id.clone());
7049                        if full == base_table(&read.table) {
7050                            let version = values
7051                                .remove("authority_version")
7052                                .expect("selected version");
7053                            let version = version
7054                                .as_i64()
7055                                .ok_or("sync.local_corrupt: authority server version is invalid")?;
7056                            if version >= 0 {
7057                                rows.insert(id.clone(), json!({"values": values, "version": version, "hasLocalIntent": intent.contains(&id)}));
7058                            }
7059                        }
7060                    }
7061                }
7062                intent.retain(|id| relevant.contains(id));
7063                let rows = rows.into_values().collect::<Vec<_>>();
7064                // Every unit covering the table, as its (unit, table) pair (§4.4).
7065                let replica = self.stored(replica_cursor(&self.conn))?;
7066                let persisted = self
7067                    .stored(load_units(&self.conn))?
7068                    .into_iter()
7069                    .filter(|unit| covers(table, &unit.scopes))
7070                    .map(|unit| {
7071                        let effective = unit.echo.as_deref().and_then(|echo| {
7072                            pair_scopes(&unit.scopes, &read.table, Some(echo))
7073                        });
7074                        let requested = read
7075                            .scopes
7076                            .keys()
7077                            .map(|v| (v.clone(), json!(unit.scopes.get(v).into_iter().collect::<Vec<_>>())))
7078                            .collect::<Map<_, _>>();
7079                        let complete = effective
7080                            .as_ref()
7081                            .is_some_and(|scopes| scopes.len() == read.scopes.len())
7082                            && unit.status == UnitStatus::Active
7083                            && !unit.bootstrapping();
7084                        let exposed = effective.map(|scopes| {
7085                            read.scopes
7086                                .keys()
7087                                .map(|v| {
7088                                    let held = scopes.iter().find(|(variable, _)| variable == v);
7089                                    (v.clone(), json!(held.map_or(&Vec::new(), |(_, values)| values)))
7090                                })
7091                                .collect::<Map<_, _>>()
7092                        });
7093                        json!({"requestedScopes": requested, "status": unit.status, "cursor": unit.cursor.unwrap_or(replica), "effectiveScopes": exposed, "complete": complete})
7094                    })
7095                    .collect::<Vec<_>>();
7096                fn covered(selectors: &[(&String, &Vec<String>)], candidates: &[&Value]) -> bool {
7097                    match selectors.split_first() {
7098                        None => !candidates.is_empty(),
7099                        Some(((variable, values), rest)) => values.iter().all(|value| {
7100                            covered(
7101                                rest,
7102                                &candidates
7103                                    .iter()
7104                                    .copied()
7105                                    .filter(|p| {
7106                                        p["effectiveScopes"][variable.as_str()]
7107                                            .as_array()
7108                                            .is_some_and(|held| {
7109                                                held.contains(&Value::String(value.clone()))
7110                                                    || held.contains(&Value::String("*".into()))
7111                                            })
7112                                    })
7113                                    .collect::<Vec<_>>(),
7114                            )
7115                        }),
7116                    }
7117                }
7118                let coverage = if covered(
7119                    &read.scopes.iter().collect::<Vec<_>>(),
7120                    &persisted
7121                        .iter()
7122                        .filter(|p| p["complete"] == true)
7123                        .collect::<Vec<_>>(),
7124                ) {
7125                    "complete"
7126                } else if persisted.iter().any(|p| {
7127                    p["status"] == "active"
7128                        && read.scopes.iter().all(|(v, values)| {
7129                            values.iter().any(|value| {
7130                                p["requestedScopes"][v].as_array().is_some_and(|held| {
7131                                    held.contains(&Value::String(value.clone()))
7132                                })
7133                            })
7134                        })
7135                }) {
7136                    "pending"
7137                } else {
7138                    "missing"
7139                };
7140                tables.push(json!({"table": read.table, "rows": rows, "localIntentRowIds": intent, "scopes": read.scopes, "coverage": coverage, "persisted": persisted}));
7141            }
7142            Ok(
7143                json!({"revision": revision, "complete": tables.iter().all(|t| t["coverage"] == "complete"), "tables": tables}),
7144            )
7145        })();
7146        match result {
7147            Ok(snapshot) => {
7148                self.conn
7149                    .execute_batch("RELEASE authority_snapshot")
7150                    .map_err(|e| e.to_string())?;
7151                Ok(snapshot)
7152            }
7153            Err(error) => {
7154                self.conn
7155                    .execute_batch("ROLLBACK TO authority_snapshot; RELEASE authority_snapshot")
7156                    .map_err(|e| format!("{error}; rollback failed: {e}"))?;
7157                Err(error)
7158            }
7159        }
7160    }
7161
7162    /// Rows, coverage, and local revision from one SQLite read snapshot.
7163    /// §7.5 atomic snapshot read. An `owner` records a failure in
7164    /// `diagnostics_snapshot().query_failures` until its next successful read.
7165    pub fn query_snapshot(
7166        &mut self,
7167        sql: &str,
7168        params: &[QueryValue],
7169        coverage: &[WindowCoverage],
7170        owner: Option<QueryOwner<'_>>,
7171    ) -> Result<QuerySnapshot, String> {
7172        let result = snapshot_connection(&self.conn, sql, params, coverage);
7173        if let Some(owner) = owner {
7174            self.record_query_read(owner, result.as_ref().err());
7175        }
7176        result.map_err(|failure| failure.to_string())
7177    }
7178
7179    /// §7.6: record the outcome of one owned snapshot read, including reads a
7180    /// host ran on a separate connection. A success removes the owner's entry;
7181    /// an identical failure leaves it unchanged; a changed failure moves last.
7182    pub fn record_query_read(&mut self, owner: QueryOwner<'_>, failure: Option<&QueryReadFailure>) {
7183        let position = self
7184            .query_failures
7185            .iter()
7186            .position(|entry| entry.id == owner.id);
7187        let Some(failure) = failure else {
7188            if let Some(index) = position {
7189                self.query_failures.remove(index);
7190            }
7191            return;
7192        };
7193        let tables: Vec<String> = BTreeSet::from_iter(owner.tables.iter().copied())
7194            .into_iter()
7195            .map(str::to_owned)
7196            .collect();
7197        let code = failure.code.unwrap_or("client.query_failed").to_owned();
7198        if position.is_some_and(|index| {
7199            let entry = &self.query_failures[index];
7200            entry.tables == tables && entry.code == code && entry.sqlite_code == failure.sqlite_code
7201        }) {
7202            return;
7203        }
7204        if let Some(index) = position {
7205            self.query_failures.remove(index);
7206        }
7207        self.query_failures.push(DiagnosticQueryFailure {
7208            id: owner.id.to_owned(),
7209            tables,
7210            code,
7211            sqlite_code: failure.sqlite_code,
7212            at_ms: self.clock_now_ms(),
7213        });
7214        if self.query_failures.len() > MAX_DIAGNOSTIC_QUERY_FAILURES {
7215            self.query_failures.remove(0);
7216        }
7217    }
7218
7219    // -- request building ---------------------------------------------------------
7220
7221    fn build_request(&self, url_capable: bool) -> Result<(Message, RequestMeta), Box<SyncOutcome>> {
7222        #[cfg(feature = "bench-internals")]
7223        let _phase = self.benchmark_phases.start(Phase::RequestPrepare);
7224        let log_epoch = self.get_meta(LOG_EPOCH_KEY).map_err(|message| {
7225            Box::new(SyncOutcome::Failed {
7226                error_code: "storage.failed".into(),
7227                message,
7228                details: None,
7229                recommended_action: None,
7230            })
7231        })?;
7232        let max_commits = self.limits.max_push_commits_per_request;
7233        let max_ops = self
7234            .limits
7235            .max_push_operations_per_request
7236            .unwrap_or(PUSH_OPS_PER_REQUEST);
7237        let byte_cap = self.limits.max_push_request_bytes;
7238        // §4.2/§5.4: bit 3 is advertised iff the transport can fetch a
7239        // bare URL — capability negotiation, decided per transport.
7240        let accept = self.limits.accept.unwrap_or(if url_capable {
7241            DEFAULT_ACCEPT | ACCEPT_SIGNED_URLS
7242        } else {
7243            DEFAULT_ACCEPT
7244        });
7245        let storage_failed = |message: String| {
7246            Box::new(SyncOutcome::Failed {
7247                error_code: "storage.failed".into(),
7248                message,
7249                details: None,
7250                recommended_action: None,
7251            })
7252        };
7253        let header = Frame::ReqHeader {
7254            client_id: self.client_id.clone(),
7255            schema_version: self.schema.version,
7256            log_epoch: log_epoch.clone(),
7257            outbox_id: self.get_meta(OUTBOX_ID_KEY).map_err(storage_failed)?,
7258        };
7259        // §1.5: the lead frames are REQ_HEADER plus the optional ROUND_FAILURE,
7260        // which stays ahead of every push, pull, unit and progress frame; the
7261        // byte-capacity probes below measure this exact sequence.
7262        let mut lead = vec![header];
7263        lead.extend(
7264            self.round_failure
7265                .iter()
7266                .map(|code| Frame::RoundFailure { code: code.clone() }),
7267        );
7268        let registered: Vec<UnitRecord> = self
7269            .stored(load_units(&self.conn))
7270            .map_err(storage_failed)?
7271            .into_iter()
7272            .filter(|unit| unit.status == UnitStatus::Active)
7273            .collect();
7274        let registration_digest = ssp2::registration::registration_digest(
7275            registered
7276                .iter()
7277                .map(|unit| scope_pairs(&unit.scopes))
7278                .collect::<Vec<_>>()
7279                .iter()
7280                .map(Vec::as_slice),
7281        );
7282        let echo_digest = self
7283            .get_meta(ECHO_DIGEST_KEY)
7284            .map_err(storage_failed)?
7285            .filter(|hex| !hex.is_empty())
7286            .map(|hex| {
7287                crate::values::hex_to_bytes(&hex)
7288                    .ok()
7289                    .and_then(|bytes| <[u8; 32]>::try_from(bytes).ok())
7290                    .ok_or_else(|| {
7291                        "sync.local_corrupt: persisted echo digest is invalid".to_owned()
7292                    })
7293            })
7294            .transpose()
7295            .map_err(storage_failed)?;
7296        let pull = Frame::PullHeader {
7297            limit_commits: self.limits.limit_commits.unwrap_or(0),
7298            limit_snapshot_rows: self.limits.limit_snapshot_rows.unwrap_or(0),
7299            max_snapshot_pages: self.limits.max_snapshot_pages.unwrap_or(0),
7300            accept,
7301            cursor: self
7302                .stored(replica_cursor(&self.conn))
7303                .map_err(storage_failed)?,
7304            registration_digest,
7305            echo_digest,
7306        };
7307        let registration_digest = bytes_to_hex(&registration_digest);
7308        // §4.1: the list travels when it changed since the server stored it,
7309        // after `unregistered` and after a reset (both clear the stored digest).
7310        let travels = self
7311            .get_meta(REGISTERED_DIGEST_KEY)
7312            .map_err(storage_failed)?
7313            .as_deref()
7314            != Some(registration_digest.as_str());
7315        let mut sub_frames: Vec<Frame> = Vec::new();
7316        if travels {
7317            sub_frames.extend(registered.iter().map(|unit| Frame::Unit {
7318                scopes: scope_pairs(&unit.scopes),
7319            }));
7320        }
7321        // §4.3: one PROGRESS per catching-up unit, in registration order.
7322        sub_frames.extend(registered.iter().filter_map(|unit| {
7323            Some(Frame::Progress {
7324                id: unit.id.clone(),
7325                cursor: unit.cursor?,
7326                bootstrap_state: unit.bootstrap_state.clone().map(RawJson),
7327            })
7328        }));
7329        // The fixed frames (order-independent size) and the header+pull probe
7330        // both encode canonically, so the byte budget is the exact wire size.
7331        let fixed_bytes = match byte_cap {
7332            Some(_) => {
7333                let mut frames = lead.clone();
7334                frames.push(pull.clone());
7335                frames.extend(sub_frames.iter().cloned());
7336                encode_message(&Message {
7337                    wire_version: WIRE_VERSION,
7338                    msg_kind: MsgKind::Request,
7339                    frames,
7340                })
7341                .len()
7342            }
7343            None => 0,
7344        };
7345        if let Some(cap) = byte_cap {
7346            if fixed_bytes > cap {
7347                return Err(Self::push_capacity_outcome("bytes", cap, fixed_bytes, None));
7348            }
7349        }
7350        let base_bytes = match byte_cap {
7351            Some(_) => {
7352                let mut frames = lead.clone();
7353                frames.push(pull.clone());
7354                encode_message(&Message {
7355                    wire_version: WIRE_VERSION,
7356                    msg_kind: MsgKind::Request,
7357                    frames,
7358                })
7359                .len()
7360            }
7361            None => 0,
7362        };
7363        let total_outbox = if log_epoch.is_some() {
7364            self.outbox.len()
7365        } else {
7366            0
7367        };
7368        let mut push_frames: Vec<Frame> = Vec::new();
7369        let mut pushed_ids = Vec::new();
7370        let mut ops_in_request = 0usize;
7371        let mut bytes_in_request = 0usize;
7372        let mut deferred_commits = 0usize;
7373        #[cfg(feature = "bench-internals")]
7374        let outbox_phase = self.benchmark_phases.start(Phase::OutboxEncode);
7375        for (index, commit) in self.outbox.iter().take(total_outbox).enumerate() {
7376            // §6.1/§7.1: the commit-count and operation caps are hard. A
7377            // first commit over the operation cap is a typed capacity error
7378            // and stays queued; a later one defers with the complete suffix.
7379            if max_commits.is_some_and(|cap| push_frames.len() >= cap) {
7380                deferred_commits = total_outbox - index;
7381                break;
7382            }
7383            if ops_in_request + commit.ops.len() > max_ops {
7384                if push_frames.is_empty() {
7385                    return Err(Self::push_capacity_outcome(
7386                        "operations",
7387                        max_ops,
7388                        commit.ops.len(),
7389                        Some(commit.client_commit_id.as_str()),
7390                    ));
7391                }
7392                deferred_commits = total_outbox - index;
7393                break;
7394            }
7395            // §7.1 chained base: the commit waits for the change of the commit
7396            // it chains on, which supplies its rebased version.
7397            if self.outbox[..index]
7398                .iter()
7399                .flat_map(|earlier| &earlier.ops)
7400                .any(|earlier| commit.ops.iter().any(|op| chains_on(earlier, op)))
7401            {
7402                deferred_commits = total_outbox - index;
7403                break;
7404            }
7405            let operations = commit
7406                .ops
7407                .iter()
7408                .map(|op| {
7409                    let payload = match self
7410                        .schema
7411                        .table(&op.table)
7412                        .and_then(|table| Some((table, op.presented_values(table)?)))
7413                    {
7414                        Some((table, values)) => {
7415                            // §0: outbox entries encode at send time with the
7416                            // current codec (validated at mutate()). §6.1: the
7417                            // values map's keys are the sparse presence set; an
7418                            // insert-intent operation presents every column (§7.1).
7419                            // §5.11: encrypted columns are encrypted here, with
7420                            // an absent key-id selector read from the stored row.
7421                            // `sync_inner` runs `drop_unencodable_outbox` first,
7422                            // so an encrypt failure cannot reach this encode; the
7423                            // author seam validated every other failure.
7424                            let fallback =
7425                                self.stored_key_fallback(table, &op.row_id)
7426                                    .map_err(|message| {
7427                                        Box::new(SyncOutcome::Failed {
7428                                            error_code: "storage.failed".into(),
7429                                            message,
7430                                            details: None,
7431                                            recommended_action: None,
7432                                        })
7433                                    })?;
7434                            encode_sparse_row_json(
7435                                table,
7436                                &op.row_id,
7437                                &values,
7438                                &self.encryption,
7439                                fallback.as_ref(),
7440                            )
7441                            .ok()
7442                        }
7443                        None => None,
7444                    };
7445                    Ok(ssp2::model::Operation {
7446                        table: op.table.clone(),
7447                        row_id: op.row_id.clone(),
7448                        op: if op.upsert { Op::Upsert } else { Op::Delete },
7449                        base_version: op.base_version,
7450                        payload,
7451                    })
7452                })
7453                .collect::<Result<Vec<_>, Box<SyncOutcome>>>()?;
7454            // §6.3.2, §7.1: the commit's outbox sequence.
7455            let client_seq = self
7456                .conn
7457                .prepare_cached("SELECT seq FROM _syncular_outbox WHERE commit_id = ?1")
7458                .and_then(|mut statement| {
7459                    statement.query_row([&commit.client_commit_id], |row| row.get(0))
7460                })
7461                .map_err(|error| {
7462                    storage_failed(Self::sqlite_failure(&self.storage_failure, error))
7463                })?;
7464            let frame = Frame::PushCommit {
7465                client_commit_id: commit.client_commit_id.clone(),
7466                client_seq,
7467                operations,
7468            };
7469            let frame_bytes = match byte_cap {
7470                Some(_) => {
7471                    let mut frames = lead.clone();
7472                    frames.push(frame.clone());
7473                    frames.push(pull.clone());
7474                    encode_message(&Message {
7475                        wire_version: WIRE_VERSION,
7476                        msg_kind: MsgKind::Request,
7477                        frames,
7478                    })
7479                    .len()
7480                        - base_bytes
7481                }
7482                None => 0,
7483            };
7484            // The head is always encoded so the caller can report the full
7485            // projected request size; a later commit defers once it no longer
7486            // fits.
7487            if let Some(cap) = byte_cap {
7488                if !push_frames.is_empty() && fixed_bytes + bytes_in_request + frame_bytes > cap {
7489                    deferred_commits = total_outbox - index;
7490                    break;
7491                }
7492            }
7493            push_frames.push(frame);
7494            pushed_ids.push(commit.client_commit_id.clone());
7495            ops_in_request += commit.ops.len();
7496            bytes_in_request += frame_bytes;
7497        }
7498        #[cfg(feature = "bench-internals")]
7499        drop(outbox_phase);
7500        if let Some(cap) = byte_cap {
7501            if fixed_bytes + bytes_in_request > cap {
7502                // The head alone does not fit this budget; the commit stays
7503                // queued and no transport round is attempted.
7504                let blocked = if push_frames.is_empty() {
7505                    None
7506                } else {
7507                    self.outbox.first().map(|c| c.client_commit_id.as_str())
7508                };
7509                return Err(Self::push_capacity_outcome(
7510                    "bytes",
7511                    cap,
7512                    fixed_bytes + bytes_in_request,
7513                    blocked,
7514                ));
7515            }
7516        }
7517        let mut frames = lead;
7518        frames.extend(push_frames);
7519        frames.push(pull);
7520        frames.extend(sub_frames);
7521        let message = Message {
7522            wire_version: WIRE_VERSION,
7523            msg_kind: MsgKind::Request,
7524            frames,
7525        };
7526        Ok((
7527            message,
7528            RequestMeta {
7529                pushed_ids,
7530                registration_digest,
7531                accept,
7532                deferred_commits,
7533            },
7534        ))
7535    }
7536
7537    /// §7.1: a configured push capacity limit cannot carry this request.
7538    fn push_capacity_outcome(
7539        kind: &str,
7540        limit: usize,
7541        size: usize,
7542        client_commit_id: Option<&str>,
7543    ) -> Box<SyncOutcome> {
7544        let mut details = Map::new();
7545        details.insert("kind".to_owned(), Value::from(kind));
7546        details.insert("limit".to_owned(), Value::from(limit as u64));
7547        details.insert("size".to_owned(), Value::from(size as u64));
7548        if let Some(id) = client_commit_id {
7549            details.insert("clientCommitId".to_owned(), Value::from(id));
7550        }
7551        Box::new(SyncOutcome::Failed {
7552            error_code: "client.push_request_too_large".into(),
7553            message: "the push request exceeds a configured capacity limit".into(),
7554            details: Some(Value::Object(details)),
7555            recommended_action: None,
7556        })
7557    }
7558
7559    // -- sync -------------------------------------------------------------------
7560
7561    /// Cloneable observation handle; no client lock is needed to read it.
7562    pub fn progress(&self) -> ProgressObserver {
7563        self.progress.clone()
7564    }
7565
7566    pub fn sync(&mut self, transport: &mut dyn Transport) -> SyncOutcome {
7567        let prepared = match self.prepare_sync_round(transport.supports_url_fetch()) {
7568            Ok(prepared) => prepared,
7569            Err(outcome) => return *outcome,
7570        };
7571        // The round's network deadline rides `PreparedSyncRound` and is scoped
7572        // to each exchange, so uploads, continuations, the main request, and
7573        // every segment fetch share one monotonic anchor without leaving
7574        // deadline state on a reused transport.
7575        let mut next = prepared;
7576        loop {
7577            match self.apply_sync_round(next.exchange(transport)) {
7578                crate::AppliedSyncRound::Continue(prepared) => next = prepared,
7579                crate::AppliedSyncRound::Complete {
7580                    outcome, controls, ..
7581                } => {
7582                    if self.transport_enabled {
7583                        for text in controls {
7584                            let _ = transport.realtime_send(&text);
7585                        }
7586                    } else {
7587                        self.disconnect_realtime(transport);
7588                    }
7589                    return outcome;
7590                }
7591            }
7592        }
7593    }
7594
7595    fn finish_sync_round(&mut self, started_at_ms: i64, mut outcome: SyncOutcome) -> SyncOutcome {
7596        if let SyncOutcome::Failed {
7597            error_code,
7598            message,
7599            details,
7600            ..
7601        } = &mut outcome
7602        {
7603            if let Some((failure, code)) = self
7604                .storage_failure
7605                .borrow()
7606                .as_ref()
7607                .and_then(|failure| failure.code.map(|code| (failure, code)))
7608            {
7609                *error_code = code.to_owned();
7610                *message = match code {
7611                    "client.storage_busy" => "local SQLite storage is busy",
7612                    "client.storage_full" => "local SQLite storage is full",
7613                    "client.storage_io" => "local SQLite storage I/O failed",
7614                    "client.storage_corrupt" => "local SQLite storage is corrupt",
7615                    _ => unreachable!("unrecognized classified storage failure"),
7616                }
7617                .to_owned();
7618                *details = failure.details();
7619            }
7620        }
7621        let details = match &outcome {
7622            SyncOutcome::Failed { details, .. } => details.as_ref(),
7623            _ => None,
7624        };
7625        let cause_kind = details
7626            .and_then(|details| details.get("causeKind"))
7627            .and_then(Value::as_str)
7628            .and_then(|cause| match cause {
7629                "network" => Some(crate::TransportCauseKind::Network),
7630                "connect" => Some(crate::TransportCauseKind::Connect),
7631                "tls" => Some(crate::TransportCauseKind::Tls),
7632                "io" => Some(crate::TransportCauseKind::Io),
7633                "request" => Some(crate::TransportCauseKind::Request),
7634                "protocol" => Some(crate::TransportCauseKind::Protocol),
7635                "timeout" => Some(crate::TransportCauseKind::Timeout),
7636                "round_deadline" => Some(crate::TransportCauseKind::RoundDeadline),
7637                "aborted" => Some(crate::TransportCauseKind::Aborted),
7638                "body" => Some(crate::TransportCauseKind::Body),
7639                "status" => Some(crate::TransportCauseKind::Status),
7640                "redirect" => Some(crate::TransportCauseKind::Redirect),
7641                "unknown" => Some(crate::TransportCauseKind::Unknown),
7642                _ => None,
7643            });
7644        let http_status = if matches!(
7645            cause_kind,
7646            Some(
7647                crate::TransportCauseKind::Status
7648                    | crate::TransportCauseKind::Body
7649                    | crate::TransportCauseKind::Redirect
7650            )
7651        ) {
7652            details
7653                .and_then(|details| details.get("httpStatus"))
7654                .and_then(Value::as_f64)
7655                .filter(|status| status.fract() == 0.0 && (100.0..=599.0).contains(status))
7656                .map(|status| status as u16)
7657        } else {
7658            None
7659        };
7660        let retry_delay_ms = self.round_retry_delay_ms;
7661        self.progress.update(|p| match &outcome {
7662            SyncOutcome::Ok(report) | SyncOutcome::BudgetExhausted(report)
7663                if report.failed.is_empty() =>
7664            {
7665                p.state = ProgressState::Complete
7666            }
7667            SyncOutcome::Ok(_) | SyncOutcome::BudgetExhausted(_) => {
7668                p.state = ProgressState::Failed;
7669                p.error_code = Some("sync.scope_revoked".into());
7670            }
7671            SyncOutcome::Failed { error_code, .. } => {
7672                p.state = ProgressState::Failed;
7673                p.error_code = Some(Self::diagnostic_code(error_code));
7674                p.cause_kind = cause_kind;
7675                p.http_status = http_status;
7676                p.retry_delay_ms = retry_delay_ms;
7677            }
7678            SyncOutcome::RealtimeUnavailable { .. } => {
7679                p.state = ProgressState::Failed;
7680                p.error_code = Some(REALTIME_UNAVAILABLE_CODE.to_owned());
7681                p.retry_delay_ms = retry_delay_ms;
7682            }
7683        });
7684        let completed_at_ms = self.clock_now_ms();
7685        self.last_round = Some(match &outcome {
7686            SyncOutcome::Ok(report) | SyncOutcome::BudgetExhausted(report) => DiagnosticLastRound {
7687                status: "succeeded".to_owned(),
7688                started_at_ms,
7689                completed_at_ms,
7690                duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
7691                counters: Some(DiagnosticRoundCounters {
7692                    pushed: report.pushed,
7693                    applied: report.applied.len(),
7694                    rejected: report.rejected.len(),
7695                    retryable: report.retryable.len(),
7696                    conflicts: report.conflicts,
7697                    commits_applied: report.commits_applied,
7698                    segment_rows_applied: report.segment_rows_applied,
7699                    bootstrapping: report.bootstrapping.len(),
7700                    resets: report.resets.len(),
7701                    revoked: report.revoked.len(),
7702                    failed: report.failed.len(),
7703                    deferred_commits: report.deferred_commits,
7704                }),
7705                error_code: None,
7706                cause_kind: None,
7707                http_status: None,
7708            },
7709            SyncOutcome::Failed { error_code, .. } => DiagnosticLastRound {
7710                status: "failed".to_owned(),
7711                started_at_ms,
7712                completed_at_ms,
7713                duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
7714                counters: None,
7715                error_code: Some(Self::diagnostic_code(error_code)),
7716                cause_kind,
7717                http_status,
7718            },
7719            SyncOutcome::RealtimeUnavailable { .. } => DiagnosticLastRound {
7720                status: "failed".to_owned(),
7721                started_at_ms,
7722                completed_at_ms,
7723                duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
7724                counters: None,
7725                error_code: Some(REALTIME_UNAVAILABLE_CODE.to_owned()),
7726                cause_kind: None,
7727                http_status: None,
7728            },
7729        });
7730        outcome
7731    }
7732
7733    /// Capture one request on the client owner. Mutations made after this
7734    /// call remain in the outbox for a later round; no network I/O occurs here.
7735    pub fn prepare_sync_round(
7736        &mut self,
7737        url_capable: bool,
7738    ) -> Result<crate::PreparedSyncRound, Box<SyncOutcome>> {
7739        if !self.transport_enabled {
7740            return Err(Box::new(SyncOutcome::Failed {
7741                error_code: "sync.offline".into(),
7742                message: "transport gate is closed".into(),
7743                details: None,
7744                recommended_action: None,
7745            }));
7746        }
7747        if self.active_round.is_some() {
7748            return Err(Box::new(SyncOutcome::Failed {
7749                error_code: "client.round_in_progress".into(),
7750                message: "a sync round is already in flight".into(),
7751                details: None,
7752                recommended_action: None,
7753            }));
7754        }
7755        self.storage_failure.borrow_mut().take();
7756        self.progress.start();
7757        self.round_retry_delay_ms = None;
7758        let started_at_ms = self.clock_now_ms();
7759        let prepared = (|| {
7760            if self.stopped {
7761                return Err(Box::new(SyncOutcome::Ok(SyncReport {
7762                    schema_floor: self.schema_floor.clone(),
7763                    ..SyncReport::default()
7764                })));
7765            }
7766            if self.security_preflight {
7767                return Err(Box::new(SyncOutcome::Failed {
7768                    error_code: SECURITY_PREFLIGHT_REQUIRED_CODE.into(),
7769                    message: "security preflight must complete before syncing".into(),
7770                    details: None,
7771                    recommended_action: None,
7772                }));
7773            }
7774            self.set_sync_needed(false, false);
7775            self.drain_pending_evictions()
7776                .and_then(|_| self.drop_incompatible_outbox())
7777                .and_then(|_| self.drop_unencodable_outbox(true, None))
7778                .map_err(|message| {
7779                    Box::new(SyncOutcome::Failed {
7780                        error_code: "storage.failed".into(),
7781                        message,
7782                        details: None,
7783                        recommended_action: None,
7784                    })
7785                })?;
7786            // §5.11: reveal rows sealed under a key the keyring gained since.
7787            self.reveal_sealed_rows().map_err(|error| {
7788                Box::new(SyncOutcome::Failed {
7789                    error_code: error.code,
7790                    message: error.message,
7791                    details: error.details,
7792                    recommended_action: None,
7793                })
7794            })?;
7795            let has_log_epoch = self.get_meta(LOG_EPOCH_KEY).map_err(|message| {
7796                Box::new(SyncOutcome::Failed {
7797                    error_code: "storage.failed".into(),
7798                    message,
7799                    details: None,
7800                    recommended_action: None,
7801                })
7802            })?;
7803            let uploads = if has_log_epoch.is_some() && self.schema_has_blobs() {
7804                self.pending_blob_uploads().map_err(|e| {
7805                    Box::new(SyncOutcome::Failed {
7806                        error_code: e.code,
7807                        message: e.message,
7808                        details: e.details,
7809                        recommended_action: None,
7810                    })
7811                })?
7812            } else {
7813                Vec::new()
7814            };
7815            if self.realtime_state != RealtimeState::Connected
7816                && self.realtime_policy == RealtimePolicy::Required
7817            {
7818                let retry_delay_ms = self.schedule_background_retry();
7819                self.realtime_retry_delay_ms = Some(retry_delay_ms);
7820                return Err(Box::new(SyncOutcome::RealtimeUnavailable {
7821                    state: self.realtime_state,
7822                    reason_code: self.realtime_reason_code.clone(),
7823                    retry_delay_ms,
7824                }));
7825            }
7826            let (message, meta) = self.build_request(url_capable)?;
7827            let id = uuid::Uuid::new_v4();
7828            self.active_round = Some(id);
7829            let phase = if uploads.is_empty() {
7830                crate::round::RoundPhase::Sync
7831            } else {
7832                crate::round::RoundPhase::UploadGrant
7833            };
7834            Ok(crate::PreparedSyncRound {
7835                id,
7836                started_at_ms,
7837                message,
7838                meta,
7839                uploads: uploads.into(),
7840                phase,
7841                encoded: None,
7842                continuation: None,
7843                completed: None,
7844                realtime: self.realtime_state == RealtimeState::Connected,
7845                // §5.5: the owning unit's scope map, each value a one-element list.
7846                scopes: self
7847                    .stored(load_units(&self.conn))
7848                    .map_err(|message| {
7849                        Box::new(SyncOutcome::Failed {
7850                            error_code: "storage.failed".into(),
7851                            message,
7852                            details: None,
7853                            recommended_action: None,
7854                        })
7855                    })?
7856                    .into_iter()
7857                    .map(|unit| {
7858                        (
7859                            unit.id,
7860                            canonical_scope_json(&unit_scope_lists(&unit.scopes)),
7861                        )
7862                    })
7863                    .collect(),
7864                #[cfg(feature = "bench-internals")]
7865                benchmark_phases: self.benchmark_phases.clone(),
7866                progress: self.progress.clone(),
7867                fixed_now: self.now_ms,
7868                round_deadline_at: None,
7869            })
7870        })();
7871        prepared.map_err(|outcome| Box::new(self.finish_sync_round(started_at_ms, *outcome)))
7872    }
7873
7874    /// Discard the reply of an old authorization or registration context. Its
7875    /// submitted commits stay durable and can be retried with the same ids.
7876    pub fn cancel_sync_round(&mut self) {
7877        self.active_round = None;
7878    }
7879
7880    /// Apply captured network results on the owner, preserving all normal
7881    /// version, schema, content-address and cursor checks. No network I/O.
7882    pub fn apply_sync_round(
7883        &mut self,
7884        completed: crate::CompletedSyncRound,
7885    ) -> crate::AppliedSyncRound {
7886        let mut prepared = completed.prepared;
7887        if self.active_round != Some(prepared.id) || self.security_preflight {
7888            return crate::AppliedSyncRound::Complete {
7889                outcome: SyncOutcome::Failed {
7890                    error_code: "client.round_cancelled".into(),
7891                    message: "sync round belongs to a released client context".into(),
7892                    details: None,
7893                    recommended_action: None,
7894                },
7895                controls: Vec::new(),
7896                more: false,
7897                bootstrap_advanced: false,
7898            };
7899        }
7900        self.storage_failure.borrow_mut().take();
7901        let (response, mut downloads, transport_failed) = match completed.exchange {
7902            crate::round::ExchangeResult::Upload { id, result } => {
7903                let result = match result {
7904                    // §5.9.7: a non-retryable server refusal (`blob.too_large`)
7905                    // is the body's final outcome. The commits naming it take
7906                    // the incompatible-commit path with the server's code and
7907                    // leave this round's request; the round continues.
7908                    Err(error)
7909                        if error
7910                            .details
7911                            .as_ref()
7912                            .and_then(|details| details.get("retryable"))
7913                            == Some(&Value::Bool(false)) =>
7914                    {
7915                        self.conn
7916                            .prepare(
7917                                "SELECT commit_id FROM _syncular_blob_commit_refs WHERE blob_id = ?",
7918                            )
7919                            .and_then(|mut statement| {
7920                                statement
7921                                    .query_map(rusqlite::params![id], |row| row.get::<_, String>(0))?
7922                                    .collect::<Result<HashSet<_>, _>>()
7923                            })
7924                            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
7925                            .and_then(|refused| {
7926                                let commits = self
7927                                    .outbox
7928                                    .iter()
7929                                    .filter(|commit| refused.contains(&commit.client_commit_id))
7930                                    .cloned()
7931                                    .collect::<Vec<_>>();
7932                                if !commits.is_empty() {
7933                                    self.drop_outbox_commits(&commits, &error.code, &error.message)?;
7934                                }
7935                                prepared.message.frames.retain(|frame| {
7936                                    !matches!(frame, Frame::PushCommit { client_commit_id, .. }
7937                                        if refused.contains(client_commit_id))
7938                                });
7939                                prepared.meta.pushed_ids.retain(|id| !refused.contains(id));
7940                                prepared.continuation = Some(crate::round::RoundPhase::UploadGrant);
7941                                Ok(())
7942                            })
7943                            .map_err(|message| TransportError::new("client.failed", message))
7944                    }
7945                    result => result,
7946                };
7947                let result = result.and_then(|()| {
7948                    self.conn
7949                        .execute(
7950                            "DELETE FROM _syncular_blob_uploads WHERE blob_id = ?",
7951                            rusqlite::params![id],
7952                        )
7953                        .map(|_| ())
7954                        .map_err(|error| {
7955                            TransportError::new(
7956                                "client.failed",
7957                                Self::sqlite_failure(&self.storage_failure, error),
7958                            )
7959                        })
7960                });
7961                match result {
7962                    Ok(()) => return crate::AppliedSyncRound::Continue(prepared),
7963                    Err(TransportError {
7964                        code,
7965                        message,
7966                        details,
7967                    }) => {
7968                        self.active_round = None;
7969                        if Self::unreachable_transport(&details) {
7970                            // The round's work is still owed; the host's
7971                            // connectivity resume runs it.
7972                            self.set_sync_needed(true, false);
7973                        } else if Self::retryable_failure_code(&code) {
7974                            self.schedule_background_retry();
7975                        }
7976                        let recommended_action = recommended_action_of(&details);
7977                        let outcome = self.finish_sync_round(
7978                            prepared.started_at_ms,
7979                            SyncOutcome::Failed {
7980                                error_code: code,
7981                                message,
7982                                details,
7983                                recommended_action,
7984                            },
7985                        );
7986                        return crate::AppliedSyncRound::Complete {
7987                            outcome,
7988                            controls: Vec::new(),
7989                            more: false,
7990                            bootstrap_advanced: false,
7991                        };
7992                    }
7993                }
7994            }
7995            crate::round::ExchangeResult::Reply {
7996                response,
7997                downloads,
7998                transport_failed,
7999            } => (response, downloads, transport_failed),
8000        };
8001        self.active_round = None;
8002        let before = self.resume_tokens();
8003        let outcome = (|| {
8004            let response = match response {
8005                Ok(response) => {
8006                    // §1.5: a complete, decodable response ends the report, an
8007                    // in-band ERROR included (the server already recorded it).
8008                    self.round_failure = None;
8009                    response
8010                }
8011                Err(TransportError {
8012                    code,
8013                    message,
8014                    details,
8015                }) => {
8016                    // §1.5: the server cannot see a round whose transport
8017                    // failed or whose response never decoded, so its code rides
8018                    // the following requests. A code the frame grammar rejects
8019                    // is not reportable, and the previous code stays.
8020                    if ssp2::model::valid_round_failure_code(&code) {
8021                        self.round_failure = Some(code.clone());
8022                    }
8023                    if transport_failed {
8024                        if prepared.realtime {
8025                            self.set_realtime_state(RealtimeState::Lost, Some(&code));
8026                        }
8027                        if let Err(message) = self.record_lease_error(&code) {
8028                            return SyncOutcome::Failed {
8029                                error_code: "storage.failed".to_owned(),
8030                                message,
8031                                details: None,
8032                                recommended_action: None,
8033                            };
8034                        }
8035                    }
8036                    if Self::retryable_failure_code(&code) && !Self::unreachable_transport(&details)
8037                    {
8038                        let delay = self.schedule_background_retry();
8039                        if prepared.realtime && transport_failed {
8040                            self.realtime_retry_delay_ms = Some(delay);
8041                        }
8042                    }
8043                    let recommended_action = recommended_action_of(&details);
8044                    return SyncOutcome::Failed {
8045                        error_code: code,
8046                        message,
8047                        details,
8048                        recommended_action,
8049                    };
8050                }
8051            };
8052            let mut outcome = self.process_response(&mut downloads, response, &prepared.meta);
8053            if let SyncOutcome::Ok(report) = &mut outcome {
8054                report.deferred_commits = prepared.meta.deferred_commits;
8055            }
8056            match &outcome {
8057                SyncOutcome::Ok(_) => self.reset_background_retry(),
8058                SyncOutcome::Failed {
8059                    error_code,
8060                    details,
8061                    ..
8062                } if Self::retryable_failure_code(error_code)
8063                    && !Self::unreachable_transport(details) =>
8064                {
8065                    self.schedule_background_retry();
8066                }
8067                _ => {}
8068            }
8069            if prepared.meta.deferred_commits > 0 {
8070                self.set_sync_needed(true, true);
8071            }
8072            self.previous_version_lifetime_check();
8073            outcome
8074        })();
8075        match &outcome {
8076            // §7.1: a unit still catching up needs another round, also when the
8077            // drive that started it ends early.
8078            SyncOutcome::Ok(report) if !report.bootstrapping.is_empty() => {
8079                self.set_sync_needed(true, false)
8080            }
8081            // The round's work is still owed; the host's connectivity resume
8082            // runs it.
8083            SyncOutcome::Failed { details, .. } if Self::unreachable_transport(details) => {
8084                self.set_sync_needed(true, false)
8085            }
8086            _ => {}
8087        }
8088        let more = self.transport_enabled
8089            && matches!(&outcome, SyncOutcome::Ok(report)
8090                if report.schema_floor.is_none()
8091                    && (!report.bootstrapping.is_empty()
8092                        || report.commits_applied > 0
8093                        || report.segment_rows_applied > 0
8094                        || !report.resets.is_empty()
8095                        || report.deferred_commits > 0
8096                        || self.sync_needed));
8097        let bootstrap_advanced = matches!(&outcome, SyncOutcome::Ok(report) if report.segment_rows_applied > 0 && !report.bootstrapping.is_empty())
8098            && before != self.resume_tokens();
8099        crate::AppliedSyncRound::Complete {
8100            outcome: self.finish_sync_round(prepared.started_at_ms, outcome),
8101            controls: if self.transport_enabled {
8102                downloads.controls
8103            } else {
8104                Vec::new()
8105            },
8106            more,
8107            bootstrap_advanced,
8108        }
8109    }
8110
8111    /// Every unit's resume token: a bootstrap that advanced changes one (§4.7).
8112    fn resume_tokens(&self) -> Option<Vec<(String, Option<String>)>> {
8113        load_units(&self.conn).ok().map(|units| {
8114            units
8115                .into_iter()
8116                .map(|unit| (unit.id, unit.bootstrap_state))
8117                .collect()
8118        })
8119    }
8120
8121    /// §7.8: one idle drive's default total deadline, the TS client's value.
8122    pub const SYNC_IDLE_TIMEOUT_MS: u64 = 60_000;
8123
8124    pub fn sync_until_idle(
8125        &mut self,
8126        transport: &mut dyn Transport,
8127        max_rounds: Option<u32>,
8128    ) -> SyncOutcome {
8129        self.sync_until_idle_with_deadline(transport, max_rounds, Some(Self::SYNC_IDLE_TIMEOUT_MS))
8130    }
8131
8132    /// As [`SyncClient::sync_until_idle`], with an explicit total deadline in
8133    /// milliseconds (`None` disables it). The deadline is checked between
8134    /// rounds; one blocking transport call is bounded by the transport's own
8135    /// whole-round deadline (§1.8).
8136    pub fn sync_until_idle_with_deadline(
8137        &mut self,
8138        transport: &mut dyn Transport,
8139        max_rounds: Option<u32>,
8140        timeout_ms: Option<u64>,
8141    ) -> SyncOutcome {
8142        // §7.8: an explicit budget is a strict positive round count. Zero is a
8143        // caller error, not an exhaustion.
8144        if max_rounds == Some(0) {
8145            return SyncOutcome::Failed {
8146                details: None,
8147                error_code: "sync.invalid_request".into(),
8148                message: "maxRounds must be a positive integer".into(),
8149                recommended_action: None,
8150            };
8151        }
8152        if timeout_ms.is_some_and(|ms| ms == 0 || i64::try_from(ms).is_err()) {
8153            return SyncOutcome::Failed {
8154                details: None,
8155                error_code: "sync.invalid_request".into(),
8156                message: "timeoutMs must be a positive integer".into(),
8157                recommended_action: None,
8158            };
8159        }
8160        let started_at_ms = self.clock_now_ms();
8161        let deadline_ms = timeout_ms.map(|ms| started_at_ms.saturating_add(ms as i64));
8162        self.sync_until_idle_bounded(transport, max_rounds, started_at_ms, deadline_ms)
8163    }
8164
8165    fn sync_until_idle_bounded(
8166        &mut self,
8167        transport: &mut dyn Transport,
8168        max_rounds: Option<u32>,
8169        started_at_ms: i64,
8170        deadline_ms: Option<i64>,
8171    ) -> SyncOutcome {
8172        let rounds = max_rounds.unwrap_or(20);
8173        let mut aggregate = SyncReport::default();
8174        let mut spent = 0;
8175        while spent < rounds {
8176            if let Some(deadline_ms) = deadline_ms {
8177                if self.clock_now_ms() >= deadline_ms {
8178                    return SyncOutcome::Failed {
8179                        error_code: "sync.idle_timeout".into(),
8180                        message: "sync_until_idle exceeded its deadline".into(),
8181                        details: Some(serde_json::json!({
8182                            "timeoutMs": deadline_ms
8183                                .saturating_sub(started_at_ms) as u64,
8184                            "rounds": spent,
8185                        })),
8186                        recommended_action: Some("retryLater".into()),
8187                    };
8188                }
8189            }
8190            spent += 1;
8191            let before = self.resume_tokens();
8192            match self.sync(transport) {
8193                SyncOutcome::Failed {
8194                    error_code,
8195                    message,
8196                    details,
8197                    recommended_action,
8198                } => {
8199                    return SyncOutcome::Failed {
8200                        details,
8201                        error_code,
8202                        message,
8203                        recommended_action,
8204                    };
8205                }
8206                outcome @ SyncOutcome::RealtimeUnavailable { .. } => return outcome,
8207                SyncOutcome::BudgetExhausted(_) => {
8208                    unreachable!("sync() never exhausts a round budget")
8209                }
8210                SyncOutcome::Ok(report) => {
8211                    if max_rounds.is_none()
8212                        && report.segment_rows_applied > 0
8213                        && !report.bootstrapping.is_empty()
8214                        && before != self.resume_tokens()
8215                    {
8216                        spent = 0;
8217                    }
8218                    // TS parity: a disabled transport or a schema floor ends
8219                    // the loop. The latest round still merges first.
8220                    let stop = !self.transport_enabled() || report.schema_floor.is_some();
8221                    aggregate.merge(&report);
8222                    if stop {
8223                        return SyncOutcome::Ok(aggregate);
8224                    }
8225                    // §4.5: pull again whenever the response contained
8226                    // commits or segments; resets re-bootstrap; a pending
8227                    // resume token continues paging (§4.7); deferred outbox
8228                    // commits and a raised sync-needed signal push on the next
8229                    // round.
8230                    let more = !report.bootstrapping.is_empty()
8231                        || report.commits_applied > 0
8232                        || report.segment_rows_applied > 0
8233                        || !report.resets.is_empty()
8234                        || report.deferred_commits > 0
8235                        || self.sync_needed;
8236                    if !more {
8237                        return SyncOutcome::Ok(aggregate);
8238                    }
8239                }
8240            }
8241        }
8242        // §7.8: partial success. The aggregate report is retained and the run
8243        // is explicitly not idle.
8244        SyncOutcome::BudgetExhausted(aggregate)
8245    }
8246
8247    fn process_response(
8248        &mut self,
8249        transport: &mut dyn Transport,
8250        response: Message,
8251        meta: &RequestMeta,
8252    ) -> SyncOutcome {
8253        #[cfg(feature = "bench-internals")]
8254        let _phase = self.benchmark_phases.start(Phase::ResponseApply);
8255        let mut report = SyncReport {
8256            pushed: meta.pushed_ids.len() as u32,
8257            ..SyncReport::default()
8258        };
8259        let mut rejection_details_by_commit: HashMap<String, BTreeMap<i32, RejectionDetails>> =
8260            HashMap::new();
8261        let pushed_ids = meta
8262            .pushed_ids
8263            .iter()
8264            .map(String::as_str)
8265            .collect::<HashSet<_>>();
8266        let mut last_final_push_result_id: Option<String> = None;
8267        for frame in &response.frames {
8268            match frame {
8269                Frame::PushResultDetails {
8270                    client_commit_id,
8271                    entries,
8272                } => {
8273                    let details = rejection_details_by_commit
8274                        .entry(client_commit_id.clone())
8275                        .or_default();
8276                    for entry in entries {
8277                        let parsed = match RejectionDetails::parse(&entry.details.0) {
8278                            Ok(value) => value,
8279                            Err(message) => {
8280                                return SyncOutcome::Failed {
8281                                    details: None,
8282                                    error_code: "sync.invalid_request".to_owned(),
8283                                    message,
8284                                    recommended_action: None,
8285                                };
8286                            }
8287                        };
8288                        details.insert(entry.op_index, parsed);
8289                    }
8290                }
8291                Frame::PushResult {
8292                    client_commit_id,
8293                    status,
8294                    results,
8295                    ..
8296                } if pushed_ids.contains(client_commit_id.as_str())
8297                    && Self::push_result_is_final(*status, results) =>
8298                {
8299                    last_final_push_result_id = Some(client_commit_id.clone());
8300                }
8301                _ => {}
8302            }
8303        }
8304        let mut frames = response.frames.into_iter().peekable();
8305        match frames.next() {
8306            Some(Frame::RespHeader {
8307                required_schema_version,
8308                latest_schema_version,
8309                log_epoch,
8310                reset_required,
8311            }) => {
8312                if let Some(required) = required_schema_version {
8313                    // §1.6 schema-floor response: nothing else is processed;
8314                    // stop syncing and surface the upgrade requirement.
8315                    let floor = SchemaFloor {
8316                        required_schema_version: Some(required),
8317                        latest_schema_version,
8318                    };
8319                    if let Err(message) = self.set_schema_floor(Some(floor.clone())) {
8320                        return SyncOutcome::Failed {
8321                            details: None,
8322                            error_code: "storage.failed".to_owned(),
8323                            message,
8324                            recommended_action: None,
8325                        };
8326                    }
8327                    report.schema_floor = Some(floor);
8328                    return SyncOutcome::Ok(report);
8329                }
8330                if reset_required {
8331                    if frames.next().is_some() {
8332                        return SyncOutcome::Failed {
8333                            details: None,
8334                            error_code: "client.invalid_host_response".to_owned(),
8335                            message: "log-epoch reset response contains body frames".to_owned(),
8336                            recommended_action: None,
8337                        };
8338                    }
8339                    match self.run_log_epoch_reset(&log_epoch) {
8340                        Ok(resets) => {
8341                            report.resets = resets;
8342                            return SyncOutcome::Ok(report);
8343                        }
8344                        Err(message) => {
8345                            return SyncOutcome::Failed {
8346                                details: None,
8347                                error_code: "sync.local_corrupt".to_owned(),
8348                                message,
8349                                recommended_action: None,
8350                            };
8351                        }
8352                    }
8353                }
8354                let stored_epoch = match self.get_meta(LOG_EPOCH_KEY) {
8355                    Ok(stored) => stored,
8356                    Err(message) => {
8357                        return SyncOutcome::Failed {
8358                            details: None,
8359                            error_code: "storage.failed".to_owned(),
8360                            message,
8361                            recommended_action: None,
8362                        };
8363                    }
8364                };
8365                if stored_epoch.as_deref() != Some(log_epoch.as_str()) {
8366                    return SyncOutcome::Failed {
8367                        details: None,
8368                        error_code: "client.invalid_host_response".to_owned(),
8369                        message: "server changed logEpoch without requiring a reset".to_owned(),
8370                        recommended_action: None,
8371                    };
8372                }
8373            }
8374            _ => {
8375                return SyncOutcome::Failed {
8376                    details: None,
8377                    error_code: "sync.invalid_request".to_owned(),
8378                    message: "response does not start with RESP_HEADER".to_owned(),
8379                    recommended_action: None,
8380                };
8381            }
8382        }
8383
8384        if let Err(message) = self.begin_coalesced_apply() {
8385            return SyncOutcome::Failed {
8386                details: None,
8387                error_code: "storage.failed".into(),
8388                message,
8389                recommended_action: None,
8390            };
8391        }
8392        let mut failure: Option<(String, String, Option<String>, Option<Value>)> = None;
8393        // §4.5 client application: the units whose persisted echo a live
8394        // change must match, fixed at LIVE_START after every SUB_END persisted.
8395        let mut live: Option<(LiveStatus, String, Vec<UnitRecord>)> = None;
8396        let mut live_ended = false;
8397        while let Some(frame) = frames.next() {
8398            match frame {
8399                frame @ Frame::PushResult { .. } => {
8400                    let successful = matches!(
8401                        &frame,
8402                        Frame::PushResult {
8403                            status: PushStatus::Applied | PushStatus::Cached,
8404                            ..
8405                        }
8406                    );
8407                    let mut results = vec![frame];
8408                    if successful {
8409                        while matches!(
8410                            frames.peek(),
8411                            Some(Frame::PushResult {
8412                                status: PushStatus::Applied | PushStatus::Cached,
8413                                ..
8414                            })
8415                        ) {
8416                            if let Some(next) = frames.next() {
8417                                results.push(next);
8418                            }
8419                        }
8420                    }
8421                    if self
8422                        .handle_push_results(
8423                            &results,
8424                            &pushed_ids,
8425                            &rejection_details_by_commit,
8426                            &mut report,
8427                            last_final_push_result_id.as_deref(),
8428                        )
8429                        .is_err()
8430                    {
8431                        failure = Some((
8432                            "client.outcome_persistence_failed".to_owned(),
8433                            "local commit outcome could not be persisted".to_owned(),
8434                            None,
8435                            None,
8436                        ));
8437                        break;
8438                    }
8439                }
8440                Frame::PushResultDetails { .. } => {}
8441                Frame::SubStart {
8442                    id,
8443                    status,
8444                    reason_code,
8445                    bootstrap,
8446                    fenced,
8447                } => {
8448                    // §1.4 rule 5: an ERROR closes the section; its body still
8449                    // applied, its SUB_END values are never persisted.
8450                    let mut body = Vec::new();
8451                    let mut sub_end: Option<(i64, Option<String>)> = None;
8452                    let mut error: Option<Frame> = None;
8453                    for inner in frames.by_ref() {
8454                        match inner {
8455                            Frame::SubEnd {
8456                                next_cursor,
8457                                bootstrap_state,
8458                            } => {
8459                                sub_end = Some((next_cursor, bootstrap_state.map(|r| r.0)));
8460                                break;
8461                            }
8462                            inner @ Frame::Error { .. } => {
8463                                error = Some(inner);
8464                                break;
8465                            }
8466                            Frame::Unknown { .. } => {}
8467                            other => body.push(other),
8468                        }
8469                    }
8470                    let ended = sub_end.is_some();
8471                    let section = SectionStart {
8472                        id,
8473                        status,
8474                        reason_code,
8475                        bootstrap,
8476                        fenced,
8477                    };
8478                    match self.process_section(
8479                        transport,
8480                        &section,
8481                        body,
8482                        sub_end,
8483                        meta,
8484                        &mut report,
8485                    ) {
8486                        Err(SectionError::Abort(code, message)) => {
8487                            failure = Some((code, message, None, None));
8488                            break;
8489                        }
8490                        Err(SectionError::Transport(error)) => {
8491                            failure = Some((error.code, error.message, None, error.details));
8492                            break;
8493                        }
8494                        _ => {}
8495                    }
8496                    if ended {
8497                        // The counters stay: `complete` and `failed` report them.
8498                        self.progress.update(|p| {
8499                            p.unit_id = None;
8500                            p.table = None;
8501                            p.segment_id = None;
8502                        });
8503                    }
8504                    match error {
8505                        Some(Frame::Error {
8506                            code,
8507                            message,
8508                            recommended_action,
8509                            details,
8510                            ..
8511                        }) => {
8512                            failure = Some(Self::error_frame_failure(
8513                                code,
8514                                message,
8515                                recommended_action,
8516                                details,
8517                            ));
8518                            break;
8519                        }
8520                        Some(_) => unreachable!("only an ERROR closes a section early"),
8521                        None if !ended => {
8522                            failure = Some((
8523                                "sync.invalid_request".to_owned(),
8524                                "unit section without SUB_END".to_owned(),
8525                                None,
8526                                None,
8527                            ));
8528                            break;
8529                        }
8530                        None => {}
8531                    }
8532                }
8533                Frame::LiveStart {
8534                    status,
8535                    reason_code,
8536                } => {
8537                    let receivers = match self.stored(load_units(&self.conn)) {
8538                        Ok(units) => units
8539                            .into_iter()
8540                            .filter(|unit| {
8541                                unit.status == UnitStatus::Active
8542                                    && unit.echo.is_some()
8543                                    && !unit.bootstrapping()
8544                            })
8545                            .collect(),
8546                        Err(message) => {
8547                            failure = Some(("storage.failed".to_owned(), message, None, None));
8548                            break;
8549                        }
8550                    };
8551                    live = Some((status, reason_code, receivers));
8552                }
8553                Frame::Commit {
8554                    commit_seq,
8555                    tables,
8556                    changes,
8557                    ..
8558                } => {
8559                    let Some((LiveStatus::Active, _, receivers)) = &live else {
8560                        continue;
8561                    };
8562                    let receivers = receivers.clone();
8563                    match self.apply_live_commit(
8564                        &receivers,
8565                        &tables,
8566                        changes,
8567                        commit_seq,
8568                        AppliedServerCommitSource::Pull,
8569                    ) {
8570                        Ok(applied) => report.commits_applied += u32::from(applied),
8571                        Err(error) => {
8572                            failure = Some((error.code, error.message, None, error.details));
8573                            break;
8574                        }
8575                    }
8576                }
8577                Frame::LiveEnd {
8578                    through_seq,
8579                    echo_digest,
8580                } => {
8581                    let Some((status, reason_code, _)) = live.take() else {
8582                        continue;
8583                    };
8584                    live_ended = true;
8585                    live = Some((status, String::new(), Vec::new()));
8586                    if let Err(message) = self.finish_live_section(
8587                        status,
8588                        &reason_code,
8589                        through_seq,
8590                        echo_digest,
8591                        &meta.registration_digest,
8592                        &mut report,
8593                    ) {
8594                        failure = Some(("storage.failed".to_owned(), message, None, None));
8595                        break;
8596                    }
8597                }
8598                Frame::Lease {
8599                    lease_id,
8600                    expires_at_ms,
8601                } => {
8602                    // §7.3.5: persist the opaque lease; a fresh lease clears
8603                    // any prior lease error (the outage/revocation is over).
8604                    if let Err(message) = self.set_lease_state(Some(LeaseState {
8605                        lease_id: Some(lease_id),
8606                        expires_at_ms: Some(expires_at_ms),
8607                        error_code: None,
8608                    })) {
8609                        failure = Some(("storage.failed".to_owned(), message, None, None));
8610                        break;
8611                    }
8612                }
8613                Frame::Error {
8614                    code,
8615                    message,
8616                    recommended_action,
8617                    details,
8618                    ..
8619                } => {
8620                    // §1.6: the whole request failed; already-completed
8621                    // units keep their applied data and cursors.
8622                    failure = Some(Self::error_frame_failure(
8623                        code,
8624                        message,
8625                        recommended_action,
8626                        details,
8627                    ));
8628                    break;
8629                }
8630                Frame::Unknown { .. } => {}
8631                _ => {
8632                    failure = Some((
8633                        "sync.invalid_request".to_owned(),
8634                        "unexpected frame in response".to_owned(),
8635                        None,
8636                        None,
8637                    ));
8638                    break;
8639                }
8640            }
8641        }
8642
8643        let applied = match failure {
8644            Some(failure) => Err(failure),
8645            // Local apply boundaries already reconciled their affected overlay.
8646            // Refuse round success if any remaining dirty work fails to
8647            // reconcile. §4.8 E1: the push half may have drained commits that
8648            // pinned rows of a shrunk window unit, so retry deferred evictions.
8649            None => self
8650                .rebuild_overlay_if_dirty()
8651                .and_then(|()| self.drain_pending_evictions())
8652                .map_err(|message| ("storage.failed".to_owned(), message, None, None)),
8653        };
8654        // §1.4 rule 4: frames completed before a failure stay committed; a
8655        // failed COMMIT of the coalesced transaction fails the round.
8656        if let Err(message) = self.finish_coalesced_apply() {
8657            return SyncOutcome::Failed {
8658                details: None,
8659                error_code: "storage.failed".into(),
8660                message,
8661                recommended_action: None,
8662            };
8663        }
8664        if let Err((error_code, message, recommended_action, details)) = applied {
8665            return SyncOutcome::Failed {
8666                details,
8667                error_code,
8668                message,
8669                recommended_action,
8670            };
8671        }
8672        let catching_up = match self.stored(load_units(&self.conn)) {
8673            Ok(units) => units
8674                .into_iter()
8675                .filter(|unit| unit.status == UnitStatus::Active && unit.cursor.is_some())
8676                .map(|unit| unit.id)
8677                .collect::<Vec<_>>(),
8678            Err(message) => {
8679                return SyncOutcome::Failed {
8680                    details: None,
8681                    error_code: "storage.failed".into(),
8682                    message,
8683                    recommended_action: None,
8684                }
8685            }
8686        };
8687        // §7.1: a response without LIVE_END, one answered `unregistered` or
8688        // `reset`, or a unit still catching up leaves sync-needed raised;
8689        // after `unregistered` the next round starts at once.
8690        let status = live.map(|(status, _, _)| status);
8691        if !live_ended || status != Some(LiveStatus::Active) || !catching_up.is_empty() {
8692            self.set_sync_needed(true, status == Some(LiveStatus::Unregistered));
8693        }
8694        self.ack_after_pull(transport);
8695        // §7.4.5: the reset is over once the first post-reset pull round
8696        // leaves no unit catching up: the tables are rebuilt and current.
8697        if self.upgrading && catching_up.is_empty() {
8698            self.set_upgrading(false);
8699        }
8700        report.bootstrapping = catching_up;
8701        SyncOutcome::Ok(report)
8702    }
8703
8704    /// §1.6 / §10.1: the failure an in-band `ERROR` frame reports.
8705    fn error_frame_failure(
8706        code: String,
8707        message: String,
8708        recommended_action: String,
8709        details: Option<RawJson>,
8710    ) -> (String, String, Option<String>, Option<Value>) {
8711        let recommended_action = (!recommended_action.is_empty()).then_some(recommended_action);
8712        let details = details
8713            .and_then(|details| serde_json::from_str::<Value>(&details.0).ok())
8714            .filter(Value::is_object);
8715        (code, message, recommended_action, details)
8716    }
8717
8718    /// §4.5 at `LIVE_END` of a pull: join rule J for an `active` section, the
8719    /// §4.6 replica reset for `reset`, nothing for `unregistered`; then the
8720    /// replica cursor, the echo digest and the digest of the list the server
8721    /// now stores.
8722    fn finish_live_section(
8723        &mut self,
8724        status: LiveStatus,
8725        reason_code: &str,
8726        through_seq: i64,
8727        echo_digest: Option<[u8; 32]>,
8728        registration_digest: &str,
8729        report: &mut SyncReport,
8730    ) -> Result<(), String> {
8731        let rollback = self.capture();
8732        let resets = report.resets.len();
8733        self.begin_observation("syncular_live_end")?;
8734        let finished = (|| {
8735            let mut batch = ChangeAccumulator::default();
8736            let before = self.stored(replica_cursor(&self.conn))?;
8737            let mut cursor = if status == LiveStatus::Unregistered {
8738                before
8739            } else {
8740                -1
8741            };
8742            let joined = before.max(through_seq);
8743            for unit in self.stored(load_units(&self.conn))? {
8744                if unit.status != UnitStatus::Active || unit.bootstrapping() {
8745                    continue;
8746                }
8747                match status {
8748                    // §4.6 replica reset: every live unit bootstraps again;
8749                    // post-bootstrap units keep their cursors.
8750                    LiveStatus::Reset if unit.cursor.is_none() => {
8751                        for (base_key, value) in self.stored(window_refs(&self.conn, &unit.id))? {
8752                            batch.window(&base_key, &value);
8753                        }
8754                        report.resets.push(unit.id.clone());
8755                        self.stored(save_unit(
8756                            &self.conn,
8757                            &UnitRecord {
8758                                reason_code: Some(reason_code.to_owned()),
8759                                cursor: Some(-1),
8760                                seeded: Vec::new(),
8761                                ..unit
8762                            },
8763                        ))?;
8764                    }
8765                    // §4.5 join rule J.
8766                    LiveStatus::Active => {
8767                        let next = unit.cursor.unwrap_or(joined).max(through_seq);
8768                        if next >= joined {
8769                            cursor = joined;
8770                        }
8771                        if unit.cursor.is_some() {
8772                            self.stored(save_unit(
8773                                &self.conn,
8774                                &UnitRecord {
8775                                    cursor: (next < joined).then_some(next),
8776                                    ..unit
8777                                },
8778                            ))?;
8779                        }
8780                    }
8781                    _ => {}
8782                }
8783            }
8784            self.set_meta(REPLICA_CURSOR_KEY, &cursor.to_string())?;
8785            self.set_meta(
8786                ECHO_DIGEST_KEY,
8787                &echo_digest
8788                    .map(|digest| bytes_to_hex(&digest))
8789                    .unwrap_or_default(),
8790            )?;
8791            if status == LiveStatus::Unregistered {
8792                self.delete_meta(REGISTERED_DIGEST_KEY)?;
8793            } else {
8794                self.set_meta(REGISTERED_DIGEST_KEY, registration_digest)?;
8795            }
8796            self.finish_observation("syncular_live_end", batch)
8797        })();
8798        if finished.is_err() {
8799            self.rollback_observation("syncular_live_end");
8800            self.restore(rollback);
8801            report.resets.truncate(resets);
8802        }
8803        finished
8804    }
8805
8806    // -- push results (§6.3, §7.2) ------------------------------------------------
8807
8808    fn handle_push_results(
8809        &mut self,
8810        frames: &[Frame],
8811        pushed_ids: &HashSet<&str>,
8812        rejection_details_by_commit: &HashMap<String, BTreeMap<i32, RejectionDetails>>,
8813        report: &mut SyncReport,
8814        last_final_push_result_id: Option<&str>,
8815    ) -> Result<(), String> {
8816        self.begin_observation("syncular_push_results")?;
8817        let rollback = self.capture();
8818        let conflict_count = self.conflicts.len();
8819        let applied_count = report.applied.len();
8820        let rejected_count = report.rejected.len();
8821        let retryable_count = report.retryable.len();
8822        let mut drained = Vec::new();
8823        let mut batch = ChangeAccumulator::default();
8824        let persisted = (|| {
8825            for frame in frames {
8826                let Frame::PushResult {
8827                    client_commit_id,
8828                    status,
8829                    results,
8830                    commit_seq,
8831                } = frame
8832                else {
8833                    return Err("unexpected acknowledgement frame".to_owned());
8834                };
8835                if !pushed_ids.contains(client_commit_id.as_str()) {
8836                    continue;
8837                }
8838                let Some(index) = self
8839                    .outbox
8840                    .iter()
8841                    .position(|commit| commit.client_commit_id == *client_commit_id)
8842                else {
8843                    continue;
8844                };
8845                let status = *status;
8846                let rejection_details = rejection_details_by_commit.get(client_commit_id);
8847                let operations = self.outbox[index].ops.clone();
8848                match status {
8849                    PushStatus::Applied | PushStatus::Cached => {
8850                        // §7.2: a lost ack replays as `cached` — proceed as if the
8851                        // ack had arrived.
8852                        let sequence =
8853                            commit_seq.ok_or_else(|| "ACK lacks a commit sequence".to_owned())?;
8854                        let affected: HashSet<_> = operations
8855                            .iter()
8856                            .map(|op| (op.table.as_str(), op.row_id.as_str()))
8857                            .collect();
8858                        let prior = self.acknowledged_operations(Some(&affected), None)?;
8859                        for (op_index, operation) in operations.iter().enumerate() {
8860                            let delivered: bool = self.conn.query_row("SELECT EXISTS(SELECT 1 FROM _syncular_row_deliveries WHERE tbl=?1 AND id=?2 AND commit_seq>=?3)", rusqlite::params![operation.table, operation.row_id, sequence], |row| row.get(0))
8861                                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
8862                            if delivered {
8863                                continue;
8864                            }
8865                            let table = self
8866                                .schema
8867                                .table(&operation.table)
8868                                .ok_or_else(|| "sync.unknown_table".to_owned())?;
8869                            let mut intended: Map<String, Value> = self
8870                                .stored_row(table, &operation.row_id, true)?
8871                                .map(|(row, _)| {
8872                                    table
8873                                        .columns
8874                                        .iter()
8875                                        .zip(&row)
8876                                        .map(|(column, value)| {
8877                                            (column.name.clone(), column_value_to_json(value))
8878                                        })
8879                                        .collect()
8880                                })
8881                                .unwrap_or_default();
8882                            let mut scope_image = intended.clone();
8883                            for op in prior
8884                                .iter()
8885                                .chain(
8886                                    self.failed_commits
8887                                        .iter()
8888                                        .flat_map(|(commit, _)| &commit.ops),
8889                                )
8890                                .chain(
8891                                    self.outbox
8892                                        .iter()
8893                                        .take(index)
8894                                        .flat_map(|commit| &commit.ops),
8895                                )
8896                                .chain(operations.iter().take(op_index + 1))
8897                            {
8898                                if op.table != operation.table || op.row_id != operation.row_id {
8899                                    continue;
8900                                }
8901                                if let Some(values) = &op.values {
8902                                    intended.extend(values.clone());
8903                                } else {
8904                                    scope_image = intended.clone();
8905                                    intended.clear();
8906                                }
8907                            }
8908                            self.conn.execute("INSERT INTO _syncular_acked_rows(commit_id,idx,tbl,id,commit_seq,op_json,intent_json) VALUES (?1,?2,?3,?4,?5,?6,?7)",
8909                                rusqlite::params![client_commit_id, op_index as i64, operation.table, operation.row_id, sequence,
8910                                    serde_json::to_string(&CommitOperation::from(operation)).map_err(|_| "sync.local_corrupt".to_owned())?,
8911                                    serde_json::to_string(if operation.upsert { &intended } else { &scope_image }).map_err(|_| "sync.local_corrupt".to_owned())?])
8912                                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
8913                        }
8914                        let journal_results = results
8915                            .iter()
8916                            .map(|result| {
8917                                let op_index = match result {
8918                                    OpResult::Applied { op_index }
8919                                    | OpResult::Conflict { op_index, .. }
8920                                    | OpResult::Error { op_index, .. } => *op_index,
8921                                };
8922                                CommitOperationOutcome::Applied { op_index }
8923                            })
8924                            .collect::<Vec<_>>();
8925                        let outcome_status = if status == PushStatus::Applied {
8926                            CommitOutcomeStatus::Applied
8927                        } else {
8928                            CommitOutcomeStatus::Cached
8929                        };
8930                        let persisted = self
8931                            .persist_commit_outcome(
8932                                client_commit_id,
8933                                outcome_status,
8934                                &journal_results,
8935                                None,
8936                            )
8937                            .and_then(|()| self.delete_outbox_persisted(client_commit_id));
8938                        persisted?;
8939                        self.sync_needed = true;
8940                        batch.status = true;
8941                        batch.outcomes = true;
8942                    }
8943                    PushStatus::Rejected => {
8944                        if results.iter().any(|result| {
8945                            matches!(
8946                                result,
8947                                OpResult::Error {
8948                                    code,
8949                                    retryable: true,
8950                                    ..
8951                                } if code == "sync.idempotency_cache_miss"
8952                            )
8953                        }) {
8954                            // §6.3/§7.2: a serving failure, not an outcome — keep the
8955                            // exact commit queued for an identical retry.
8956                            report.retryable.push(client_commit_id.to_owned());
8957                            continue;
8958                        }
8959
8960                        let mut journal_results = Vec::with_capacity(results.len());
8961                        let mut conflicts = Vec::new();
8962                        let mut rejections = Vec::new();
8963                        for result in results {
8964                            match result {
8965                                OpResult::Applied { op_index } => {
8966                                    journal_results.push(CommitOperationOutcome::Applied {
8967                                        op_index: *op_index,
8968                                    });
8969                                }
8970                                OpResult::Conflict {
8971                                    op_index,
8972                                    code,
8973                                    message,
8974                                    server_version,
8975                                    server_row,
8976                                    conflict_columns: conflict_columns_bitmap,
8977                                } => {
8978                                    let operation = operations
8979                                        .get(*op_index as usize)
8980                                        .map(CommitOperation::from);
8981                                    let (table, row_id) = operation
8982                                        .as_ref()
8983                                        .map(|op| (op.table.clone(), op.row_id.clone()))
8984                                        .unwrap_or_default();
8985                                    let server_row_json = self
8986                                        .schema
8987                                        .table(&table)
8988                                        .and_then(|t| {
8989                                            decode_row_bytes(t, server_row, &self.encryption)
8990                                                .ok()
8991                                                .map(|(row, _)| (t, row))
8992                                        })
8993                                        .map(|(t, row)| {
8994                                            let mut map = Map::new();
8995                                            for (i, column) in t.columns.iter().enumerate() {
8996                                                map.insert(
8997                                                    column.name.clone(),
8998                                                    column_value_to_json(
8999                                                        row.get(i).unwrap_or(&None),
9000                                                    ),
9001                                                );
9002                                            }
9003                                            map
9004                                        })
9005                                        .unwrap_or_default();
9006                                    // §6.3: decode the presence-layout bitmap
9007                                    // into the marked column names.
9008                                    let conflict_columns = self
9009                                        .schema
9010                                        .table(&table)
9011                                        .map(|t| {
9012                                            t.columns
9013                                                .iter()
9014                                                .enumerate()
9015                                                .filter(|(index, _)| {
9016                                                    conflict_columns_bitmap
9017                                                        .get(index / 8)
9018                                                        .is_some_and(|byte| {
9019                                                            byte & (1 << (index % 8)) != 0
9020                                                        })
9021                                                })
9022                                                .map(|(_, column)| column.name.clone())
9023                                                .collect::<Vec<_>>()
9024                                        })
9025                                        .unwrap_or_default();
9026                                    let conflict = ConflictRecord {
9027                                        client_commit_id: client_commit_id.to_owned(),
9028                                        op_index: *op_index,
9029                                        table,
9030                                        row_id,
9031                                        code: code.clone(),
9032                                        message: message.clone(),
9033                                        server_version: *server_version,
9034                                        server_row: server_row_json,
9035                                        conflict_columns,
9036                                        operation,
9037                                    };
9038                                    journal_results.push(CommitOperationOutcome::Conflict {
9039                                        conflict: conflict.clone(),
9040                                    });
9041                                    conflicts.push(conflict);
9042                                }
9043                                OpResult::Error {
9044                                    op_index,
9045                                    code,
9046                                    message,
9047                                    retryable,
9048                                } => {
9049                                    let rejection = RejectionRecord {
9050                                        client_commit_id: client_commit_id.to_owned(),
9051                                        op_index: *op_index,
9052                                        code: code.clone(),
9053                                        message: message.clone(),
9054                                        retryable: *retryable,
9055                                        details: rejection_details
9056                                            .and_then(|details| details.get(op_index))
9057                                            .cloned(),
9058                                        operation: operations
9059                                            .get(*op_index as usize)
9060                                            .map(CommitOperation::from),
9061                                    };
9062                                    journal_results.push(CommitOperationOutcome::Error {
9063                                        rejection: rejection.clone(),
9064                                    });
9065                                    rejections.push(rejection);
9066                                }
9067                            }
9068                        }
9069                        if self.retain_failed_commits {
9070                            self.retain_failed_commit(client_commit_id, &operations)?;
9071                        }
9072                        let outcome_status = if conflicts.is_empty() {
9073                            CommitOutcomeStatus::Rejected
9074                        } else {
9075                            CommitOutcomeStatus::Conflict
9076                        };
9077                        let persisted = self
9078                            .persist_commit_outcome(
9079                                client_commit_id,
9080                                outcome_status,
9081                                &journal_results,
9082                                Some(&operations),
9083                            )
9084                            .and_then(|()| self.delete_outbox_persisted(client_commit_id));
9085                        persisted?;
9086                        batch.conflicts = !conflicts.is_empty();
9087                        batch.rejections = !rejections.is_empty();
9088                        batch.status = true;
9089                        batch.outcomes = true;
9090                        self.conflicts.extend(conflicts);
9091                        self.rejections.extend(rejections);
9092                    }
9093                }
9094                drained.push(self.outbox.remove(index));
9095                if batch.status {
9096                    for operation in &operations {
9097                        if !self.record_row_scopes(
9098                            &mut batch,
9099                            &operation.table,
9100                            &operation.row_id,
9101                            false,
9102                        ) {
9103                            batch.table(&operation.table);
9104                        }
9105                    }
9106                }
9107                match status {
9108                    PushStatus::Applied | PushStatus::Cached => {
9109                        report.applied.push(client_commit_id.to_owned())
9110                    }
9111                    PushStatus::Rejected => report.rejected.push(client_commit_id.to_owned()),
9112                }
9113            }
9114            if batch.outcomes && frames.iter().any(|frame| matches!(frame, Frame::PushResult { client_commit_id, .. } if Some(client_commit_id.as_str()) == last_final_push_result_id)) {
9115                self.prune_commit_outcomes()?;
9116            }
9117            self.reconcile_rows(&drained)?;
9118            self.rebuild_overlay_if_dirty()?;
9119            self.finish_observation("syncular_push_results", batch)
9120        })();
9121        if let Err(error) = persisted {
9122            self.rollback_observation("syncular_push_results");
9123            self.restore(rollback);
9124            report.applied.truncate(applied_count);
9125            report.rejected.truncate(rejected_count);
9126            report.retryable.truncate(retryable_count);
9127            return Err(error);
9128        }
9129        if !rollback.sync_needed && self.sync_needed {
9130            self.sync_intent_queue.push_back(SyncIntent::Interactive);
9131        }
9132        report.conflicts += (self.conflicts.len() - conflict_count) as u32;
9133        Ok(())
9134    }
9135
9136    fn push_result_is_final(status: PushStatus, results: &[OpResult]) -> bool {
9137        status != PushStatus::Rejected
9138            || !results.iter().any(|result| {
9139                matches!(
9140                    result,
9141                    OpResult::Error {
9142                        code,
9143                        retryable: true,
9144                        ..
9145                    } if code == "sync.idempotency_cache_miss"
9146                )
9147            })
9148    }
9149
9150    // -- unit sections and the live section ------------------------------------------
9151
9152    /// One unit section (§4.4): apply its segments, then persist its
9153    /// `SUB_END` (§4.6 reset, §3.3 revocation and purge, the echo, cursor and
9154    /// resume token). A section of a unit the app released while the round was
9155    /// in flight is skipped; a fail-closed first page fails the unit and skips
9156    /// the rest of its section (§5.6).
9157    fn process_section(
9158        &mut self,
9159        transport: &mut dyn Transport,
9160        start: &SectionStart,
9161        body: Vec<Frame>,
9162        sub_end: Option<(i64, Option<String>)>,
9163        meta: &RequestMeta,
9164        report: &mut SyncReport,
9165    ) -> Result<(), SectionError> {
9166        let storage = |message| SectionError::Abort("storage.failed".to_owned(), message);
9167        let unit = self
9168            .stored(get_unit(&self.conn, &start.id))
9169            .map_err(storage)?
9170            .filter(|unit| unit.status == UnitStatus::Active);
9171        // §7.7: a section boundary ends the previous payload.
9172        self.progress.update(|p| {
9173            p.phase = ProgressPhase::Import;
9174            p.unit_id = unit.as_ref().map(|unit| unit.id.clone());
9175            p.table = None;
9176            p.segment_id = None;
9177            p.bytes_received = 0;
9178            p.bytes_total = None;
9179            p.rows_processed = 0;
9180            p.rows_total = None;
9181            p.import_write_ms = None;
9182            p.import_commit_ms = None;
9183        });
9184        let Some(unit) = unit else {
9185            return Ok(());
9186        };
9187        let echo: Vec<EchoRecord> = start
9188            .fenced
9189            .iter()
9190            .map(|record| EchoRecord {
9191                table: record.table.clone(),
9192                status: record.status.name().to_owned(),
9193                scopes: record.scopes.iter().cloned().collect(),
9194            })
9195            .collect();
9196        // §5.6: a resumed bootstrap keeps the tables it already seeded.
9197        let mut seeded: BTreeSet<String> = if unit.bootstrap_state.is_none() {
9198            BTreeSet::new()
9199        } else {
9200            unit.seeded.iter().cloned().collect()
9201        };
9202        let applied = (|| {
9203            for frame in body {
9204                match frame {
9205                    Frame::SegmentInline { payload } => {
9206                        let segment = decode_rows_segment(&payload).map_err(|e| {
9207                            SectionError::Abort(e.code.as_str().to_owned(), e.detail)
9208                        })?;
9209                        self.progress.update(|p| {
9210                            p.phase = ProgressPhase::Import;
9211                            p.unit_id = Some(unit.id.clone());
9212                            p.table = Some(segment.table.clone());
9213                            p.segment_id = None;
9214                            p.bytes_received = payload.len() as u64;
9215                            p.bytes_total = Some(payload.len() as u64);
9216                            p.rows_processed = 0;
9217                            p.rows_total =
9218                                Some(segment.blocks.iter().map(|block| block.len() as u64).sum());
9219                            p.import_write_ms = None;
9220                            p.import_commit_ms = None;
9221                        });
9222                        let (pair, clear, effective) =
9223                            self.segment_pair(&unit, &echo, &seeded, &segment.table)?;
9224                        report.segment_rows_applied +=
9225                            self.apply_segment(&pair, &segment, clear, &effective, &unit.id)?;
9226                        seeded.insert(segment.table);
9227                    }
9228                    Frame::SegmentRef {
9229                        segment_id,
9230                        byte_length,
9231                        media_type,
9232                        table,
9233                        row_count,
9234                        as_of_commit_seq,
9235                        scope_digest,
9236                        row_cursor,
9237                        next_row_cursor,
9238                        url,
9239                        url_expires_at_ms,
9240                        ..
9241                    } => {
9242                        // §4.2: reject a descriptor whose mediaType was not
9243                        // advertised — never skip or guess.
9244                        let advertised = match media_type {
9245                            MediaType::Rows => {
9246                                meta.accept & ACCEPT_EXTERNAL_ROWS != 0
9247                                    || meta.accept & ACCEPT_INLINE_ROWS != 0
9248                            }
9249                            MediaType::Sqlite => meta.accept & ACCEPT_SQLITE != 0,
9250                        };
9251                        if !advertised {
9252                            return Err(SectionError::Abort(
9253                                "sync.invalid_request".to_owned(),
9254                                format!(
9255                                    "SEGMENT_REF mediaType {} was not advertised in accept (§4.2)",
9256                                    media_type.name()
9257                                ),
9258                            ));
9259                        }
9260                        self.progress.update(|p| {
9261                            p.phase = ProgressPhase::Download;
9262                            p.unit_id = Some(unit.id.clone());
9263                            p.table = Some(table.clone());
9264                            p.segment_id = Some(segment_id.clone());
9265                            p.bytes_received = 0;
9266                            p.bytes_total = u64::try_from(byte_length).ok();
9267                            p.rows_processed = 0;
9268                            p.rows_total = u64::try_from(row_count).ok();
9269                            p.import_write_ms = None;
9270                            p.import_commit_ms = None;
9271                        });
9272                        let observer = self.progress.clone();
9273                        let mut on_progress = |bytes| observer.update(|p| p.bytes_received = bytes);
9274                        let bytes = if let Some(url) = url {
9275                            // §5.4: a url-carrying descriptor MUST be fetched
9276                            // from that URL; failure invalidates the whole
9277                            // descriptor (no fall-through to §5.5 — re-pull
9278                            // recovers, §1.4 rule 5).
9279                            if meta.accept & ACCEPT_SIGNED_URLS == 0 {
9280                                return Err(SectionError::Abort(
9281                                    "sync.invalid_request".to_owned(),
9282                                    "SEGMENT_REF carries a url but accept bit 3 was not advertised (§5.4)"
9283                                        .to_owned(),
9284                                ));
9285                            }
9286                            // §5.4: MUST NOT start a fetch at/past expiry.
9287                            if url_expires_at_ms.is_some_and(|exp| {
9288                                exp <= transport
9289                                    .segment_fetch_started_at(&url)
9290                                    .unwrap_or_else(|| self.clock_now_ms())
9291                            }) {
9292                                return Err(SectionError::Abort(
9293                                    "sync.segment_expired".to_owned(),
9294                                    format!(
9295                                        "signed URL for segment {segment_id} expired before fetch — re-pull mints fresh descriptors (§5.4)"
9296                                    ),
9297                                ));
9298                            }
9299                            transport
9300                                .fetch_url(
9301                                    &url,
9302                                    u64::try_from(byte_length).unwrap_or(0),
9303                                    &mut on_progress,
9304                                )
9305                                .map_err(|e| self.segment_fetch_failed(e))?
9306                        } else {
9307                            // §5.5: the owning unit's scope map, each value a one-element list.
9308                            let requested_scopes_json =
9309                                canonical_scope_json(&unit_scope_lists(&unit.scopes));
9310                            transport
9311                                .download_segment(
9312                                    &SegmentRequest {
9313                                        segment_id: segment_id.clone(),
9314                                        table: table.clone(),
9315                                        requested_scopes_json,
9316                                        byte_length: u64::try_from(byte_length).unwrap_or(0),
9317                                    },
9318                                    &mut on_progress,
9319                                )
9320                                .map_err(|e| {
9321                                    // §5.5: access changed since the response;
9322                                    // the next round's SUB_START settles the unit.
9323                                    if e.code == "sync.forbidden" {
9324                                        SectionError::Refused
9325                                    } else {
9326                                        self.segment_fetch_failed(e)
9327                                    }
9328                                })?
9329                        };
9330                        // §5.4: the descriptor's byteLength caps the body before
9331                        // §5.1 verifies the content address; the client discards
9332                        // the segment and re-pulls on either failure.
9333                        if bytes.len() as i64 > byte_length {
9334                            self.schedule_background_retry();
9335                            return Err(SectionError::Abort(
9336                                "sync.invalid_request".to_owned(),
9337                                "segment exceeds its byteLength".to_owned(),
9338                            ));
9339                        }
9340                        on_progress(bytes.len() as u64);
9341                        let digest = Sha256::digest(&bytes);
9342                        let expected = segment_id
9343                            .strip_prefix("sha256:")
9344                            .unwrap_or(segment_id.as_str());
9345                        if bytes_to_hex(&digest) != expected {
9346                            // §5.1: the client discards the segment and re-pulls.
9347                            self.schedule_background_retry();
9348                            return Err(SectionError::Abort(
9349                                "sync.invalid_request".to_owned(),
9350                                "segment bytes do not match the content address (§5.1)".to_owned(),
9351                            ));
9352                        }
9353                        self.progress.update(|p| p.phase = ProgressPhase::Import);
9354                        if media_type == MediaType::Sqlite {
9355                            // §5.3: images are whole-table — a paged sqlite
9356                            // descriptor is invalid.
9357                            if row_cursor.is_some() || next_row_cursor.is_some() {
9358                                return Err(SectionError::Abort(
9359                                    "sync.invalid_request".to_owned(),
9360                                    "sqlite segments are whole-table: rowCursor/nextRowCursor must be absent (§5.3)"
9361                                        .to_owned(),
9362                                ));
9363                            }
9364                            let (pair, clear, effective) =
9365                                self.segment_pair(&unit, &echo, &seeded, &table)?;
9366                            report.segment_rows_applied += self.apply_sqlite_segment(
9367                                &pair,
9368                                &bytes,
9369                                clear,
9370                                &effective,
9371                                &unit.id,
9372                                row_count,
9373                                as_of_commit_seq,
9374                                &scope_digest,
9375                            )?;
9376                            seeded.insert(table);
9377                        } else {
9378                            let segment = decode_rows_segment(&bytes).map_err(|e| {
9379                                SectionError::Abort(e.code.as_str().to_owned(), e.detail)
9380                            })?;
9381                            let (pair, clear, effective) =
9382                                self.segment_pair(&unit, &echo, &seeded, &segment.table)?;
9383                            report.segment_rows_applied +=
9384                                self.apply_segment(&pair, &segment, clear, &effective, &unit.id)?;
9385                            seeded.insert(segment.table);
9386                        }
9387                    }
9388                    _ => {
9389                        return Err(SectionError::Abort(
9390                            "sync.invalid_request".to_owned(),
9391                            "unexpected frame inside a unit section".to_owned(),
9392                        ));
9393                    }
9394                }
9395            }
9396            Ok(())
9397        })();
9398        match applied {
9399            Ok(()) => {}
9400            // The unit keeps catching up with its cursor unpersisted.
9401            Err(SectionError::Refused) => return Ok(()),
9402            Err(SectionError::FailClosed) => {
9403                // §5.6: the failure is unit-local; the rest of the response
9404                // still applies and the failed unit leaves the list.
9405                self.begin_observation("syncular_section_failure")
9406                    .map_err(storage)?;
9407                let failed = (|| {
9408                    let mut batch = ChangeAccumulator::default();
9409                    for (base_key, value) in self.stored(window_refs(&self.conn, &unit.id))? {
9410                        batch.window(&base_key, &value);
9411                    }
9412                    batch.status = true;
9413                    self.stored(save_unit(
9414                        &self.conn,
9415                        &UnitRecord {
9416                            status: UnitStatus::Failed,
9417                            reason_code: Some("sync.scope_revoked".to_owned()),
9418                            ..unit.clone()
9419                        },
9420                    ))?;
9421                    self.finish_observation("syncular_section_failure", batch)
9422                })();
9423                if let Err(message) = failed {
9424                    self.rollback_observation("syncular_section_failure");
9425                    return Err(storage(message));
9426                }
9427                report.failed.push(unit.id);
9428                return Ok(());
9429            }
9430            Err(error) => return Err(error),
9431        }
9432        let Some((next_cursor, bootstrap_state)) = sub_end else {
9433            return Ok(());
9434        };
9435        let rollback = self.capture();
9436        let counts = (
9437            report.resets.len(),
9438            report.revoked.len(),
9439            report.failed.len(),
9440        );
9441        self.begin_observation("syncular_sub_end")
9442            .map_err(storage)?;
9443        let finished = self
9444            .finish_section(start, echo, next_cursor, bootstrap_state, seeded, report)
9445            .and_then(|batch| {
9446                self.rebuild_overlay_if_dirty()?;
9447                self.finish_observation("syncular_sub_end", batch)
9448            });
9449        if let Err(message) = finished {
9450            self.rollback_observation("syncular_sub_end");
9451            self.restore(rollback);
9452            report.resets.truncate(counts.0);
9453            report.revoked.truncate(counts.1);
9454            report.failed.truncate(counts.2);
9455            return Err(storage(message));
9456        }
9457        Ok(())
9458    }
9459
9460    /// §5.6: the table, first-page flag and effective scopes of one segment
9461    /// of `unit`'s section.
9462    #[allow(clippy::type_complexity)]
9463    fn segment_pair(
9464        &self,
9465        unit: &UnitRecord,
9466        echo: &[EchoRecord],
9467        seeded: &BTreeSet<String>,
9468        name: &str,
9469    ) -> Result<(TableSchema, bool, Vec<(String, Vec<String>)>), SectionError> {
9470        let table = self.schema.table(name).cloned().ok_or_else(|| {
9471            SectionError::Abort(
9472                "sync.schema_mismatch".to_owned(),
9473                format!("segment table {name:?} is not in the client schema"),
9474            )
9475        })?;
9476        let effective = covers(&table, &unit.scopes)
9477            .then(|| pair_scopes(&unit.scopes, name, Some(echo)))
9478            .flatten()
9479            .ok_or_else(|| {
9480                SectionError::Abort(
9481                    "sync.invalid_request".to_owned(),
9482                    "a segment names no active pair of its unit".to_owned(),
9483                )
9484            })?;
9485        Ok((table, !seeded.contains(name), effective))
9486    }
9487
9488    /// §4.4 `SUB_END` of an active, revoked or reset section, inside the
9489    /// caller's observation: a reset discards cursor, resume token and
9490    /// first-page record and keeps the echo; a revoked pair purges by its last
9491    /// echo (§3.3); otherwise the echo replaces the persisted one, a revoked
9492    /// fenced record that turns active again re-bootstraps the unit, and a
9493    /// bootstrap section persists its pin, resume token and seeded tables.
9494    fn finish_section(
9495        &mut self,
9496        start: &SectionStart,
9497        echo: Vec<EchoRecord>,
9498        next_cursor: i64,
9499        bootstrap_state: Option<String>,
9500        seeded: BTreeSet<String>,
9501        report: &mut SyncReport,
9502    ) -> Result<ChangeAccumulator, String> {
9503        #[cfg(feature = "bench-internals")]
9504        let _phase = self.benchmark_phases.start(Phase::CursorPersist);
9505        let mut batch = ChangeAccumulator::default();
9506        // §4.4: SUB_END writes only into a unit that is still registered.
9507        let Some(unit) = self
9508            .stored(get_unit(&self.conn, &start.id))?
9509            .filter(|unit| unit.status == UnitStatus::Active)
9510        else {
9511            return Ok(batch);
9512        };
9513        for (base_key, value) in self.stored(window_refs(&self.conn, &unit.id))? {
9514            batch.window(&base_key, &value);
9515        }
9516        if start.status == SubStatus::Reset {
9517            // §4.6: staleness, never a purge; the rows stay until the
9518            // bootstrap replaces them.
9519            report.resets.push(unit.id.clone());
9520            self.stored(save_unit(
9521                &self.conn,
9522                &UnitRecord {
9523                    reason_code: Some(start.reason_code.clone()),
9524                    cursor: Some(-1),
9525                    bootstrap_state: None,
9526                    seeded: Vec::new(),
9527                    ..unit
9528                },
9529            ))?;
9530            return Ok(batch);
9531        }
9532        let revoked = start.status == SubStatus::Revoked;
9533        let mut units = None;
9534        let mut failed = false;
9535        let mut rearmed = false;
9536        for table in self.schema.tables.clone() {
9537            if !covers(&table, &unit.scopes) {
9538                continue;
9539            }
9540            let last = unit
9541                .echo
9542                .as_deref()
9543                .and_then(|last| pair_scopes(&unit.scopes, &table.name, Some(last)));
9544            let next = (!revoked)
9545                .then(|| pair_scopes(&unit.scopes, &table.name, Some(&echo)))
9546                .flatten();
9547            // §3.3: a revoked fenced record that turns active again re-bootstraps.
9548            rearmed |= unit.echo.is_some() && last.is_none() && next.is_some();
9549            // §3.3: purge a pair the response revokes, keyed on the echo the
9550            // client persisted while the pair was active.
9551            if let (Some(last), None) = (&last, &next) {
9552                let units = match units {
9553                    Some(ref units) => units,
9554                    None => units.insert(self.stored(load_units(&self.conn))?),
9555                };
9556                let owners = unit_owners(units, &table, &unit.id);
9557                failed |= !self.purge_pair(&mut batch, &table, last, &owners)?;
9558            }
9559        }
9560        if revoked || failed {
9561            if revoked {
9562                report.revoked.push(unit.id.clone());
9563            }
9564            if failed {
9565                report.failed.push(unit.id.clone());
9566            }
9567            batch.status = true;
9568            self.stored(save_unit(
9569                &self.conn,
9570                &UnitRecord {
9571                    status: if failed {
9572                        UnitStatus::Failed
9573                    } else {
9574                        UnitStatus::Revoked
9575                    },
9576                    reason_code: Some(if failed {
9577                        "sync.scope_revoked".to_owned()
9578                    } else {
9579                        start.reason_code.clone()
9580                    }),
9581                    cursor: Some(-1),
9582                    bootstrap_state: None,
9583                    seeded: Vec::new(),
9584                    echo: None,
9585                    ..unit
9586                },
9587            ))?;
9588            return Ok(batch);
9589        }
9590        let completed = start.bootstrap && !rearmed && bootstrap_state.is_none();
9591        let (cursor, bootstrap_state, seeded) = if rearmed {
9592            (Some(-1), None, Vec::new())
9593        } else if start.bootstrap {
9594            let seeded = if bootstrap_state.is_none() {
9595                Vec::new()
9596            } else {
9597                seeded.into_iter().collect()
9598            };
9599            (Some(next_cursor), bootstrap_state, seeded)
9600        } else {
9601            (
9602                unit.cursor,
9603                unit.bootstrap_state.clone(),
9604                unit.seeded.clone(),
9605            )
9606        };
9607        self.stored(save_unit(
9608            &self.conn,
9609            &UnitRecord {
9610                reason_code: None,
9611                echo: Some(echo.clone()),
9612                cursor,
9613                bootstrap_state,
9614                seeded,
9615                ..unit.clone()
9616            },
9617        ))?;
9618        if !completed {
9619            return Ok(batch);
9620        }
9621        // §7.1: a completed bootstrap covers acknowledged intent of its pairs
9622        // at or below its pin.
9623        for table in self.schema.tables.clone() {
9624            let Some(effective) = covers(&table, &unit.scopes)
9625                .then(|| pair_scopes(&unit.scopes, &table.name, Some(&echo)))
9626                .flatten()
9627            else {
9628                continue;
9629            };
9630            let mut statement = self
9631                .conn
9632                .prepare("SELECT id,intent_json FROM _syncular_acked_rows WHERE tbl=?1 AND commit_seq<=?2")
9633                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9634            let rows = statement
9635                .query_map(rusqlite::params![table.name, next_cursor], |row| {
9636                    Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
9637                })
9638                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9639            let mut covered = HashSet::new();
9640            for row in rows {
9641                let (id, intent) =
9642                    row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9643                let values: Map<String, Value> =
9644                    serde_json::from_str(&intent).map_err(|_| "sync.local_corrupt".to_owned())?;
9645                if owns_local_row(&table, &effective, &[], &values) {
9646                    covered.insert(id);
9647                }
9648            }
9649            drop(statement);
9650            for id in covered {
9651                self.conn
9652                    .execute(
9653                        "DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2 AND commit_seq<=?3",
9654                        rusqlite::params![table.name, id, next_cursor],
9655                    )
9656                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9657                self.overlay_dirty.table(&table.name);
9658                batch.table(&table.name);
9659            }
9660        }
9661        Ok(batch)
9662    }
9663
9664    /// §3.3: purge one revoked pair inside the caller's observation: delete
9665    /// the rows matching its last effective scopes that no owner holds, drop
9666    /// the retained intent and every pending commit that provably writes into
9667    /// them, and delete now-unauthorized blob bodies. `Ok(false)` when the
9668    /// table has no local scope-column mapping (fail closed: nothing deleted).
9669    fn purge_pair(
9670        &mut self,
9671        batch: &mut ChangeAccumulator,
9672        table: &TableSchema,
9673        last: &[(String, Vec<String>)],
9674        owners: &[Vec<(String, Vec<String>)>],
9675    ) -> Result<bool, String> {
9676        if local_scope_predicate(table, last, owners).is_err() {
9677            return Ok(false);
9678        }
9679        for commit in self.drop_failed_commits(|name, values| {
9680            name == table.name && owns_local_row(table, last, owners, values)
9681        })? {
9682            batch.conflicts = true;
9683            batch.rejections = true;
9684            batch.outcomes = true;
9685            for operation in commit.ops {
9686                batch.table(&operation.table);
9687            }
9688        }
9689        if !self.purge_scope_rows(&table.name, last, owners)? {
9690            return Err("storage.failed: the revoked scope purge failed".to_owned());
9691        }
9692        self.record_scope_map(batch, &table.name, last);
9693        if self.drop_doomed_outbox(table, last, owners)? {
9694            batch.status = true;
9695            batch.rejections = true;
9696            batch.outcomes = true;
9697        }
9698        // §5.9.7 B2: revocation deletes now-unauthorized blob bodies.
9699        self.delete_unreferenced_cached_blobs()?;
9700        Ok(true)
9701    }
9702
9703    /// §4.5 client application: apply one live-section or delta `COMMIT`
9704    /// with only the changes that match the persisted echo of a receiving
9705    /// unit; every applied change then goes through the version guard.
9706    /// Returns whether any change of the frame applied.
9707    fn apply_live_commit(
9708        &mut self,
9709        receivers: &[UnitRecord],
9710        tables: &[String],
9711        changes: Vec<ssp2::model::Change>,
9712        commit_seq: i64,
9713        source: AppliedServerCommitSource,
9714    ) -> Result<bool, TransportError> {
9715        let changes: Vec<ssp2::model::Change> = changes
9716            .into_iter()
9717            .filter(|change| {
9718                // An unknown table fails loud in the apply below.
9719                let Some(table) = tables
9720                    .get(change.table_index as usize)
9721                    .and_then(|name| self.schema.table(name))
9722                else {
9723                    return true;
9724                };
9725                receivers.iter().any(|unit| {
9726                    covers(table, &unit.scopes)
9727                        && pair_scopes(&unit.scopes, &table.name, unit.echo.as_deref()).is_some_and(
9728                            |scopes| {
9729                                scopes.iter().all(|(variable, values)| {
9730                                    change
9731                                        .scopes
9732                                        .iter()
9733                                        .find(|(key, _)| key == variable)
9734                                        .is_some_and(|(_, value)| values.contains(value))
9735                                })
9736                            },
9737                        )
9738                })
9739            })
9740            .collect();
9741        if changes.is_empty() {
9742            return Ok(false);
9743        }
9744        let affected_tables = changes
9745            .iter()
9746            .filter_map(|change| tables.get(change.table_index as usize))
9747            .cloned()
9748            .collect::<BTreeSet<_>>();
9749        let previous_batches = self.change_queue.len();
9750        self.apply_commit_frame(tables, &changes, commit_seq)?;
9751        if self.change_queue.len() > previous_batches {
9752            if let Some(batch) = self.change_queue.back_mut() {
9753                batch
9754                    .applied_commits
9755                    .extend(
9756                        affected_tables
9757                            .into_iter()
9758                            .map(|table| AppliedServerCommit {
9759                                table,
9760                                commit_seq,
9761                                source,
9762                            }),
9763                    );
9764            }
9765        }
9766        Ok(true)
9767    }
9768
9769    // Both pull and realtime frames use the same durable observation boundary.
9770    fn apply_commit_frame(
9771        &mut self,
9772        tables: &[String],
9773        changes: &[ssp2::model::Change],
9774        sequence: i64,
9775    ) -> Result<(), TransportError> {
9776        #[cfg(feature = "bench-internals")]
9777        let _phase = self.benchmark_phases.start(Phase::CommitApply);
9778        let rollback = self.capture();
9779        self.coalesce_unit(changes.len())
9780            .and_then(|()| self.begin_observation("syncular_commit"))
9781            .map_err(|message| TransportError::new("storage.failed", message))?;
9782        let applied = (|| {
9783            let mut batch = ChangeAccumulator::default();
9784            self.record_commit_changes(&mut batch, tables, changes);
9785            // §7.1 chained base: a change names the version an acknowledged own
9786            // commit wrote; pending upserts chained on it move to that version.
9787            for change in changes {
9788                let Some(table) = tables.get(change.table_index as usize) else {
9789                    continue;
9790                };
9791                let acknowledged = self
9792                    .conn
9793                    .prepare_cached("SELECT op_json FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2 AND commit_seq=?3")
9794                    .and_then(|mut statement| {
9795                        statement
9796                            .query_map(rusqlite::params![table, change.row_id, sequence], |row| row.get::<_, String>(0))?
9797                            .collect::<Result<Vec<_>, _>>()
9798                    })
9799                    .map_err(|error| TransportError::new("storage.failed", Self::sqlite_failure(&self.storage_failure, error)))?
9800                    .iter()
9801                    .map(|op| serde_json::from_str::<CommitOperation>(op).map(OutboxOp::from))
9802                    .collect::<Result<Vec<_>, _>>()
9803                    .map_err(|_| TransportError::new("storage.failed", "sync.local_corrupt: invalid acknowledged operation".to_owned()))?;
9804                for index in 0..self.outbox.len() {
9805                    if !self.outbox[index]
9806                        .ops
9807                        .iter()
9808                        .any(|op| acknowledged.iter().any(|earlier| chains_on(earlier, op)))
9809                    {
9810                        continue;
9811                    }
9812                    let commit = Arc::make_mut(&mut self.outbox[index]);
9813                    for op in &mut commit.ops {
9814                        if acknowledged.iter().any(|earlier| chains_on(earlier, op)) {
9815                            op.base_version = Some(change.row_version);
9816                        }
9817                    }
9818                    let ops = serde_json::to_string(
9819                        &commit
9820                            .ops
9821                            .iter()
9822                            .map(CommitOperation::from)
9823                            .collect::<Vec<_>>(),
9824                    )
9825                    .map_err(|_| {
9826                        TransportError::new(
9827                            "storage.failed",
9828                            "sync.local_corrupt: unencodable outbox".to_owned(),
9829                        )
9830                    })?;
9831                    let id = commit.client_commit_id.clone();
9832                    self.conn
9833                        .execute(
9834                            "UPDATE _syncular_outbox SET ops_json=?1 WHERE commit_id=?2",
9835                            rusqlite::params![ops, id],
9836                        )
9837                        .map_err(|error| {
9838                            TransportError::new(
9839                                "storage.failed",
9840                                Self::sqlite_failure(&self.storage_failure, error),
9841                            )
9842                        })?;
9843                }
9844            }
9845            {
9846                let storage_failed = |error| {
9847                    TransportError::new(
9848                        "storage.failed",
9849                        Self::sqlite_failure(&self.storage_failure, error),
9850                    )
9851                };
9852                let pinned: HashSet<(&str, &str)> = self
9853                    .outbox
9854                    .iter()
9855                    .flat_map(|commit| &commit.ops)
9856                    .map(|op| (op.table.as_str(), op.row_id.as_str()))
9857                    .collect();
9858                for change in changes {
9859                    let table = tables.get(change.table_index as usize).ok_or_else(|| {
9860                        TransportError::new(
9861                            "sync.invalid_request",
9862                            "change tableIndex out of range".to_owned(),
9863                        )
9864                    })?;
9865                    if pinned.contains(&(table.as_str(), change.row_id.as_str())) {
9866                        self.conn.prepare_cached("INSERT INTO _syncular_row_deliveries(tbl,id,commit_seq) VALUES (?1,?2,?3) ON CONFLICT(tbl,id) DO UPDATE SET commit_seq=MAX(commit_seq,excluded.commit_seq)")
9867                            .and_then(|mut statement| statement.execute(rusqlite::params![table, change.row_id, sequence]))
9868                            .map_err(storage_failed)?;
9869                    }
9870                    self.conn.prepare_cached("DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2 AND commit_seq<=?3")
9871                        .and_then(|mut statement| statement.execute(rusqlite::params![table, change.row_id, sequence]))
9872                        .map_err(storage_failed)?;
9873                }
9874            }
9875            self.apply_commit_changes(tables, changes)?;
9876            self.rebuild_overlay_if_dirty()
9877                .map_err(|message| TransportError::new("storage.failed", message))?;
9878            self.finish_observation("syncular_commit", batch)
9879                .map_err(|message| TransportError::new("storage.failed", message))
9880        })();
9881        if applied.is_err() {
9882            self.rollback_observation("syncular_commit");
9883            self.restore(rollback);
9884        }
9885        applied
9886    }
9887
9888    fn apply_commit_changes(
9889        &mut self,
9890        tables: &[String],
9891        changes: &[ssp2::model::Change],
9892    ) -> Result<(), TransportError> {
9893        // A clean overlay can reconcile changed primary keys independently
9894        // when their tables have no secondary unique constraints. Such a
9895        // constraint can change whether another row's write succeeds, so a
9896        // table with one and any local intent takes the table rebuild.
9897        let changed_tables: BTreeSet<&str> = changes
9898            .iter()
9899            .filter_map(|change| tables.get(change.table_index as usize))
9900            .map(String::as_str)
9901            .collect();
9902        let mut mirror_visible = !self.overlay_dirty.get();
9903        for name in &changed_tables {
9904            if mirror_visible
9905                && self
9906                    .schema
9907                    .table(name)
9908                    .is_some_and(|table| table.indexes.iter().any(|index| index.unique))
9909            {
9910                let (pending, protected) = self
9911                    .table_intent(name)
9912                    .map_err(|message| TransportError::new("storage.failed", message))?;
9913                mirror_visible = !pending && !protected;
9914            }
9915        }
9916        for change in changes {
9917            let table_name = tables.get(change.table_index as usize).ok_or_else(|| {
9918                TransportError::new(
9919                    "sync.invalid_request",
9920                    "change tableIndex out of range".to_owned(),
9921                )
9922            })?;
9923            let table = self.schema.table(table_name).ok_or_else(|| {
9924                TransportError::new(
9925                    "sync.schema_mismatch",
9926                    format!("change targets unknown table {table_name:?}"),
9927                )
9928            })?;
9929            let landed = match change.op {
9930                Op::Upsert => {
9931                    let payload = change.row.as_ref().ok_or_else(|| {
9932                        TransportError::new(
9933                            "sync.invalid_request",
9934                            "upsert change without row payload".to_owned(),
9935                        )
9936                    })?;
9937                    // §5.11: decrypt encrypted columns on apply, or seal the row.
9938                    let (row, sealed) = {
9939                        #[cfg(feature = "bench-internals")]
9940                        let _phase = self.benchmark_phases.start(Phase::RowDecode);
9941                        // §5.11: a decrypt failure keeps its own identity.
9942                        decode_row_bytes(table, payload, &self.encryption)?
9943                    };
9944                    let table_name = table.name.clone();
9945                    // §4.5 version guard: only a newer upsert lands.
9946                    let landed = self
9947                        .write_base_row(&table_name, &row, change.row_version, sealed.as_ref())
9948                        .map_err(|m| TransportError::new("sync.invalid_request", m))?;
9949                    if mirror_visible && landed {
9950                        self.write_row(
9951                            &visible_table(&table_name),
9952                            &table_name,
9953                            &row,
9954                            change.row_version,
9955                            sealed.as_ref(),
9956                        )
9957                        .map_err(|m| TransportError::new("sync.invalid_request", m))?;
9958                    }
9959                    landed
9960                }
9961                // §4.5 version guard: a delete at or below the local version
9962                // is skipped, a delete of an absent row is a no-op.
9963                Op::Delete => self
9964                    .delete_base_row(table_name, &change.row_id, change.row_version)
9965                    .map_err(|m| TransportError::new("sync.invalid_request", m))?,
9966            };
9967            // The change dropped the row's acknowledged intent either way
9968            // (§7.1), so the overlay of a skipped change rebuilds from the
9969            // unchanged base.
9970            self.overlay_dirty.table(table_name);
9971            if mirror_visible && !(landed && change.op == Op::Upsert) {
9972                let predicate = row_id_predicate(table);
9973                let visible = visible_table(table_name);
9974                let mut statements = vec![format!("DELETE FROM {visible} WHERE {predicate}")];
9975                if !landed {
9976                    statements.push(format!(
9977                        "INSERT INTO {visible} SELECT * FROM {} WHERE {predicate}",
9978                        base_table(table_name)
9979                    ));
9980                }
9981                for statement in &statements {
9982                    self.conn
9983                        .prepare_cached(statement)
9984                        .and_then(|mut statement| {
9985                            statement.execute(rusqlite::params![change.row_id])
9986                        })
9987                        .map_err(|error| {
9988                            TransportError::new("sync.invalid_request", error.to_string())
9989                        })?;
9990                }
9991            }
9992        }
9993        if mirror_visible {
9994            let changed_rows: HashSet<(&str, &str)> = changes
9995                .iter()
9996                .map(|change| {
9997                    (
9998                        tables[change.table_index as usize].as_str(),
9999                        change.row_id.as_str(),
10000                    )
10001                })
10002                .collect();
10003            self.replay_overlay(&changed_tables, Some(&changed_rows))
10004                .map_err(|message| TransportError::new("storage.failed", message))?;
10005            self.overlay_dirty.set(false);
10006        }
10007        Ok(())
10008    }
10009
10010    /// §5.6 segment application: validate against the generated schema,
10011    /// clear the pair's scope on the table's first page (fail closed
10012    /// without a mapping), then upsert each row with its segment-carried
10013    /// server version under the §4.5 version guard (§5.2).
10014    fn apply_segment(
10015        &mut self,
10016        table: &TableSchema,
10017        segment: &RowsSegment,
10018        first_page: bool,
10019        effective: &[(String, Vec<String>)],
10020        unit_id: &str,
10021    ) -> Result<u32, SectionError> {
10022        // §5.2: the column table validates against the generated schema —
10023        // order, names, types, nullability; mismatch is fatal. §5.11: the
10024        // server sends the WIRE types (bytes for an encrypted column), so
10025        // validate against wire_columns.
10026        let column = if segment.columns.len() != table.wire_columns.len() {
10027            Some(Value::from("count"))
10028        } else {
10029            segment
10030                .columns
10031                .iter()
10032                .zip(table.wire_columns.iter())
10033                .position(|(a, b)| a.name != b.name || a.ty != b.ty || a.nullable != b.nullable)
10034                .map(Value::from)
10035        };
10036        if segment.table != table.name
10037            || segment.schema_version != self.schema.version
10038            || column.is_some()
10039        {
10040            let mut details = Map::new();
10041            details.insert("table".to_owned(), Value::from(table.name.as_str()));
10042            if let Some(column) = column {
10043                details.insert("column".to_owned(), column);
10044            }
10045            return Err(SectionError::Transport(TransportError {
10046                code: "sync.schema_mismatch".to_owned(),
10047                message: "rows segment columns do not match the generated schema (§5.2)".to_owned(),
10048                details: Some(Value::Object(details)),
10049            }));
10050        }
10051        let mut applied = 0u32;
10052        let empty = Vec::new();
10053        let blocks = if segment.blocks.is_empty() {
10054            std::slice::from_ref(&empty)
10055        } else {
10056            &segment.blocks
10057        };
10058        for (index, block) in blocks.iter().enumerate() {
10059            let clear = first_page && index == 0;
10060            if !clear && block.is_empty() {
10061                continue;
10062            }
10063            let rollback = self.capture();
10064            let (pending, protected) = self
10065                .table_intent(&table.name)
10066                .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10067            let reconcile_pending =
10068                !protected && pending && table.indexes.iter().any(|index| index.unique);
10069            let mirror_visible = !protected && !reconcile_pending;
10070            self.coalesce_unit(block.len())
10071                .and_then(|()| self.begin_observation("syncular_segment_block"))
10072                .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10073            let outcome = (|| {
10074                self.rebuild_overlay_if_dirty()
10075                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10076                let mut changed_keys = Vec::new();
10077                let mut batch = ChangeAccumulator::default();
10078                if clear && self.clear_scope(table, effective, unit_id)? || !block.is_empty() {
10079                    batch.table(&table.name);
10080                }
10081                for (row_index, row) in block.iter().enumerate() {
10082                    let decrypted;
10083                    let mut sealed = None;
10084                    let values = if table.has_encrypted_columns() {
10085                        let mut values = row.values.clone();
10086                        sealed = crate::values::decrypt_segment_row(
10087                            table,
10088                            &mut values,
10089                            &self.encryption,
10090                        )
10091                        .map_err(SectionError::Transport)?;
10092                        decrypted = values;
10093                        &decrypted
10094                    } else {
10095                        &row.values
10096                    };
10097                    if !self
10098                        .write_base_row(&table.name, values, row.server_version, sealed.as_ref())
10099                        .map_err(|message| {
10100                            SectionError::Abort("sync.invalid_request".into(), message)
10101                        })?
10102                    {
10103                        continue;
10104                    }
10105                    if reconcile_pending {
10106                        changed_keys.push(
10107                            owned_sql_value(RowParam::Cell(&values[table.pk_index])).map_err(
10108                                |message| SectionError::Abort("storage.failed".into(), message),
10109                            )?,
10110                        );
10111                    }
10112                    if mirror_visible {
10113                        self.write_row(
10114                            &visible_table(&table.name),
10115                            &table.name,
10116                            values,
10117                            row.server_version,
10118                            sealed.as_ref(),
10119                        )
10120                        .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10121                    }
10122                    let processed = u64::from(applied) + row_index as u64 + 1;
10123                    if processed.is_multiple_of(1024) {
10124                        self.progress.update(|p| p.rows_processed = processed);
10125                    }
10126                }
10127                self.reconcile_import(table, &mut changed_keys, protected, reconcile_pending)
10128                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10129                self.finish_observation("syncular_segment_block", batch)
10130                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))
10131            })();
10132            if let Err(error) = outcome {
10133                self.rollback_observation("syncular_segment_block");
10134                self.restore(rollback);
10135                return Err(error);
10136            }
10137            applied += block.len() as u32;
10138            self.progress
10139                .update(|p| p.rows_processed = u64::from(applied));
10140            std::thread::yield_now();
10141        }
10142        self.progress
10143            .update(|p| p.rows_processed = u64::from(applied));
10144        Ok(applied)
10145    }
10146
10147    /// §5.3 sqlite-image application: validate the in-file metadata
10148    /// against the descriptor, validate column names/order against the
10149    /// generated schema, run the §5.6 first-page clear when fresh, then
10150    /// replace-or-upsert every image row with its `_syncular_version`.
10151    /// Mechanics: the image lands in a temp file read through a second
10152    /// rusqlite connection, equivalent to ATTACH + INSERT…SELECT.
10153    /// Each image chunk commits its writes and revision before readers continue.
10154    #[allow(clippy::too_many_arguments)]
10155    fn apply_sqlite_segment(
10156        &mut self,
10157        table: &TableSchema,
10158        bytes: &[u8],
10159        first_page: bool,
10160        effective: &[(String, Vec<String>)],
10161        unit_id: &str,
10162        row_count: i64,
10163        as_of_commit_seq: i64,
10164        scope_digest: &str,
10165    ) -> Result<u32, SectionError> {
10166        let invalid = |detail: &str| {
10167            SectionError::Abort(
10168                "sync.invalid_request".to_owned(),
10169                format!("sqlite segment rejected: {detail} (§5.3)"),
10170            )
10171        };
10172        // §5.11: an image bypasses the decrypt seam, so a server cannot serve
10173        // one for a table with encrypted columns.
10174        if table.has_encrypted_columns() {
10175            return Err(invalid("table has encrypted columns"));
10176        }
10177
10178        // The image lands beside the replica so it shares the replica's
10179        // directory permissions and volume; an in-memory replica has no
10180        // directory and uses the system temp dir. The file is owner-only.
10181        let directory = self
10182            .previous_version_replica_path()
10183            .and_then(|replica| std::path::Path::new(&replica).parent().map(Into::into))
10184            .unwrap_or_else(std::env::temp_dir);
10185        let path = directory.join(format!(".syncular-image-{}.tmp", uuid::Uuid::new_v4()));
10186        let outcome = (|| {
10187            let mut options = std::fs::OpenOptions::new();
10188            options.write(true).create_new(true);
10189            #[cfg(unix)]
10190            std::os::unix::fs::OpenOptionsExt::mode(&mut options, 0o600);
10191            std::io::Write::write_all(
10192                &mut options
10193                    .open(&path)
10194                    .map_err(|_| invalid("image temp file write failed"))?,
10195                bytes,
10196            )
10197            .map_err(|_| invalid("image temp file write failed"))?;
10198            let img = rusqlite::Connection::open_with_flags(
10199                &path,
10200                rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY,
10201            )
10202            .map_err(|_| invalid("bytes do not open as a SQLite database"))?;
10203            self.apply_sqlite_image(
10204                &img,
10205                table,
10206                first_page,
10207                effective,
10208                unit_id,
10209                row_count,
10210                as_of_commit_seq,
10211                scope_digest,
10212            )
10213        })();
10214        let _ = std::fs::remove_file(&path);
10215        outcome
10216    }
10217
10218    #[allow(clippy::too_many_arguments)]
10219    fn apply_sqlite_image(
10220        &mut self,
10221        img: &rusqlite::Connection,
10222        table: &crate::schema::TableSchema,
10223        first_fresh_page: bool,
10224        effective: &[(String, Vec<String>)],
10225        unit_id: &str,
10226        row_count: i64,
10227        as_of_commit_seq: i64,
10228        scope_digest: &str,
10229    ) -> Result<u32, SectionError> {
10230        let invalid = |detail: String| {
10231            SectionError::Abort(
10232                "sync.invalid_request".to_owned(),
10233                format!("sqlite segment rejected: {detail} (§5.3)"),
10234            )
10235        };
10236
10237        // 1. Metadata vs descriptor + client state (§5.3 rule 2; exactly
10238        //    one row).
10239        type MetaRow = (i64, String, i64, i64, String, i64, i64);
10240        let meta: MetaRow = img
10241            .query_row(
10242                "SELECT format, \"table\", \"schemaVersion\", \"asOfCommitSeq\",
10243                        \"scopeDigest\", \"rowCount\",
10244                        (SELECT count(*) FROM _syncular_segment)
10245                 FROM _syncular_segment",
10246                [],
10247                |row| {
10248                    Ok((
10249                        row.get(0)?,
10250                        row.get(1)?,
10251                        row.get(2)?,
10252                        row.get(3)?,
10253                        row.get(4)?,
10254                        row.get(5)?,
10255                        row.get(6)?,
10256                    ))
10257                },
10258            )
10259            .map_err(|_| invalid("missing or unreadable _syncular_segment metadata".to_owned()))?;
10260        let (format, meta_table, schema_version, pin, digest, meta_rows, meta_count) = meta;
10261        if meta_count != 1 {
10262            return Err(invalid(format!(
10263                "_syncular_segment must contain exactly one row, found {meta_count}"
10264            )));
10265        }
10266        if format != 1 {
10267            return Err(invalid(format!("format {format}")));
10268        }
10269        if meta_table != table.name {
10270            return Err(invalid(format!("image table {meta_table:?}")));
10271        }
10272        if schema_version != i64::from(self.schema.version) {
10273            return Err(invalid(format!("schemaVersion {schema_version}")));
10274        }
10275        if pin != as_of_commit_seq {
10276            return Err(invalid(format!("asOfCommitSeq {pin}")));
10277        }
10278        if digest != scope_digest {
10279            return Err(invalid("scopeDigest mismatch".to_owned()));
10280        }
10281        if meta_rows != row_count {
10282            return Err(invalid(format!("rowCount {meta_rows}")));
10283        }
10284
10285        // 2. Column names and order vs the generated schema (§5.3 rule 3).
10286        let mut names: Vec<String> = Vec::new();
10287        let mut primary_keys: Vec<String> = Vec::new();
10288        {
10289            let mut stmt = img
10290                .prepare(&format!("PRAGMA table_info({})", quote_ident(&table.name)))
10291                .map_err(|_| invalid("image data table missing".to_owned()))?;
10292            let mut rows = stmt
10293                .query([])
10294                .map_err(|_| invalid("image data table unreadable".to_owned()))?;
10295            while let Some(row) = rows
10296                .next()
10297                .map_err(|_| invalid("image data table unreadable".to_owned()))?
10298            {
10299                if row
10300                    .get::<_, i64>(5)
10301                    .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?
10302                    > 0
10303                {
10304                    primary_keys.push(row.get(1).map_err(|error| {
10305                        invalid(Self::sqlite_failure(&self.storage_failure, error))
10306                    })?);
10307                }
10308                names.push(
10309                    row.get::<_, String>(1)
10310                        .map_err(|_| invalid("image data table unreadable".to_owned()))?,
10311                );
10312            }
10313        }
10314        let mut expected: Vec<&str> = table.columns.iter().map(|c| c.name.as_str()).collect();
10315        expected.push("_syncular_version");
10316        if names.len() != expected.len() || names.iter().zip(expected.iter()).any(|(a, b)| a != b) {
10317            return Err(SectionError::Transport(TransportError {
10318                code: "sync.schema_mismatch".to_owned(),
10319                message: "sqlite segment columns do not match the generated schema (§5.3)"
10320                    .to_owned(),
10321                details: Some(serde_json::json!({ "table": table.name })),
10322            }));
10323        }
10324
10325        if primary_keys != vec![table.primary_key.clone()] {
10326            return Err(invalid(
10327                "image primary key does not match generated schema".into(),
10328            ));
10329        }
10330        let actual_count: i64 = img
10331            .query_row(
10332                &format!("SELECT count(*) FROM {}", quote_ident(&table.name)),
10333                [],
10334                |row| row.get(0),
10335            )
10336            .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
10337        if actual_count != row_count {
10338            return Err(invalid("image row count does not match descriptor".into()));
10339        }
10340        let insert = self.insert_row_sql(&base_table(&table.name), table);
10341        let visible_insert = self.insert_row_sql(&visible_table(&table.name), table);
10342        let column_list = names
10343            .iter()
10344            .map(|n| quote_ident(n))
10345            .collect::<Vec<_>>()
10346            .join(", ");
10347        let mut stmt = img
10348            .prepare(&format!(
10349                "SELECT {column_list} FROM {} ORDER BY {}",
10350                quote_ident(&table.name),
10351                quote_ident(&table.primary_key)
10352            ))
10353            .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
10354        let mut rows = stmt
10355            .query([])
10356            .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
10357        let mut applied = 0u32;
10358        let mut limit = 1024u32;
10359        let (mut write, mut commit) = (std::time::Duration::ZERO, std::time::Duration::ZERO);
10360        loop {
10361            let clear = applied == 0 && first_fresh_page;
10362            let rollback = self.capture();
10363            let (pending, protected) = self
10364                .table_intent(&table.name)
10365                .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10366            let reconcile_pending =
10367                !protected && pending && table.indexes.iter().any(|index| index.unique);
10368            let mirror_visible = !protected && !reconcile_pending;
10369            // §5.3: the first chunk holds 1,024 rows. A later chunk holds at
10370            // least 1,024 rows, at most twice the rows of the one before, at
10371            // most 65,536, and at most the rows the one before applied per
10372            // 250 ms from its start to its COMMIT; a chunk that replays local
10373            // writes holds at most 1,024.
10374            let chunk = if pending || protected {
10375                limit.min(1024)
10376            } else {
10377                limit
10378            };
10379            self.coalesce_unit(chunk as usize)
10380                .and_then(|()| self.begin_observation("syncular_image_chunk"))
10381                .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10382            let started = std::time::Instant::now();
10383            let result = (|| {
10384                self.rebuild_overlay_if_dirty()
10385                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10386                let mut changed_keys = Vec::new();
10387                let mut batch = ChangeAccumulator::default();
10388                if clear && self.clear_scope(table, effective, unit_id)? {
10389                    batch.table(&table.name);
10390                }
10391                let mut processed = 0u32;
10392                {
10393                    let mut ins = self.conn.prepare_cached(&insert).map_err(|error| {
10394                        invalid(Self::sqlite_failure(&self.storage_failure, error))
10395                    })?;
10396                    let mut mirror = if mirror_visible {
10397                        Some(self.conn.prepare_cached(&visible_insert).map_err(|error| {
10398                            invalid(Self::sqlite_failure(&self.storage_failure, error))
10399                        })?)
10400                    } else {
10401                        None
10402                    };
10403                    for _ in 0..chunk {
10404                        let Some(row) = rows.next().map_err(|error| {
10405                            invalid(Self::sqlite_failure(&self.storage_failure, error))
10406                        })?
10407                        else {
10408                            break;
10409                        };
10410                        for (i, column) in table.columns.iter().enumerate() {
10411                            let cell = row.get_ref(i).map_err(|error| {
10412                                invalid(Self::sqlite_failure(&self.storage_failure, error))
10413                            })?;
10414                            let param = image_cell_param(column, cell).map_err(invalid)?;
10415                            ins.raw_bind_parameter(i + 1, &param).map_err(|error| {
10416                                invalid(Self::sqlite_failure(&self.storage_failure, error))
10417                            })?;
10418                            if let Some(mirror) = &mut mirror {
10419                                mirror.raw_bind_parameter(i + 1, &param).map_err(|error| {
10420                                    invalid(Self::sqlite_failure(&self.storage_failure, error))
10421                                })?;
10422                            }
10423                        }
10424                        let version_index = table.columns.len();
10425                        let version: i64 = row.get(version_index).map_err(|error| {
10426                            invalid(Self::sqlite_failure(&self.storage_failure, error))
10427                        })?;
10428                        if version < 1 {
10429                            return Err(invalid("image row version must be positive".into()));
10430                        }
10431                        ins.raw_bind_parameter(version_index + 1, version)
10432                            .map_err(|error| {
10433                                invalid(Self::sqlite_failure(&self.storage_failure, error))
10434                            })?;
10435                        // §4.5 version guard: the base upsert decides.
10436                        let landed = ins.raw_execute().map_err(|error| {
10437                            invalid(Self::sqlite_failure(&self.storage_failure, error))
10438                        })? > 0;
10439                        processed += 1;
10440                        if processed.is_multiple_of(1024) && processed < chunk {
10441                            self.progress
10442                                .update(|p| p.rows_processed = u64::from(applied + processed));
10443                        }
10444                        if !landed {
10445                            continue;
10446                        }
10447                        if let Some(mirror) = &mut mirror {
10448                            mirror
10449                                .raw_bind_parameter(version_index + 1, version)
10450                                .map_err(|error| {
10451                                    invalid(Self::sqlite_failure(&self.storage_failure, error))
10452                                })?;
10453                            mirror.raw_execute().map_err(|error| {
10454                                invalid(Self::sqlite_failure(&self.storage_failure, error))
10455                            })?;
10456                        }
10457                        if reconcile_pending {
10458                            changed_keys.push(row.get::<_, SqlValue>(table.pk_index).map_err(
10459                                |error| invalid(Self::sqlite_failure(&self.storage_failure, error)),
10460                            )?);
10461                        }
10462                    }
10463                }
10464                if processed > 0 {
10465                    batch.table(&table.name);
10466                    self.overlay_dirty.table(&table.name);
10467                }
10468                self.reconcile_import(table, &mut changed_keys, protected, reconcile_pending)
10469                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10470                let written = std::time::Instant::now();
10471                self.finish_observation("syncular_image_chunk", batch)
10472                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10473                Ok((processed, written))
10474            })();
10475            let (processed, written) = match result {
10476                Ok(done) => done,
10477                Err(error) => {
10478                    self.rollback_observation("syncular_image_chunk");
10479                    self.restore(rollback);
10480                    return Err(error);
10481                }
10482            };
10483            // The chunk commits before the yield, also inside a coalesced apply.
10484            let coalesced = self.coalesced.is_some();
10485            if coalesced {
10486                self.commit_coalesced_apply()
10487                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10488            }
10489            write += written - started;
10490            commit += written.elapsed();
10491            applied += processed;
10492            let last = processed < chunk || i64::from(applied) == actual_count;
10493            // A file replica's WAL keeps its high-water size until truncated.
10494            if last && self.conn.is_autocommit() {
10495                self.conn
10496                    .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))
10497                    .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
10498            }
10499            if coalesced {
10500                self.begin_coalesced_apply()
10501                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10502            }
10503            self.progress.update(|p| {
10504                p.rows_processed = u64::from(applied);
10505                p.import_write_ms = u64::try_from(write.as_millis()).ok();
10506                p.import_commit_ms = u64::try_from(commit.as_millis()).ok();
10507            });
10508            let duration = started.elapsed().as_secs_f64() * 1000.0;
10509            #[cfg(test)]
10510            let duration = self.image_chunk_ms.get().unwrap_or(duration);
10511            std::thread::yield_now();
10512            if last {
10513                break;
10514            }
10515            // A zero duration saturates the cast and leaves the doubling.
10516            limit = (processed * 2)
10517                .min(65_536)
10518                .min((f64::from(processed) * 250.0 / duration) as u32)
10519                .max(1024);
10520        }
10521        if i64::from(applied) != actual_count {
10522            return Err(invalid(format!(
10523                "image yielded {applied} rows, expected {actual_count}"
10524            )));
10525        }
10526        Ok(applied)
10527    }
10528
10529    /// §7.1 keyed reconciliation of the rows `drained` commits touch: restore
10530    /// them from the base and replay their remaining intent. A table with a
10531    /// secondary unique index restores and replays every row with intent,
10532    /// since one row's value can admit or defer another row's write.
10533    fn reconcile_rows(&self, drained: &[Arc<OutboxCommit>]) -> Result<(), String> {
10534        let names: BTreeSet<&str> = drained
10535            .iter()
10536            .flat_map(|commit| &commit.ops)
10537            .map(|op| op.table.as_str())
10538            .collect();
10539        for name in names {
10540            let Some(table) = self.schema.table(name) else {
10541                continue;
10542            };
10543            let ops = drained.iter().flat_map(|commit| &commit.ops);
10544            let tables = BTreeSet::from([name]);
10545            let rows: HashSet<_> = ops
10546                .clone()
10547                .map(|op| (op.table.as_str(), op.row_id.as_str()))
10548                .collect();
10549            let unique = table.indexes.iter().any(|index| index.unique);
10550            let acknowledged = if unique {
10551                self.acknowledged_operations(None, Some(&tables))?
10552            } else {
10553                Vec::new()
10554            };
10555            let peers = acknowledged
10556                .iter()
10557                .chain(
10558                    self.failed_commits
10559                        .iter()
10560                        .flat_map(|(commit, _)| &commit.ops),
10561                )
10562                .chain(self.outbox.iter().flat_map(|commit| &commit.ops))
10563                .filter(|_| unique);
10564            self.restore_base_rows(table, &mut Vec::new(), peers.chain(ops.clone()))?;
10565            self.replay_overlay(&tables, (!unique).then_some(&rows))?;
10566        }
10567        Ok(())
10568    }
10569
10570    /// Whether `table` has pending outbox intent, and whether it has protected
10571    /// (acknowledged or retained failed) intent.
10572    fn table_intent(&self, table: &str) -> Result<(bool, bool), String> {
10573        let touches = |op: &OutboxOp| op.table == table;
10574        Ok((
10575            self.outbox
10576                .iter()
10577                .flat_map(|commit| &commit.ops)
10578                .any(touches),
10579            self.failed_commits
10580                .iter()
10581                .flat_map(|(commit, _)| &commit.ops)
10582                .any(touches)
10583                || self
10584                    .conn
10585                    .query_row(
10586                        "SELECT EXISTS(SELECT 1 FROM _syncular_acked_rows WHERE tbl=?1)",
10587                        [table],
10588                        |row| row.get(0),
10589                    )
10590                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
10591        ))
10592    }
10593
10594    /// §7.1 reconciliation after an import block or chunk. Keyed replay covers
10595    /// only pending intent, so a table with protected intent rebuilds; a
10596    /// `restore` restores the imported `keys` and every pending row first.
10597    fn reconcile_import(
10598        &mut self,
10599        table: &TableSchema,
10600        keys: &mut Vec<SqlValue>,
10601        protected: bool,
10602        restore: bool,
10603    ) -> Result<(), String> {
10604        if protected {
10605            return self.rebuild_overlay_if_dirty();
10606        }
10607        let pending = self.outbox.iter().flat_map(|commit| &commit.ops);
10608        if restore {
10609            self.restore_base_rows(table, keys, pending.clone())?;
10610        }
10611        self.apply_outbox_ops(pending.filter(|op| op.table == table.name))?;
10612        self.overlay_dirty.set(false);
10613        Ok(())
10614    }
10615
10616    /// Restore the visible rows of `keys` and of the rows `ops` touch in
10617    /// `table` from the base before the caller replays their intent. The
10618    /// caller owns the transaction and publishes its revision after replay.
10619    fn restore_base_rows<'a>(
10620        &self,
10621        table: &TableSchema,
10622        keys: &mut Vec<SqlValue>,
10623        ops: impl IntoIterator<Item = &'a OutboxOp>,
10624    ) -> Result<(), String> {
10625        let visible = visible_table(&table.name);
10626        let base = base_table(&table.name);
10627        let pk = quote_ident(&table.primary_key);
10628        let predicate = row_id_predicate(table);
10629        let mut lookup = self.conn.prepare_cached(&format!(
10630            "SELECT {pk} FROM {base} WHERE {predicate} UNION SELECT {pk} FROM {visible} WHERE {predicate}"
10631        )).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
10632        let mut deleted = HashSet::new();
10633        for op in ops.into_iter().filter(|op| op.table == table.name) {
10634            if op.upsert {
10635                if let Some(values) = &op.values {
10636                    let key = json_to_column_value(
10637                        &table.columns[table.pk_index],
10638                        values.get(&table.primary_key),
10639                    )?;
10640                    keys.push(owned_sql_value(RowParam::Cell(&key))?);
10641                }
10642            } else if deleted.insert(&op.row_id) {
10643                keys.extend(
10644                    lookup
10645                        .query_map([&op.row_id], |row| row.get::<_, SqlValue>(0))
10646                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
10647                        .collect::<Result<Vec<_>, _>>()
10648                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
10649                );
10650            }
10651        }
10652        // Delete the complete affected set before restoring any base value, so
10653        // old optimistic unique values cannot collide with the new server rows.
10654        for chunk in keys.chunks(400) {
10655            let holes = vec!["?"; chunk.len()].join(",");
10656            self.conn
10657                .prepare_cached(&format!("DELETE FROM {visible} WHERE {pk} IN ({holes})"))
10658                .and_then(|mut statement| statement.execute(rusqlite::params_from_iter(chunk)))
10659                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
10660        }
10661        let updates = table
10662            .columns
10663            .iter()
10664            .map(|column| column.name.as_str())
10665            .chain(std::iter::once("_syncular_version"))
10666            .chain(table.sealed_columns().iter().copied())
10667            .filter(|name| *name != table.primary_key)
10668            .map(|name| {
10669                let name = quote_ident(name);
10670                format!("{name}=excluded.{name}")
10671            })
10672            .collect::<Vec<_>>()
10673            .join(",");
10674        for chunk in keys.chunks(400) {
10675            let holes = vec!["?"; chunk.len()].join(",");
10676            self.conn.prepare_cached(&format!("INSERT INTO {visible} SELECT * FROM {base} WHERE {pk} IN ({holes}) ON CONFLICT ({pk}) DO UPDATE SET {updates}"))
10677                .and_then(|mut statement| statement.execute(rusqlite::params_from_iter(chunk)))
10678                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
10679        }
10680        Ok(())
10681    }
10682
10683    // -- application-authorized local purge ---------------------------------------
10684
10685    fn compile_local_data_purge(
10686        &self,
10687        input: &LocalDataPurgeInput,
10688    ) -> Result<(Vec<CompiledLocalDataPurgeTarget>, String), String> {
10689        let invalid = |message: String| format!("sync.invalid_request: {message}");
10690        if input.purge_id.is_empty()
10691            || input.purge_id.len() > 128
10692            || !is_local_operation_code_like(&input.purge_id)
10693        {
10694            return Err(invalid(
10695                "local purge purgeId must be a 1–128 character code-like identifier".to_owned(),
10696            ));
10697        }
10698        if input.targets.is_empty() || input.targets.len() > MAX_LOCAL_PURGE_TARGETS {
10699            return Err(invalid(format!(
10700                "local purge needs between 1 and {MAX_LOCAL_PURGE_TARGETS} targets"
10701            )));
10702        }
10703
10704        let mut deduplicated: BTreeMap<
10705            String,
10706            (CompiledLocalDataPurgeTarget, LocalDataPurgeTarget),
10707        > = BTreeMap::new();
10708        for target in &input.targets {
10709            let table = self.schema.table(&target.table).ok_or_else(|| {
10710                invalid(format!(
10711                    "local purge names unknown table {:?}",
10712                    target.table
10713                ))
10714            })?;
10715            if target.selectors.is_empty() || target.selectors.len() > MAX_LOCAL_PURGE_SELECTORS {
10716                return Err(invalid(format!(
10717                    "local purge target {:?} needs between 1 and {MAX_LOCAL_PURGE_SELECTORS} selectors",
10718                    target.table
10719                )));
10720            }
10721            let mut selectors = Vec::with_capacity(target.selectors.len());
10722            let mut canonical_selectors = BTreeMap::new();
10723            for (column_name, raw_values) in &target.selectors {
10724                let Some((column_index, column)) = table
10725                    .columns
10726                    .iter()
10727                    .enumerate()
10728                    .find(|(_, column)| column.name == *column_name)
10729                else {
10730                    return Err(invalid(format!(
10731                        "local purge target {:?} names unknown column {:?}",
10732                        target.table, column_name
10733                    )));
10734                };
10735                let encrypted = table
10736                    .encrypted_columns
10737                    .iter()
10738                    .any(|candidate| candidate.index == column_index);
10739                if column.ty != ColumnType::String || encrypted {
10740                    return Err(invalid(format!(
10741                        "local purge selector {:?}.{:?} must be a plaintext string column",
10742                        target.table, column_name
10743                    )));
10744                }
10745                if raw_values.is_empty() || raw_values.len() > MAX_LOCAL_PURGE_VALUES {
10746                    return Err(invalid(format!(
10747                        "local purge selector {:?}.{:?} needs between 1 and {MAX_LOCAL_PURGE_VALUES} values",
10748                        target.table, column_name
10749                    )));
10750                }
10751                let mut values = raw_values.clone();
10752                values.sort();
10753                values.dedup();
10754                if values.iter().any(|value| {
10755                    value.is_empty()
10756                        || value.len() > MAX_LOCAL_PURGE_VALUE_LENGTH
10757                        || !is_local_operation_code_like(value)
10758                }) {
10759                    return Err(invalid(format!(
10760                        "local purge selector values must be 1–{MAX_LOCAL_PURGE_VALUE_LENGTH} character code-like identifiers"
10761                    )));
10762                }
10763                selectors.push((column_name.clone(), values.clone()));
10764                canonical_selectors.insert(column_name.clone(), values);
10765            }
10766            let canonical = LocalDataPurgeTarget {
10767                table: target.table.clone(),
10768                selectors: canonical_selectors,
10769            };
10770            let key = serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
10771            deduplicated.insert(
10772                key,
10773                (
10774                    CompiledLocalDataPurgeTarget {
10775                        table: target.table.clone(),
10776                        selectors,
10777                    },
10778                    canonical,
10779                ),
10780            );
10781        }
10782        let targets = deduplicated
10783            .values()
10784            .map(|(compiled, _)| compiled.clone())
10785            .collect::<Vec<_>>();
10786        let canonical = deduplicated
10787            .values()
10788            .map(|(_, target)| target.clone())
10789            .collect::<Vec<_>>();
10790        let canonical_plan =
10791            serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
10792        Ok((targets, canonical_plan))
10793    }
10794
10795    fn local_purge_base_row_ids(
10796        &self,
10797        targets: &[CompiledLocalDataPurgeTarget],
10798    ) -> Result<BTreeMap<String, BTreeSet<String>>, String> {
10799        let mut by_table: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
10800        for target in targets {
10801            let table = self
10802                .schema
10803                .table(&target.table)
10804                .ok_or_else(|| format!("sync.invalid_request: unknown table {:?}", target.table))?;
10805            let mut clauses = Vec::with_capacity(target.selectors.len());
10806            let mut params = Vec::new();
10807            for (column, values) in &target.selectors {
10808                clauses.push(format!(
10809                    "{} IN ({})",
10810                    quote_ident(column),
10811                    values.iter().map(|_| "?").collect::<Vec<_>>().join(", ")
10812                ));
10813                params.extend(values.iter().cloned().map(SqlValue::Text));
10814            }
10815            let sql = format!(
10816                "SELECT CAST({} AS TEXT) FROM {} WHERE {}",
10817                quote_ident(&table.primary_key),
10818                base_table(&target.table),
10819                clauses.join(" AND ")
10820            );
10821            let mut statement = self
10822                .conn
10823                .prepare(&sql)
10824                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
10825            let rows = statement
10826                .query_map(rusqlite::params_from_iter(params), |row| {
10827                    let value = sql_ref_to_json(&table.columns[table.pk_index], row.get_ref(0)?);
10828                    render_row_id_json(Some(&value))
10829                        .map_err(|message| rusqlite::Error::ToSqlConversionFailure(message.into()))
10830                })
10831                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
10832            let ids = by_table.entry(target.table.clone()).or_default();
10833            for row in rows {
10834                ids.insert(
10835                    row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
10836                );
10837            }
10838        }
10839        Ok(by_table)
10840    }
10841
10842    fn local_purge_values_match(
10843        target: &CompiledLocalDataPurgeTarget,
10844        values: &Map<String, Value>,
10845    ) -> bool {
10846        target.selectors.iter().all(|(column, allowed)| {
10847            values
10848                .get(column)
10849                .and_then(Value::as_str)
10850                .is_some_and(|value| allowed.iter().any(|candidate| candidate == value))
10851        })
10852    }
10853
10854    /// Apply one host-authorized local security purge. The host MUST gate the
10855    /// corresponding subscriptions first; this primitive owns local SQLite
10856    /// cleanup and never grants/revokes server authority by itself.
10857    pub fn purge_local_data(
10858        &mut self,
10859        input: &LocalDataPurgeInput,
10860    ) -> Result<LocalDataPurgeResult, String> {
10861        let (targets, canonical_plan) = self.compile_local_data_purge(input)?;
10862        let meta_key = format!("localPurge:{}", input.purge_id);
10863        let applied_plan = self
10864            .conn
10865            .query_row(
10866                "SELECT value FROM _syncular_meta WHERE key = ?1",
10867                rusqlite::params![meta_key],
10868                |row| row.get::<_, String>(0),
10869            )
10870            .optional()
10871            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
10872        if let Some(applied_plan) = applied_plan {
10873            if applied_plan != canonical_plan {
10874                return Err(format!(
10875                    "sync.invalid_request: local purge id {:?} was already used with a different plan",
10876                    input.purge_id
10877                ));
10878            }
10879            // A retry after a failed rewrite finishes it.
10880            self.erase_purged_pages()?;
10881            return Ok(LocalDataPurgeResult {
10882                already_applied: true,
10883                purged_rows: 0,
10884                dropped_commits: 0,
10885            });
10886        }
10887
10888        // RFC 0005 D7: the container file cannot be a purge target (its name is
10889        // not in the running schema), so purge drops it as a fixed step, before
10890        // any mirror row is touched and unconditionally.
10891        let _ = self.drop_previous_version();
10892
10893        let rollback = self.capture();
10894        self.begin_observation("syncular_local_purge")?;
10895        let applied = (|| -> Result<(ChangeAccumulator, LocalDataPurgeResult), String> {
10896            let row_ids = self.local_purge_base_row_ids(&targets)?;
10897            let retained_dropped = self.drop_failed_commits(|table, values| {
10898                targets.iter().any(|target| {
10899                    target.table == table && Self::local_purge_values_match(target, values)
10900                })
10901            })?;
10902            let doomed = self
10903                .outbox
10904                .iter()
10905                .filter(|commit| {
10906                    commit.ops.iter().any(|operation| {
10907                        let matching_targets = targets
10908                            .iter()
10909                            .filter(|target| target.table == operation.table)
10910                            .collect::<Vec<_>>();
10911                        if matching_targets.is_empty() {
10912                            return false;
10913                        }
10914                        if row_ids
10915                            .get(&operation.table)
10916                            .is_some_and(|ids| ids.contains(&operation.row_id))
10917                        {
10918                            return true;
10919                        }
10920                        operation.values.as_ref().is_some_and(|values| {
10921                            matching_targets
10922                                .iter()
10923                                .any(|target| Self::local_purge_values_match(target, values))
10924                        })
10925                    })
10926                })
10927                .cloned()
10928                .collect::<Vec<_>>();
10929            let doomed_ids = doomed
10930                .iter()
10931                .map(|commit| commit.client_commit_id.clone())
10932                .collect::<BTreeSet<_>>();
10933            let mut batch = ChangeAccumulator::default();
10934            for commit in &retained_dropped {
10935                batch.conflicts = true;
10936                batch.rejections = true;
10937                batch.outcomes = true;
10938                for operation in &commit.ops {
10939                    batch.table(&operation.table);
10940                }
10941            }
10942            let mut rejections = Vec::new();
10943            // §5.11 purge erasure: the journal keeps a dropped commit's identity and
10944            // codes, never its operations, so no purged value survives in it.
10945            for commit in &doomed {
10946                for operation in &commit.ops {
10947                    batch.table(&operation.table);
10948                    self.overlay_dirty.table(&operation.table);
10949                }
10950                let results = (0..commit.ops.len())
10951                    .map(|op_index| {
10952                        let rejection = RejectionRecord {
10953                            client_commit_id: commit.client_commit_id.clone(),
10954                            op_index: op_index as i32,
10955                            code: "client.local_data_purged".to_owned(),
10956                            message: "the commit was dropped by an application-authorized local data purge".to_owned(),
10957                            retryable: false,
10958                            details: None,
10959                            operation: None,
10960                        };
10961                        rejections.push(rejection.clone());
10962                        CommitOperationOutcome::Error { rejection }
10963                    })
10964                    .collect::<Vec<_>>();
10965                self.persist_commit_outcome(
10966                    &commit.client_commit_id,
10967                    CommitOutcomeStatus::Rejected,
10968                    &results,
10969                    None,
10970                )?;
10971                self.delete_outbox_persisted(&commit.client_commit_id)?;
10972            }
10973            if !doomed.is_empty() {
10974                self.outbox
10975                    .retain(|commit| !doomed_ids.contains(&commit.client_commit_id));
10976                self.rejections.extend(rejections);
10977                self.prune_commit_outcomes()?;
10978                batch.status = true;
10979                batch.rejections = true;
10980                batch.outcomes = true;
10981            }
10982
10983            let mut purged_rows = 0usize;
10984            for (table, ids) in &row_ids {
10985                if ids.is_empty() {
10986                    continue;
10987                }
10988                batch.table(table);
10989                self.overlay_dirty.table(table);
10990                for row_id in ids {
10991                    self.delete_base_row(table, row_id, i64::MAX)?;
10992                    purged_rows += 1;
10993                }
10994            }
10995            // §5.11 purge erasure: a journaled outcome with an operation or
10996            // conflict row the purge reaches keeps its identity, status, ids
10997            // and codes only.
10998            let touches = |table: &str, row_id: &str, values: Option<&Map<String, Value>>| {
10999                targets.iter().any(|target| {
11000                    target.table == table
11001                        && (row_ids.get(table).is_some_and(|ids| ids.contains(row_id))
11002                            || values.is_some_and(|values| {
11003                                Self::local_purge_values_match(target, values)
11004                            }))
11005                })
11006            };
11007            let entries = self
11008                .conn
11009                .prepare("SELECT client_commit_id, results_json, operations_json FROM _syncular_commit_outcomes")
11010                .and_then(|mut statement| {
11011                    statement
11012                        .query_map([], |row| {
11013                            Ok((
11014                                row.get::<_, String>(0)?,
11015                                row.get::<_, String>(1)?,
11016                                row.get::<_, Option<String>>(2)?,
11017                            ))
11018                        })?
11019                        .collect::<Result<Vec<_>, _>>()
11020                })
11021                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11022            for (client_commit_id, results_json, operations_json) in entries {
11023                let corrupt = |_| "sync.local_corrupt: persisted commit outcome is invalid".to_owned();
11024                let mut results: Vec<CommitOperationOutcome> =
11025                    serde_json::from_str(&results_json).map_err(corrupt)?;
11026                let operations: Vec<CommitOperation> = operations_json
11027                    .map(|json| serde_json::from_str(&json))
11028                    .transpose()
11029                    .map_err(corrupt)?
11030                    .unwrap_or_default();
11031                let named = operations
11032                    .iter()
11033                    .chain(results.iter().filter_map(|result| match result {
11034                        CommitOperationOutcome::Conflict { conflict } => conflict.operation.as_ref(),
11035                        CommitOperationOutcome::Error { rejection } => rejection.operation.as_ref(),
11036                        CommitOperationOutcome::Applied { .. } => None,
11037                    }))
11038                    .any(|operation| touches(&operation.table, &operation.row_id, operation.values.as_ref()))
11039                    || results.iter().any(|result| {
11040                        matches!(result, CommitOperationOutcome::Conflict { conflict }
11041                            if touches(&conflict.table, &conflict.row_id, Some(&conflict.server_row)))
11042                    });
11043                if !named {
11044                    continue;
11045                }
11046                for result in &mut results {
11047                    match result {
11048                        CommitOperationOutcome::Conflict { conflict } => {
11049                            conflict.server_row = Map::new();
11050                            conflict.conflict_columns.clear();
11051                            conflict.operation = None;
11052                        }
11053                        CommitOperationOutcome::Error { rejection } => {
11054                            rejection.details = None;
11055                            rejection.operation = None;
11056                        }
11057                        CommitOperationOutcome::Applied { .. } => {}
11058                    }
11059                }
11060                let results_json = serde_json::to_string(&results).map_err(|error| error.to_string())?;
11061                self.conn
11062                    .execute(
11063                        "UPDATE _syncular_commit_outcomes SET results_json = ?1, operations_json = NULL WHERE client_commit_id = ?2",
11064                        rusqlite::params![results_json, client_commit_id],
11065                    )
11066                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11067                batch.outcomes = true;
11068            }
11069            self.rebuild_overlay_if_dirty()?;
11070            self.delete_unreferenced_cached_blobs()?;
11071            self.conn
11072                .execute(
11073                    "INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
11074                    rusqlite::params![meta_key, canonical_plan],
11075                )
11076                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11077            Ok((
11078                batch,
11079                LocalDataPurgeResult {
11080                    already_applied: false,
11081                    purged_rows,
11082                    dropped_commits: doomed.len(),
11083                },
11084            ))
11085        })()
11086        .and_then(|(batch, result)| {
11087            self.finish_observation("syncular_local_purge", batch)
11088                .map(|()| result)
11089        });
11090        let result = match applied {
11091            Ok(result) => result,
11092            Err(error) => {
11093                self.rollback_observation("syncular_local_purge");
11094                self.restore(rollback);
11095                return Err(error);
11096            }
11097        };
11098        self.cancel_sync_round();
11099        self.erase_purged_pages()?;
11100        Ok(result)
11101    }
11102
11103    /// §5.11 purge erasure: SQLite leaves a deleted value in the freed pages of the
11104    /// database file, and the WAL keeps the frames written before the purge,
11105    /// until either is rewritten. VACUUM rebuilds the file from the remaining
11106    /// rows, and the checkpoint truncates the WAL.
11107    fn erase_purged_pages(&self) -> Result<(), String> {
11108        let erased = self
11109            .conn
11110            .execute_batch("PRAGMA secure_delete = ON; VACUUM;")
11111            .and_then(|()| {
11112                self.conn
11113                    .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))
11114            });
11115        let reset = self.conn.execute_batch("PRAGMA secure_delete = OFF");
11116        erased
11117            .and(reset)
11118            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
11119    }
11120
11121    /// Application-authorized recovery of the replicated projection. Unlike a
11122    /// local security purge, this retains the entire outbox, device identity,
11123    /// lease, outcomes, unit registrations, and protected bookkeeping.
11124    /// The projection reset, unit rewind, optimistic replay, and
11125    /// idempotency marker share one savepoint for interruption safety.
11126    pub fn rebootstrap_local_data(
11127        &mut self,
11128        input: &LocalDataRebootstrapInput,
11129    ) -> Result<LocalDataRebootstrapResult, String> {
11130        if input.rebootstrap_id.is_empty()
11131            || input.rebootstrap_id.len() > 128
11132            || !is_local_operation_code_like(&input.rebootstrap_id)
11133        {
11134            return Err(
11135                "sync.invalid_request: local rebootstrap rebootstrapId must be a 1–128 character code-like identifier"
11136                    .to_owned(),
11137            );
11138        }
11139        let meta_key = format!("localRebootstrap:{}", input.rebootstrap_id);
11140        if let Some(persisted) = meta_get(&self.conn, &meta_key).map_err(|_| {
11141            "sync.local_corrupt: persisted local rebootstrap receipt is unreadable".to_owned()
11142        })? {
11143            let (retained_commits, reset_subscriptions) =
11144                decode_local_rebootstrap_receipt(&persisted)?;
11145            return Ok(LocalDataRebootstrapResult {
11146                already_applied: true,
11147                retained_commits,
11148                reset_subscriptions,
11149            });
11150        }
11151        if self.stopped || self.schema_floor.is_some() {
11152            return Err(
11153                "sync.invalid_request: local rebootstrap cannot bypass an active schema-floor stop; update the application first"
11154                    .to_owned(),
11155            );
11156        }
11157
11158        let retained_commits = self.outbox.len();
11159        let reset_subscriptions = self.stored(load_units(&self.conn))?.len();
11160        let receipt = encode_local_rebootstrap_receipt(retained_commits, reset_subscriptions)?;
11161        let rollback = self.capture();
11162        self.begin_observation("syncular_local_rebootstrap")?;
11163        let mut batch = ChangeAccumulator::default();
11164        let applied = (|| -> Result<(), String> {
11165            self.run_schema_reset_observed(&mut batch, false)?;
11166            self.conn
11167                .execute(
11168                    "INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
11169                    rusqlite::params![meta_key, receipt],
11170                )
11171                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11172            self.sync_needed = true;
11173            self.sync_intent_queue.push_back(SyncIntent::Interactive);
11174            batch.status = true;
11175            Ok(())
11176        })()
11177        .and_then(|()| self.finish_observation("syncular_local_rebootstrap", batch));
11178        if let Err(error) = applied {
11179            self.rollback_observation("syncular_local_rebootstrap");
11180            self.restore(rollback);
11181            return Err(error);
11182        }
11183
11184        Ok(LocalDataRebootstrapResult {
11185            already_applied: false,
11186            retained_commits,
11187            reset_subscriptions,
11188        })
11189    }
11190
11191    // -- scope purge + doomed outbox (§3.3) ----------------------------------------
11192
11193    /// Delete base and visible rows owned by the effective scopes
11194    /// ([`local_scope_predicate`]); `Ok(false)` = no local scope-column
11195    /// mapping for a key (the fail-closed case). A SQLite failure is an error.
11196    fn purge_scope_rows(
11197        &mut self,
11198        table_name: &str,
11199        effective: &[(String, Vec<String>)],
11200        owners: &[Vec<(String, Vec<String>)>],
11201    ) -> Result<bool, String> {
11202        let Some(table) = self.schema.table(table_name) else {
11203            return Ok(false);
11204        };
11205        let Ok(predicate) = local_scope_predicate(table, effective, owners) else {
11206            return Ok(false);
11207        };
11208        let Some((predicate, params)) = predicate else {
11209            return Ok(true);
11210        };
11211        self.overlay_dirty.table(table_name);
11212        for source in [base_table(table_name), visible_table(table_name)] {
11213            self.conn
11214                .execute(
11215                    &format!("DELETE FROM {source} WHERE {predicate}"),
11216                    rusqlite::params_from_iter(&params),
11217                )
11218                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11219        }
11220        Ok(true)
11221    }
11222
11223    /// §3.3: drop pending commits whose upserts provably land in the
11224    /// revoked effective scopes and no other owner covers (row ownership)
11225    /// — whole-commit, never per-operation.
11226    fn drop_doomed_outbox(
11227        &mut self,
11228        table: &TableSchema,
11229        effective: &[(String, Vec<String>)],
11230        owners: &[Vec<(String, Vec<String>)>],
11231    ) -> Result<bool, String> {
11232        let doomed: Vec<_> =
11233            self.outbox
11234                .iter()
11235                .filter(|commit| {
11236                    commit.ops.iter().any(|op| {
11237                        op.upsert
11238                            && op.table == table.name
11239                            && op.values.as_ref().is_some_and(|values| {
11240                                owns_local_row(table, effective, owners, values)
11241                            })
11242                    })
11243                })
11244                .cloned()
11245                .collect();
11246        if doomed.is_empty() {
11247            return Ok(false);
11248        }
11249        let mut rejections = Vec::new();
11250        for commit in &doomed {
11251            let results = commit
11252                .ops
11253                .iter()
11254                .enumerate()
11255                .map(|(op_index, operation)| {
11256                    self.overlay_dirty.table(&operation.table);
11257                    let rejection = RejectionRecord {
11258                        client_commit_id: commit.client_commit_id.clone(),
11259                        op_index: op_index as i32,
11260                        code: "sync.scope_revoked".to_owned(),
11261                        message: "the commit was dropped because its effective scope was revoked"
11262                            .to_owned(),
11263                        retryable: false,
11264                        details: None,
11265                        operation: Some(CommitOperation::from(operation)),
11266                    };
11267                    rejections.push(rejection.clone());
11268                    CommitOperationOutcome::Error { rejection }
11269                })
11270                .collect::<Vec<_>>();
11271            self.persist_commit_outcome(
11272                &commit.client_commit_id,
11273                CommitOutcomeStatus::Rejected,
11274                &results,
11275                Some(&commit.ops),
11276            )?;
11277            self.delete_outbox_persisted(&commit.client_commit_id)?;
11278        }
11279        self.prune_commit_outcomes()?;
11280        let doomed_ids = doomed
11281            .iter()
11282            .map(|commit| commit.client_commit_id.as_str())
11283            .collect::<BTreeSet<_>>();
11284        self.outbox
11285            .retain(|commit| !doomed_ids.contains(commit.client_commit_id.as_str()));
11286        self.rejections.extend(rejections);
11287        Ok(true)
11288    }
11289
11290    // -- blobs (§5.9) ----------------------------------------------------------------
11291
11292    /// §5.9.7: hash bytes into the content address, cache them, queue the
11293    /// upload (flushed before the next push, B4). Returns the canonical
11294    /// BlobRef JSON `{blobId, byteLength, mediaType?}` for a `blob_ref`
11295    /// column value.
11296    pub fn upload_blob(
11297        &mut self,
11298        bytes: &[u8],
11299        media_type: Option<String>,
11300        name: Option<String>,
11301    ) -> Result<Value, String> {
11302        let blob_id = blob_id_for(bytes);
11303        let now = self.clock_now_ms();
11304        let tx = self
11305            .conn
11306            .transaction()
11307            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11308        tx.execute(
11309            "INSERT INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES (?,?,?,?,?)
11310             ON CONFLICT(blob_id) DO NOTHING",
11311            rusqlite::params![blob_id, bytes, bytes.len() as i64, media_type, now],
11312        )
11313        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11314        tx.execute(
11315            "INSERT OR IGNORE INTO _syncular_blob_uploads(blob_id, media_type, created_at_ms) VALUES (?,?,?)",
11316            rusqlite::params![blob_id, media_type, now],
11317        )
11318        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11319        tx.commit()
11320            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11321        self.enforce_blob_cache_cap();
11322        let mut obj = Map::new();
11323        obj.insert("blobId".to_owned(), Value::from(blob_id));
11324        obj.insert("byteLength".to_owned(), Value::from(bytes.len() as i64));
11325        if let Some(mt) = media_type {
11326            obj.insert("mediaType".to_owned(), Value::from(mt));
11327        }
11328        if let Some(n) = name {
11329            obj.insert("name".to_owned(), Value::from(n));
11330        }
11331        Ok(Value::Object(obj))
11332    }
11333
11334    /// §5.9.7: resolve an owned blob body without driver encoding. A
11335    /// content-addressed cache hit serves with no fetch (B1); a miss downloads
11336    /// (§5.9.5), verifies the address, caches the bytes, and returns them.
11337    pub fn fetch_blob_bytes(
11338        &mut self,
11339        transport: &mut dyn Transport,
11340        blob_id_or_ref: &str,
11341    ) -> Result<FetchedBlob, (String, String)> {
11342        let simple = |m: String| ("client.failed".to_owned(), m);
11343        // A BlobRef declares the size that caps the download (§5.9.5); a bare
11344        // blobId declares none.
11345        let (blob_id, byte_length) = if blob_id_or_ref.starts_with("sha256:") {
11346            (blob_id_or_ref.to_owned(), None)
11347        } else {
11348            let value: Value = serde_json::from_str(blob_id_or_ref)
11349                .map_err(|_| simple("blob ref is not JSON".to_owned()))?;
11350            (
11351                value
11352                    .get("blobId")
11353                    .and_then(Value::as_str)
11354                    .ok_or_else(|| simple("blob ref has no blobId".to_owned()))?
11355                    .to_owned(),
11356                value.get("byteLength").and_then(Value::as_u64),
11357            )
11358        };
11359        if let Some(cached) = self.get_cached_blob_bytes(&blob_id).map_err(simple)? {
11360            return Ok(cached);
11361        }
11362        if !self.transport_enabled {
11363            return Err(("sync.offline".into(), "transport gate is closed".into()));
11364        }
11365        // §5.9.5: propagate the server's blob.* code (blob.forbidden /
11366        // blob.not_found) verbatim so the harness can assert on it. The
11367        // authorized endpoint serves bytes inline OR (always-issue, presign
11368        // configured) a signed url the client fetches directly — no host auth,
11369        // no fall-through: failure => re-request (the caller's next fetch_blob_bytes).
11370        #[cfg(feature = "bench-internals")]
11371        let download_phase = self.benchmark_phases.start(Phase::BlobDownload);
11372        let bytes = match transport
11373            .blob_download(&blob_id, byte_length)
11374            .map_err(|e| (e.code, e.message))?
11375        {
11376            BlobDownload::Bytes(bytes) => bytes,
11377            BlobDownload::Url {
11378                url,
11379                url_expires_at_ms,
11380            } => {
11381                // §5.9.5: MUST NOT start a fetch at/past expiry.
11382                if url_expires_at_ms.is_some_and(|exp| exp <= self.clock_now_ms()) {
11383                    return Err((
11384                        "sync.segment_expired".to_owned(),
11385                        format!(
11386                            "blob url for {blob_id} expired before fetch — re-request mints a fresh url (§5.9.5)"
11387                        ),
11388                    ));
11389                }
11390                transport
11391                    .fetch_blob_url(&url, byte_length)
11392                    .map_err(|e| (e.code, e.message))?
11393            }
11394        };
11395        #[cfg(feature = "bench-internals")]
11396        drop(download_phase);
11397        #[cfg(feature = "bench-internals")]
11398        let validate_phase = self.benchmark_phases.start(Phase::BlobValidate);
11399        // §5.9.5 inherits §5.1: the declared size caps the body before the
11400        // content address is verified.
11401        if byte_length.is_some_and(|cap| bytes.len() as u64 > cap) {
11402            return Err((
11403                "sync.invalid_request".to_owned(),
11404                "blob exceeds its byteLength".to_owned(),
11405            ));
11406        }
11407        if blob_id_for(&bytes) != blob_id {
11408            return Err(simple(format!(
11409                "blob content address mismatch for {blob_id}"
11410            )));
11411        }
11412        #[cfg(feature = "bench-internals")]
11413        drop(validate_phase);
11414        #[cfg(feature = "bench-internals")]
11415        let insert_phase = self.benchmark_phases.start(Phase::BlobCacheInsert);
11416        let now = self.clock_now_ms();
11417        self.conn
11418            .execute(
11419                "INSERT OR IGNORE INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES (?,?,?,NULL,?)",
11420                rusqlite::params![blob_id, bytes, bytes.len() as i64, now],
11421            )
11422            .map_err(|e| simple(e.to_string()))?;
11423        self.conn
11424            .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))
11425            .map_err(|e| simple(e.to_string()))?;
11426        self.enforce_blob_cache_cap();
11427        #[cfg(feature = "bench-internals")]
11428        drop(insert_phase);
11429        self.get_cached_blob_bytes(&blob_id)
11430            .map_err(simple)?
11431            .ok_or_else(|| simple("blob cache write failed".to_owned()))
11432    }
11433
11434    fn get_cached_blob_bytes(&self, blob_id: &str) -> Result<Option<FetchedBlob>, String> {
11435        #[cfg(feature = "bench-internals")]
11436        let _phase = self.benchmark_phases.start(Phase::BlobCacheRead);
11437        let mut stmt = self
11438            .conn
11439            .prepare("SELECT bytes, byte_length, media_type FROM _syncular_blobs WHERE blob_id = ?")
11440            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11441        let mut rows = stmt
11442            .query(rusqlite::params![blob_id])
11443            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11444        if let Some(row) = rows
11445            .next()
11446            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
11447        {
11448            let bytes: Vec<u8> = row
11449                .get(0)
11450                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11451            let byte_length: i64 = row
11452                .get(1)
11453                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11454            let media_type: Option<String> = row
11455                .get(2)
11456                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11457            return Ok(Some(FetchedBlob {
11458                blob_id: blob_id.to_owned(),
11459                byte_length,
11460                bytes,
11461                media_type,
11462            }));
11463        }
11464        Ok(None)
11465    }
11466
11467    /// §5.9.7 B4: upload every queued blob before push.
11468    fn pending_blob_uploads(&self) -> Result<Vec<crate::round::PendingUpload>, TransportError> {
11469        let pending: Vec<(String, Option<String>)> = {
11470            let mut stmt = self
11471                .conn
11472                .prepare(
11473                    "SELECT blob_id, media_type FROM (
11474                      SELECT blob_id, media_type, created_at_ms FROM _syncular_blob_uploads
11475                      UNION ALL
11476                      SELECT DISTINCT r.blob_id, b.media_type, 9223372036854775807 AS created_at_ms
11477                      FROM _syncular_blob_commit_refs r LEFT JOIN _syncular_blobs b ON b.blob_id = r.blob_id
11478                      WHERE NOT EXISTS (SELECT 1 FROM _syncular_blob_uploads u WHERE u.blob_id = r.blob_id)
11479                    ) ORDER BY created_at_ms, blob_id",
11480                )
11481                .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
11482            let rows = stmt
11483                .query_map([], |row| {
11484                    Ok((row.get::<_, String>(0)?, row.get::<_, Option<String>>(1)?))
11485                })
11486                .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
11487            rows.collect::<Result<_, _>>()
11488                .map_err(|error| match error {
11489                    rusqlite::Error::InvalidColumnType(..)
11490                    | rusqlite::Error::FromSqlConversionFailure(..) => TransportError::new(
11491                        "sync.local_corrupt",
11492                        "Pending blob upload metadata is invalid",
11493                    ),
11494                    _ => TransportError::new("client.failed", error.to_string()),
11495                })?
11496        };
11497        let mut uploads = Vec::new();
11498        for (blob_id, media_type) in pending {
11499            let bytes = (|| {
11500                let stored: Option<(SqlValue, SqlValue)> = self
11501                    .conn
11502                    .query_row(
11503                        "SELECT bytes, byte_length FROM _syncular_blobs WHERE blob_id = ?",
11504                        rusqlite::params![blob_id],
11505                        |row| Ok((row.get(0)?, row.get(1)?)),
11506                    )
11507                    .optional()
11508                    .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
11509                let Some((SqlValue::Blob(bytes), SqlValue::Integer(byte_length))) = stored else {
11510                    return Err(TransportError::new(
11511                        "sync.local_corrupt",
11512                        "Pending blob upload body is missing or corrupt",
11513                    ));
11514                };
11515                if byte_length != bytes.len() as i64 || blob_id_for(&bytes) != blob_id {
11516                    return Err(TransportError::new(
11517                        "sync.local_corrupt",
11518                        "Pending blob upload body is missing or corrupt",
11519                    ));
11520                }
11521                Ok(bytes)
11522            })();
11523            let failed = bytes.is_err();
11524            uploads.push(crate::round::PendingUpload {
11525                id: blob_id,
11526                bytes,
11527                media_type,
11528            });
11529            if failed {
11530                break;
11531            }
11532        }
11533        Ok(uploads)
11534    }
11535
11536    fn visible_blob_ids_sql(&self) -> String {
11537        let mut selects = Vec::new();
11538        for table in &self.schema.tables {
11539            for column in table
11540                .columns
11541                .iter()
11542                .filter(|column| column.ty == ColumnType::BlobRef)
11543            {
11544                let identifier = quote_ident(&column.name);
11545                selects.push(format!(
11546                    "SELECT json_extract({identifier}, '$.blobId') AS blob_id FROM {} \
11547                     WHERE typeof({identifier}) = 'text' AND json_valid({identifier}) \
11548                     AND json_type({identifier}, '$.blobId') = 'text'",
11549                    quote_ident(&table.name),
11550                ));
11551            }
11552        }
11553        if selects.is_empty() {
11554            "SELECT NULL AS blob_id WHERE 0".to_owned()
11555        } else {
11556            selects.join(" UNION ")
11557        }
11558    }
11559
11560    /// §5.9.7 B1 size cap. Visible rows, pending commits, and
11561    /// staged uploads pin bodies directly; no derived refcount is stored.
11562    fn enforce_blob_cache_cap(&self) {
11563        let Some(max_bytes) = self.limits.blob_cache_max_bytes else {
11564            return;
11565        };
11566        let mut total: i64 = self
11567            .conn
11568            .query_row(
11569                "SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs",
11570                [],
11571                |row| row.get(0),
11572            )
11573            .unwrap_or(0);
11574        if total <= max_bytes {
11575            return;
11576        }
11577        let query = format!(
11578            "SELECT blob_id, byte_length FROM _syncular_blobs
11579             WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
11580               AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
11581               AND blob_id NOT IN ({})
11582             ORDER BY created_at_ms ASC, blob_id ASC",
11583            self.visible_blob_ids_sql(),
11584        );
11585        let candidates: Vec<(String, i64)> = {
11586            let Ok(mut stmt) = self.conn.prepare(&query) else {
11587                return;
11588            };
11589            let Ok(rows) = stmt.query_map([], |row| {
11590                Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?))
11591            }) else {
11592                return;
11593            };
11594            rows.filter_map(Result::ok).collect()
11595        };
11596        for (blob_id, byte_length) in candidates {
11597            if total <= max_bytes {
11598                break;
11599            }
11600            let _ = self.conn.execute(
11601                "DELETE FROM _syncular_blobs WHERE blob_id = ?",
11602                rusqlite::params![blob_id],
11603            );
11604            total -= byte_length;
11605        }
11606    }
11607
11608    fn delete_unreferenced_cached_blobs(&mut self) -> Result<(), String> {
11609        #[cfg(feature = "bench-internals")]
11610        let _phase = self.benchmark_phases.start(Phase::BlobRetention);
11611        if !self.schema_has_blobs() {
11612            return Ok(());
11613        }
11614        self.rebuild_overlay_if_dirty()?;
11615        self.conn
11616            .execute(
11617                &format!(
11618                    "DELETE FROM _syncular_blobs
11619                 WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
11620                   AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
11621                   AND blob_id NOT IN ({})",
11622                    self.visible_blob_ids_sql(),
11623                ),
11624                [],
11625            )
11626            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11627        Ok(())
11628    }
11629
11630    // -- local row storage ----------------------------------------------------------
11631
11632    /// The cached per-table primary-key upsert SQL for `full_table` (see
11633    /// the `insert_sql` field: built once, reused per row).
11634    fn insert_row_sql(&self, full_table: &str, table: &crate::schema::TableSchema) -> String {
11635        if let Some(sql) = self.insert_sql.borrow().get(full_table) {
11636            return sql.clone();
11637        }
11638        let mut columns: Vec<String> = table.columns.iter().map(|c| quote_ident(&c.name)).collect();
11639        columns.push(quote_ident("_syncular_version"));
11640        columns.extend(table.sealed_columns().iter().map(|name| quote_ident(name)));
11641        let placeholders: Vec<&str> = columns.iter().map(|_| "?").collect();
11642        let primary_key = quote_ident(&table.primary_key);
11643        let updates = columns
11644            .iter()
11645            .filter(|column| **column != primary_key)
11646            .map(|column| format!("{column}=excluded.{column}"))
11647            .collect::<Vec<_>>()
11648            .join(", ");
11649        // §4.5 version guard: server state lands in the base table only above
11650        // the stored version; the visible overlay takes every write.
11651        let guard = if full_table == base_table(&table.name) {
11652            format!(" WHERE excluded._syncular_version > {full_table}._syncular_version")
11653        } else {
11654            String::new()
11655        };
11656        let sql = format!(
11657            "INSERT INTO {full_table} ({}) VALUES ({}) ON CONFLICT ({primary_key}) DO UPDATE SET {updates}{guard}",
11658            columns.join(", "),
11659            placeholders.join(", ")
11660        );
11661        self.insert_sql
11662            .borrow_mut()
11663            .insert(full_table.to_owned(), sql.clone());
11664        sql
11665    }
11666
11667    /// §4.5: write a server row under the version guard; returns whether it
11668    /// landed. `sealed` is its §5.11 sealed state, `None` for an unsealed row.
11669    fn write_base_row(
11670        &self,
11671        table_name: &str,
11672        row: &Row,
11673        version: i64,
11674        sealed: Option<&SealedRow>,
11675    ) -> Result<bool, String> {
11676        let landed = self.write_row(&base_table(table_name), table_name, row, version, sealed)?;
11677        if landed {
11678            self.overlay_dirty.table(table_name);
11679        }
11680        Ok(landed)
11681    }
11682
11683    fn write_row(
11684        &self,
11685        full_table: &str,
11686        table_name: &str,
11687        row: &Row,
11688        version: i64,
11689        sealed: Option<&SealedRow>,
11690    ) -> Result<bool, String> {
11691        #[cfg(feature = "bench-internals")]
11692        let _phase = self.benchmark_phases.start(Phase::RowWrite);
11693        let table = self
11694            .schema
11695            .table(table_name)
11696            .ok_or_else(|| format!("unknown table {table_name:?}"))?;
11697        let sql = self.insert_row_sql(full_table, table);
11698        let mut stmt = self
11699            .conn
11700            .prepare_cached(&sql)
11701            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11702        let params = row
11703            .iter()
11704            .map(RowParam::Cell)
11705            .chain(std::iter::once(RowParam::Version(version)))
11706            .chain(
11707                [
11708                    sealed.map(|sealed| sealed.key_id.as_str()),
11709                    sealed.map(|sealed| sealed.envelopes.as_str()),
11710                ]
11711                .into_iter()
11712                .take(table.sealed_columns().len())
11713                .map(RowParam::Text),
11714            );
11715        stmt.execute(rusqlite::params_from_iter(params))
11716            .map(|changed| changed > 0)
11717            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
11718    }
11719
11720    /// §4.5: delete a server row whose stored version lies below `version`;
11721    /// returns whether a row left.
11722    fn delete_base_row(
11723        &self,
11724        table_name: &str,
11725        row_id: &str,
11726        version: i64,
11727    ) -> Result<bool, String> {
11728        let table = self
11729            .schema
11730            .table(table_name)
11731            .ok_or_else(|| format!("unknown table {table_name:?}"))?;
11732        let sql = format!(
11733            "DELETE FROM {} WHERE {} AND _syncular_version < ?2",
11734            base_table(table_name),
11735            row_id_predicate(table)
11736        );
11737        let mut stmt = self
11738            .conn
11739            .prepare_cached(&sql)
11740            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11741        let landed = stmt
11742            .execute(rusqlite::params![row_id, version])
11743            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
11744            > 0;
11745        if landed {
11746            self.overlay_dirty.table(table_name);
11747        }
11748        Ok(landed)
11749    }
11750
11751    fn has_acknowledged_rows(&self) -> Result<bool, String> {
11752        self.conn
11753            .query_row(
11754                "SELECT EXISTS(SELECT 1 FROM _syncular_acked_rows)",
11755                [],
11756                |row| row.get(0),
11757            )
11758            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
11759    }
11760
11761    fn acknowledged_operations(
11762        &self,
11763        rows: Option<&HashSet<(&str, &str)>>,
11764        tables: Option<&BTreeSet<&str>>,
11765    ) -> Result<Vec<OutboxOp>, String> {
11766        let keys = rows.map(|rows| serde_json::to_string(&rows).expect("row keys"));
11767        let mut statement = self
11768            .conn
11769            .prepare_cached(if keys.is_some() {
11770                ACKED_ROWS_BY_ROW
11771            } else {
11772                "SELECT op_json,intent_json FROM _syncular_acked_rows WHERE ?1 IS NULL AND (?2 IS NULL OR tbl IN(SELECT value FROM json_each(?2))) ORDER BY commit_seq,idx"
11773            })
11774            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11775        let rows = statement
11776            .query_map(
11777                rusqlite::params![
11778                    keys,
11779                    tables.map(|tables| serde_json::to_string(tables).expect("table keys"))
11780                ],
11781                |row| {
11782                    #[cfg(test)]
11783                    self.acknowledged_replay_count
11784                        .set(self.acknowledged_replay_count.get() + 1);
11785                    Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
11786                },
11787            )
11788            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11789        rows.map(|row| {
11790            let (op, intent) =
11791                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11792            let op: CommitOperation =
11793                serde_json::from_str(&op).map_err(|_| "sync.local_corrupt".to_owned())?;
11794            let table = self
11795                .schema
11796                .table(&op.table)
11797                .ok_or_else(|| "sync.unknown_table".to_owned())?;
11798            let values = if op.op == "upsert" && self.visible_row(table, &op.row_id)?.is_none() {
11799                Some(serde_json::from_str(&intent).map_err(|_| "sync.local_corrupt".to_owned())?)
11800            } else {
11801                op.values.clone()
11802            };
11803            Ok(OutboxOp {
11804                values,
11805                ..op.into()
11806            })
11807        })
11808        .collect()
11809    }
11810
11811    fn drop_acknowledged_rows(
11812        &self,
11813        matches: impl Fn(&str, &Map<String, Value>) -> bool,
11814    ) -> Result<Vec<OutboxCommit>, String> {
11815        let mut statement = self
11816            .conn
11817            .prepare("SELECT commit_id,tbl,intent_json,id FROM _syncular_acked_rows")
11818            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11819        let rows = statement
11820            .query_map([], |row| {
11821                Ok((
11822                    row.get::<_, String>(0)?,
11823                    row.get::<_, String>(1)?,
11824                    row.get::<_, String>(2)?,
11825                    row.get::<_, String>(3)?,
11826                ))
11827            })
11828            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11829        let mut doomed = HashSet::new();
11830        for row in rows {
11831            let (id, table, intent, row_id) =
11832                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11833            let values =
11834                serde_json::from_str(&intent).map_err(|_| "sync.local_corrupt".to_owned())?;
11835            let base_matches = self
11836                .schema
11837                .table(&table)
11838                .map(|schema| self.stored_row(schema, &row_id, true))
11839                .transpose()?
11840                .flatten()
11841                .is_some_and(|(base, _)| {
11842                    let schema = self.schema.table(&table).expect("acknowledged table");
11843                    let values = schema
11844                        .columns
11845                        .iter()
11846                        .zip(base.iter())
11847                        .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
11848                        .collect();
11849                    matches(&table, &values)
11850                });
11851            if matches(&table, &values) || base_matches {
11852                doomed.insert(id);
11853            }
11854        }
11855        let mut dropped = Vec::new();
11856        for id in doomed {
11857            let mut statement = self
11858                .conn
11859                .prepare("SELECT op_json FROM _syncular_acked_rows WHERE commit_id=?1 ORDER BY idx")
11860                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11861            let operations = statement
11862                .query_map([&id], |row| row.get::<_, String>(0))
11863                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11864            let mut ops = Vec::new();
11865            for operation in operations {
11866                let json = operation
11867                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11868                let op: CommitOperation =
11869                    serde_json::from_str(&json).map_err(|_| "sync.local_corrupt".to_owned())?;
11870                self.overlay_dirty.table(&op.table);
11871                ops.push(OutboxOp {
11872                    table: op.table,
11873                    row_id: op.row_id,
11874                    upsert: op.op == "upsert",
11875                    base_version: op.base_version,
11876                    values: op.values,
11877                    insert: op.insert,
11878                });
11879            }
11880            dropped.push(OutboxCommit {
11881                client_commit_id: id.clone(),
11882                ops,
11883            });
11884            self.conn
11885                .execute("DELETE FROM _syncular_acked_rows WHERE commit_id=?1", [id])
11886                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11887        }
11888        Ok(dropped)
11889    }
11890
11891    /// [`Self::rebuild_overlay`], skipped when neither the base tables nor
11892    /// the outbox changed since the last rebuild (the no-op sync round).
11893    fn rebuild_overlay_if_dirty(&mut self) -> Result<(), String> {
11894        if self.overlay_dirty.get() {
11895            self.rebuild_overlay()?;
11896        }
11897        Ok(())
11898    }
11899
11900    /// §7.1: local reads see outbox state applied optimistically — rebuild
11901    /// every visible table as (base server state) + (pending outbox replay
11902    /// on top). Optimistic rows carry version `-1`.
11903    fn rebuild_overlay(&mut self) -> Result<(), String> {
11904        #[cfg(feature = "bench-internals")]
11905        let _phase = self.benchmark_phases.start(Phase::OverlayRebuild);
11906        #[cfg(test)]
11907        self.overlay_rebuild_count
11908            .set(self.overlay_rebuild_count.get() + 1);
11909        let dirty = self.overlay_dirty.snapshot();
11910        let tables: Vec<_> = self
11911            .schema
11912            .tables
11913            .iter()
11914            .filter(|table| {
11915                dirty
11916                    .as_ref()
11917                    .is_none_or(|names| names.is_empty() || names.contains(&table.name))
11918            })
11919            .cloned()
11920            .collect();
11921        let names: BTreeSet<_> = tables.iter().map(|table| table.name.as_str()).collect();
11922        self.begin_observation("syncular_overlay")?;
11923        let rebuilt = (|| {
11924            for table in &tables {
11925                for index in &table.fts_indexes {
11926                    self.drop_fts_triggers(index)?;
11927                }
11928                let visible = visible_table(&table.name);
11929                let base = base_table(&table.name);
11930                self.conn
11931                    .execute_batch(&format!(
11932                        "DELETE FROM {visible}; INSERT INTO {visible} SELECT * FROM {base}"
11933                    ))
11934                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11935            }
11936            self.replay_overlay(&names, None)?;
11937            for table in &tables {
11938                for index in &table.fts_indexes {
11939                    self.rebuild_fts_projection(table, index)?;
11940                    self.create_fts_triggers(table, index)?;
11941                }
11942            }
11943            self.conn
11944                .execute_batch("RELEASE syncular_overlay")
11945                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11946            Ok(())
11947        })();
11948        if rebuilt.is_err() {
11949            self.conn
11950                .execute_batch("ROLLBACK TO syncular_overlay; RELEASE syncular_overlay")
11951                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11952        } else {
11953            self.overlay_dirty.set(false);
11954        }
11955        rebuilt
11956    }
11957
11958    /// §7.1 replay of acknowledged, retained failed and pending intent, in
11959    /// that order, for the operations of `tables` on `rows` (every row for
11960    /// `None`) over their current visible state.
11961    fn replay_overlay(
11962        &self,
11963        tables: &BTreeSet<&str>,
11964        rows: Option<&HashSet<(&str, &str)>>,
11965    ) -> Result<(), String> {
11966        let replays = |op: &&OutboxOp| {
11967            tables.contains(op.table.as_str())
11968                && rows.is_none_or(|rows| rows.contains(&(op.table.as_str(), op.row_id.as_str())))
11969        };
11970        self.apply_outbox_ops(&self.acknowledged_operations(rows, Some(tables))?)?;
11971        for (commit, initial) in self.failed_commits.iter() {
11972            for operation in commit.ops.iter().filter(replays) {
11973                let Some(table) = self.schema.table(&operation.table) else {
11974                    continue;
11975                };
11976                if operation.upsert
11977                    && self.visible_row(table, &operation.row_id)?.is_none()
11978                    && operation.presented_values(table).is_some_and(|values| {
11979                        table
11980                            .columns
11981                            .iter()
11982                            .all(|column| values.contains_key(&column.name))
11983                    })
11984                {
11985                    let key = serde_json::to_string(&(&operation.table, &operation.row_id))
11986                        .expect("row key serialization");
11987                    if let Some(Value::Object(values)) = initial.get(&key) {
11988                        self.apply_outbox_ops(&[OutboxOp {
11989                            values: Some(values.clone()),
11990                            ..operation.clone()
11991                        }])?;
11992                    }
11993                }
11994            }
11995            self.apply_outbox_ops(commit.ops.iter().filter(replays))?;
11996        }
11997        self.apply_outbox_ops(
11998            self.outbox
11999                .iter()
12000                .flat_map(|commit| &commit.ops)
12001                .filter(replays),
12002        )
12003    }
12004
12005    /// §7.1 replay: every operation is applied over the current visible
12006    /// state. An operation that no longer applies (a server row now holds its
12007    /// unique value) stays pending and invisible until a later replay admits
12008    /// it or the server answers its push.
12009    fn apply_outbox_ops<'a>(
12010        &self,
12011        ops: impl IntoIterator<Item = &'a OutboxOp>,
12012    ) -> Result<(), String> {
12013        #[cfg(feature = "bench-internals")]
12014        let mut phase = self.benchmark_phases.start(Phase::PendingReplay);
12015        for op in ops {
12016            #[cfg(feature = "bench-internals")]
12017            if let Some(phase) = &mut phase {
12018                phase.unit();
12019            }
12020            if op.upsert {
12021                if let Some((table, values)) = self
12022                    .schema
12023                    .table(&op.table)
12024                    .filter(|table| table.indexes.iter().any(|index| index.unique))
12025                    .and_then(|table| Some((table, op.presented_values(table)?)))
12026                {
12027                    let mut intended: Map<String, Value> = self
12028                        .visible_row(table, &op.row_id)?
12029                        .map(|(row, _)| {
12030                            table
12031                                .columns
12032                                .iter()
12033                                .zip(&row)
12034                                .map(|(column, value)| {
12035                                    (column.name.clone(), column_value_to_json(value))
12036                                })
12037                                .collect()
12038                        })
12039                        .unwrap_or_default();
12040                    intended.extend(values.into_owned());
12041                    if !self.unique_conflicts(table, &intended, false)?.is_empty() {
12042                        continue;
12043                    }
12044                }
12045            }
12046            self.apply_outbox_op(op)?;
12047        }
12048        Ok(())
12049    }
12050
12051    fn apply_outbox_op(&self, op: &OutboxOp) -> Result<(), String> {
12052        let Some(table) = self.schema.table(&op.table) else {
12053            return Ok(());
12054        };
12055        let visible = visible_table(&table.name);
12056        if !op.upsert {
12057            let sql = format!("DELETE FROM {visible} WHERE {}", row_id_predicate(table));
12058            self.conn
12059                .prepare_cached(&sql)
12060                .and_then(|mut statement| statement.execute(rusqlite::params![op.row_id]))
12061                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12062            return Ok(());
12063        }
12064        // §7.1: an insert-intent operation that lacks a non-nullable column
12065        // cannot encode; it stays invisible until the send-time prepass drops
12066        // it as sync.outbox_incompatible (§7.4.4).
12067        let Some(values) = op.presented_values(table) else {
12068            return Ok(());
12069        };
12070        // §7.1: the overlay applies the operation's PRESENT columns
12071        // over the current local row. A partial operation over an
12072        // absent local row leaves it absent — the server answers it
12073        // with sync.row_deleted or sync.row_missing (§6.2). An
12074        // insert-intent operation presents every column.
12075        let full = table
12076            .columns
12077            .iter()
12078            .all(|column| values.contains_key(&column.name));
12079        let local = self.visible_row(table, &op.row_id)?;
12080        if local.is_none() && !full {
12081            return Ok(());
12082        }
12083        let mut row: Row = Vec::with_capacity(table.columns.len());
12084        for (index, column) in table.columns.iter().enumerate() {
12085            let incoming = match values.get(&column.name) {
12086                Some(raw) => json_to_column_value(
12087                    if raw.get("$bytes").is_some()
12088                        && table
12089                            .encrypted_columns
12090                            .iter()
12091                            .any(|encrypted| encrypted.index == index)
12092                    {
12093                        &table.wire_columns[index]
12094                    } else {
12095                        column
12096                    },
12097                    Some(raw),
12098                ),
12099                None => Ok(local
12100                    .as_ref()
12101                    .and_then(|(values, _)| values.get(index).cloned())
12102                    .flatten()),
12103            };
12104            row.push(incoming.map_err(|_| {
12105                "sync.local_corrupt: persisted optimistic row value is invalid".to_owned()
12106            })?);
12107        }
12108        let version = local.as_ref().map_or(-1, |(_, version)| *version);
12109        self.write_row(&visible, &table.name, &row, version, None)
12110            .map(|_| ())
12111            .map_err(|error| {
12112                if error.starts_with("UNIQUE constraint failed:") {
12113                    "sync.constraint_violation: local write violates a unique constraint".to_owned()
12114                } else {
12115                    error
12116                }
12117            })
12118    }
12119
12120    /// Whether the row is present in the overlay. Existence reads only the
12121    /// key: locked encrypted values stay opaque, and decoding the complete row
12122    /// would reject a valid plain write.
12123    fn visible_row_exists(&self, table: &TableSchema, row_id: &str) -> Result<bool, String> {
12124        self.conn
12125            .query_row(
12126                &format!(
12127                    "SELECT EXISTS(SELECT 1 FROM {} WHERE {})",
12128                    visible_table(&table.name),
12129                    row_id_predicate(table)
12130                ),
12131                rusqlite::params![row_id],
12132                |row| row.get(0),
12133            )
12134            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
12135    }
12136
12137    /// §5.11: a sealed row is read-only; a write could otherwise send NULL
12138    /// over ciphertext this device cannot see.
12139    fn refuse_sealed_row(&self, table: &TableSchema, row_id: &str) -> Result<(), ClientError> {
12140        if !table.has_encrypted_columns() {
12141            return Ok(());
12142        }
12143        let key_id: Option<String> = self
12144            .conn
12145            .query_row(
12146                &format!(
12147                    "SELECT \"_sync_sealed_key\" FROM {} WHERE {}",
12148                    visible_table(&table.name),
12149                    row_id_predicate(table)
12150                ),
12151                rusqlite::params![row_id],
12152                |row| row.get(0),
12153            )
12154            .optional()
12155            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
12156            .flatten();
12157        match key_id {
12158            None => Ok(()),
12159            Some(key_id) => Err(ClientError {
12160                code: "client.encryption_key_missing".to_owned(),
12161                message: "the row is sealed under a key this device does not hold".to_owned(),
12162                details: Some(json!({ "table": table.name, "keyId": key_id })),
12163                retryable: false,
12164            }),
12165        }
12166    }
12167
12168    /// §5.11 reveal: decrypt every sealed base row whose envelope keys the
12169    /// keyring now holds into its columns and clear both reserved columns,
12170    /// one change batch per table. `client.decrypt_failed` fails the caller.
12171    fn reveal_sealed_rows(&mut self) -> Result<(), TransportError> {
12172        let storage = |message: String| TransportError::new("storage.failed", message);
12173        let tables: Vec<TableSchema> = self
12174            .schema
12175            .tables
12176            .iter()
12177            .filter(|table| table.has_encrypted_columns())
12178            .cloned()
12179            .collect();
12180        for table in &tables {
12181            let base = base_table(&table.name);
12182            let pk = quote_ident(&table.primary_key);
12183            let sealed: Vec<(SqlValue, i64, String)> = self
12184                .conn
12185                .prepare_cached(&format!(
12186                    "SELECT {pk}, \"_syncular_version\", \"_sync_sealed\" FROM {base} WHERE \"_sync_sealed_key\" IS NOT NULL"
12187                ))
12188                .and_then(|mut statement| {
12189                    statement
12190                        .query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))?
12191                        .collect()
12192                })
12193                .map_err(|error| storage(Self::sqlite_failure(&self.storage_failure, error)))?;
12194            let mut revealed = Vec::new();
12195            for (key, version, envelopes) in sealed {
12196                if let Some(row) =
12197                    crate::values::reveal_sealed_row(table, &envelopes, &self.encryption)?
12198                {
12199                    revealed.push((key, version, row));
12200                }
12201            }
12202            if revealed.is_empty() {
12203                continue;
12204            }
12205            self.begin_observation("syncular_reveal").map_err(storage)?;
12206            let outcome = (|| {
12207                let assignments = table
12208                    .encrypted_columns
12209                    .iter()
12210                    .map(|enc| format!("{}=?", quote_ident(&table.columns[enc.index].name)))
12211                    .collect::<Vec<_>>()
12212                    .join(",");
12213                let mut statement = self
12214                    .conn
12215                    .prepare_cached(&format!(
12216                        "UPDATE {base} SET {assignments},\"_sync_sealed_key\"=NULL,\"_sync_sealed\"=NULL WHERE {pk}=? AND \"_syncular_version\"=?"
12217                    ))
12218                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12219                for (key, version, row) in &revealed {
12220                    let mut params = table
12221                        .encrypted_columns
12222                        .iter()
12223                        .map(|enc| owned_sql_value(RowParam::Cell(&row[enc.index])))
12224                        .collect::<Result<Vec<_>, _>>()?;
12225                    params.push(key.clone());
12226                    params.push(SqlValue::Integer(*version));
12227                    statement
12228                        .execute(rusqlite::params_from_iter(params))
12229                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12230                }
12231                drop(statement);
12232                self.overlay_dirty.table(&table.name);
12233                self.rebuild_overlay()?;
12234                let mut batch = ChangeAccumulator::default();
12235                batch.table(&table.name);
12236                self.finish_observation("syncular_reveal", batch)
12237            })();
12238            if outcome.is_err() {
12239                self.rollback_observation("syncular_reveal");
12240            }
12241            outcome.map_err(storage)?;
12242        }
12243        Ok(())
12244    }
12245
12246    /// The visible (optimistic) row's values plus version, or `None` when the
12247    /// row is absent from the overlay.
12248    fn visible_row(&self, table: &TableSchema, row_id: &str) -> Result<Option<(Row, i64)>, String> {
12249        self.stored_row(table, row_id, false)
12250    }
12251
12252    fn stored_row(
12253        &self,
12254        table: &TableSchema,
12255        row_id: &str,
12256        base: bool,
12257    ) -> Result<Option<(Row, i64)>, String> {
12258        let sql = format!(
12259            "SELECT * FROM {} WHERE {}",
12260            if base {
12261                base_table(&table.name)
12262            } else {
12263                visible_table(&table.name)
12264            },
12265            row_id_predicate(table)
12266        );
12267        let mut stmt = self
12268            .conn
12269            .prepare_cached(&sql)
12270            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12271        let mut rows = stmt
12272            .query(rusqlite::params![row_id])
12273            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12274        let Some(row) = rows
12275            .next()
12276            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
12277        else {
12278            return Ok(None);
12279        };
12280        let mut values = Vec::with_capacity(table.columns.len());
12281        for (index, column) in table.columns.iter().enumerate() {
12282            let raw = row
12283                .get_ref(index)
12284                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12285            if let rusqlite::types::ValueRef::Blob(bytes) = raw {
12286                if table
12287                    .encrypted_columns
12288                    .iter()
12289                    .any(|encrypted| encrypted.index == index)
12290                {
12291                    values.push(Some(ColumnValue::Bytes(bytes.to_vec())));
12292                    continue;
12293                }
12294            }
12295            let json = sql_ref_to_json(column, raw);
12296            values.push(json_to_column_value(column, Some(&json)).map_err(|_| {
12297                "sync.local_corrupt: persisted local row value is invalid".to_owned()
12298            })?);
12299        }
12300        let version: i64 = row
12301            .get(table.columns.len())
12302            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12303        Ok(Some((values, version)))
12304    }
12305
12306    // -- realtime (§8) ---------------------------------------------------------------
12307
12308    /// §8.8: the configured realtime policy.
12309    #[must_use]
12310    pub fn realtime_policy(&self) -> RealtimePolicy {
12311        self.realtime_policy
12312    }
12313
12314    /// §8.8: the explicit realtime availability state.
12315    #[must_use]
12316    pub fn realtime_state(&self) -> RealtimeState {
12317        self.realtime_state
12318    }
12319
12320    /// §8.8: set the realtime policy. Refused while the socket is connected:
12321    /// a policy change never silently re-routes an ownership state.
12322    pub fn set_realtime_policy(&mut self, policy: RealtimePolicy) -> Result<(), String> {
12323        if self.realtime_state == RealtimeState::Connected && policy != self.realtime_policy {
12324            return Err(
12325                "sync.invalid_request: disconnect realtime before changing the realtime policy"
12326                    .to_owned(),
12327            );
12328        }
12329        self.realtime_policy = policy;
12330        match policy {
12331            RealtimePolicy::Off => self.set_realtime_state(RealtimeState::Disabled, None),
12332            RealtimePolicy::Required | RealtimePolicy::Optional
12333                if self.realtime_state == RealtimeState::Disabled =>
12334            {
12335                self.set_realtime_state(RealtimeState::Disconnected, None);
12336            }
12337            RealtimePolicy::Required | RealtimePolicy::Optional => {}
12338        }
12339        Ok(())
12340    }
12341
12342    /// Record an availability transition, keeping the reason code bounded and
12343    /// code-like (§7.6) and clearing it where it cannot apply.
12344    fn set_realtime_state(&mut self, state: RealtimeState, reason_code: Option<&str>) {
12345        self.realtime_state = state;
12346        self.realtime_reason_code = reason_code.map(Self::diagnostic_code);
12347        if matches!(state, RealtimeState::Connected | RealtimeState::Disabled) {
12348            self.realtime_reason_code = None;
12349            self.realtime_retry_delay_ms = None;
12350        }
12351    }
12352
12353    #[must_use]
12354    fn realtime_connected(&self) -> bool {
12355        self.realtime_state == RealtimeState::Connected
12356    }
12357
12358    pub fn connect_realtime(&mut self, transport: &mut dyn Transport) -> Result<(), String> {
12359        if !self.transport_enabled {
12360            return Err("sync.offline: transport gate is closed".into());
12361        }
12362        if self.realtime_policy == RealtimePolicy::Off {
12363            return Err(
12364                "sync.invalid_request: realtime is off; realtime connect is refused".to_owned(),
12365            );
12366        }
12367        if self.realtime_connected() {
12368            return Ok(());
12369        }
12370        self.realtime_state = RealtimeState::Connecting;
12371        match transport.realtime_connect_for_client(&self.client_id) {
12372            Ok(()) => {
12373                self.set_realtime_state(RealtimeState::Connected, None);
12374                Ok(())
12375            }
12376            Err(error) => {
12377                self.set_realtime_state(RealtimeState::Refused, Some(&error.code));
12378                Err(format!("{}: {}", error.code, error.message))
12379            }
12380        }
12381    }
12382
12383    pub fn disconnect_realtime(&mut self, transport: &mut dyn Transport) {
12384        self.cancel_sync_round();
12385        // A deliberately disconnected or policy-disabled client has nothing
12386        // to release; a `lost` socket still does (the host owns the handle).
12387        if self.realtime_state == RealtimeState::Disconnected
12388            || self.realtime_state == RealtimeState::Disabled
12389        {
12390            return;
12391        }
12392        let _ = transport.realtime_close();
12393        self.set_realtime_state(
12394            if self.realtime_policy == RealtimePolicy::Off {
12395                RealtimeState::Disabled
12396            } else {
12397                RealtimeState::Disconnected
12398            },
12399            None,
12400        );
12401        self.presence.clear(); // §8.6.1: presence is per-connection
12402    }
12403
12404    /// §8.8: the host observed the socket end without disconnecting it (a
12405    /// peer close, an I/O error, a failed send; the native transport reports
12406    /// these as `Inbound::Disconnected`). Releases the transport, moves a
12407    /// `Connected` state to `Lost` with the transport's code as the reason, and schedules
12408    /// one background retry intent so the host reconnects and catches up.
12409    pub fn realtime_lost(&mut self, transport: &mut dyn Transport, error: &TransportError) {
12410        if self.realtime_state != RealtimeState::Connected {
12411            return;
12412        }
12413        self.cancel_sync_round();
12414        let _ = transport.realtime_close();
12415        self.set_realtime_state(RealtimeState::Lost, Some(&error.code));
12416        self.realtime_retry_delay_ms = Some(self.schedule_background_retry());
12417        self.presence.clear();
12418    }
12419
12420    /// §8.6.2: publish (or clear, `doc: None`) this client's presence
12421    /// document for `scope_key`. Requires a live socket; the document is
12422    /// ephemeral (lost on disconnect). Authorization is the connection's
12423    /// registration (§8.6.3) — an unheld key is rejected loudly by the
12424    /// server with `presence.forbidden`.
12425    pub fn set_presence(
12426        &mut self,
12427        transport: &mut dyn Transport,
12428        scope_key: &str,
12429        doc: Option<&Value>,
12430    ) -> Result<(), String> {
12431        if !self.transport_enabled {
12432            return Err("sync.offline: transport gate is closed".into());
12433        }
12434        if !self.realtime_connected() {
12435            return Err("setPresence requires a connected realtime socket (§8.6)".to_string());
12436        }
12437        let text = encode_presence_publish(scope_key, doc);
12438        transport
12439            .realtime_send(&text)
12440            .map_err(|e| format!("{}: {}", e.code, e.message))
12441    }
12442
12443    /// §8.6: the peers currently present on a scope key (ephemeral).
12444    pub fn presence(&self, scope_key: &str) -> Vec<PresencePeer> {
12445        self.presence
12446            .get(scope_key)
12447            .map(|peers| peers.values().cloned().collect())
12448            .unwrap_or_default()
12449    }
12450
12451    /// §8.6 apply an inbound presence fanout to the local map.
12452    fn apply_presence(
12453        &mut self,
12454        scope_key: String,
12455        kind: Option<PresenceKind>,
12456        actor_id: Option<String>,
12457        client_id: Option<String>,
12458        doc: Option<Value>,
12459        error: Option<String>,
12460    ) {
12461        // A refused publish (§8.6.2) changes no peer; the native host surfaces
12462        // it as a `presence` event carrying the server's `error` code.
12463        if error.is_some() {
12464            return;
12465        }
12466        let (Some(kind), Some(actor_id), Some(client_id)) = (kind, actor_id, client_id) else {
12467            return;
12468        };
12469        let peer_key = format!("{actor_id} {client_id}");
12470        match kind {
12471            PresenceKind::Leave => {
12472                if let Some(peers) = self.presence.get_mut(&scope_key) {
12473                    peers.remove(&peer_key);
12474                    if peers.is_empty() {
12475                        self.presence.remove(&scope_key);
12476                    }
12477                }
12478            }
12479            _ => {
12480                let doc = match doc {
12481                    Some(Value::Object(_)) => doc.unwrap(),
12482                    _ => return,
12483                };
12484                self.presence.entry(scope_key).or_default().insert(
12485                    peer_key,
12486                    PresencePeer {
12487                        actor_id,
12488                        client_id,
12489                        doc,
12490                    },
12491                );
12492            }
12493        }
12494    }
12495
12496    /// Inbound JSON control message (§8.1). Unknown events are tolerated.
12497    pub fn on_realtime_text(&mut self, text: &str) {
12498        if self.stopped || !self.transport_enabled || self.security_preflight {
12499            return;
12500        }
12501        match parse_control(text) {
12502            Ok(ControlMessage::Hello { requires_sync, .. }) => {
12503                if requires_sync {
12504                    // §8.1: pull before trusting the socket for continuity.
12505                    self.set_sync_needed(true, true);
12506                }
12507            }
12508            Ok(ControlMessage::Presence {
12509                scope_key,
12510                kind,
12511                actor_id,
12512                client_id,
12513                doc,
12514                error,
12515                ..
12516            }) => {
12517                self.apply_presence(scope_key, kind, actor_id, client_id, doc, error);
12518            }
12519            Ok(ControlMessage::Wake { .. }) => {
12520                // §8.3: any wake-up means "run a pull soon", never data.
12521                self.set_sync_needed(true, true);
12522            }
12523            _ => {}
12524        }
12525    }
12526
12527    /// Inbound binary delta: a complete SSP2 response (§8.2), applied like
12528    /// a pull response per section; an unapplied delta is a wake-up.
12529    /// Mailbox hosts send durable acknowledgements on their I/O executor.
12530    pub fn on_realtime_binary_queued(&mut self, bytes: &[u8]) -> Vec<String> {
12531        let mut transport = crate::round::DownloadResults::default();
12532        self.on_realtime_binary(&mut transport, bytes);
12533        transport.controls
12534    }
12535
12536    /// §8.2: a delta applies exactly like a live section and sets the replica
12537    /// cursor to its `throughSeq`; one at or below the replica cursor is a
12538    /// duplicate. A delta the client does not apply (no live unit, a status
12539    /// other than `active`, a decode or apply failure) is a drop and raises
12540    /// sync-needed.
12541    pub fn on_realtime_binary(&mut self, transport: &mut dyn Transport, bytes: &[u8]) {
12542        if self.stopped || !self.transport_enabled || self.security_preflight {
12543            return;
12544        }
12545        #[cfg(feature = "bench-internals")]
12546        let decode_phase = self.benchmark_phases.start(Phase::ResponseDecode);
12547        let message = match decode_message(bytes) {
12548            Ok(m) if m.msg_kind == MsgKind::Response => m,
12549            _ => {
12550                self.set_sync_needed(true, true);
12551                return;
12552            }
12553        };
12554        #[cfg(feature = "bench-internals")]
12555        drop(decode_phase);
12556        // §8.2: a delta's header means what a pull response's header means. A
12557        // schema floor (§1.6) is recorded, stops syncing and applies nothing.
12558        if let Some(Frame::RespHeader {
12559            required_schema_version: Some(required),
12560            latest_schema_version,
12561            ..
12562        }) = message.frames.first()
12563        {
12564            let floor = SchemaFloor {
12565                required_schema_version: Some(*required),
12566                latest_schema_version: *latest_schema_version,
12567            };
12568            if self.set_schema_floor(Some(floor)).is_err() {
12569                self.set_sync_needed(true, true);
12570            }
12571            return;
12572        }
12573        let through = message.frames.iter().find_map(|frame| match frame {
12574            Frame::LiveEnd { through_seq, .. } => Some(*through_seq),
12575            _ => None,
12576        });
12577        let epoch = self.get_meta(LOG_EPOCH_KEY).ok().flatten();
12578        let mut dropped = !matches!(message.frames.first(), Some(Frame::RespHeader {
12579                reset_required: false,
12580                log_epoch,
12581                ..
12582            }) if Some(log_epoch) == epoch.as_ref());
12583        let mut applied: Option<i64> = None;
12584        dropped |= self.begin_coalesced_apply().is_err();
12585        let mut receivers: Option<Vec<UnitRecord>> = None;
12586        for frame in message.frames {
12587            if dropped {
12588                break;
12589            }
12590            match frame {
12591                Frame::LiveStart { status, .. } => {
12592                    let Ok(cursor) = self.stored(replica_cursor(&self.conn)) else {
12593                        dropped = true;
12594                        break;
12595                    };
12596                    if status != LiveStatus::Active || cursor < 0 {
12597                        dropped = true;
12598                    } else if !through.is_some_and(|through| through <= cursor) {
12599                        // A catching-up unit receives no delta (§8.2).
12600                        receivers = self.stored(load_units(&self.conn)).ok().map(|units| {
12601                            units
12602                                .into_iter()
12603                                .filter(|unit| {
12604                                    unit.status == UnitStatus::Active
12605                                        && unit.echo.is_some()
12606                                        && unit.cursor.is_none()
12607                                })
12608                                .collect()
12609                        });
12610                        dropped = receivers.is_none();
12611                    }
12612                }
12613                Frame::Commit {
12614                    commit_seq,
12615                    tables,
12616                    changes,
12617                    ..
12618                } => {
12619                    let Some(receivers) = receivers.clone() else {
12620                        continue;
12621                    };
12622                    dropped = self
12623                        .apply_live_commit(
12624                            &receivers,
12625                            &tables,
12626                            changes,
12627                            commit_seq,
12628                            AppliedServerCommitSource::Delta,
12629                        )
12630                        .is_err();
12631                }
12632                Frame::LiveEnd { through_seq, .. } if receivers.is_some() => {
12633                    dropped = self
12634                        .set_meta(REPLICA_CURSOR_KEY, &through_seq.to_string())
12635                        .is_err();
12636                    applied = Some(through_seq);
12637                }
12638                Frame::Error { .. } => dropped = true,
12639                _ => {}
12640            }
12641        }
12642        // §1.4 rule 4: the ack follows the COMMIT. A failed COMMIT restored
12643        // the replica cursor, so the delta counts as a drop.
12644        if self.finish_coalesced_apply().is_err() || dropped {
12645            applied = None;
12646            dropped = true;
12647        }
12648        if let Some(cursor) = applied {
12649            // §8.2 ack point: the delta's throughSeq.
12650            let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
12651            let _ = transport.realtime_send(&ack);
12652        }
12653        if dropped {
12654            self.set_sync_needed(true, true);
12655        }
12656    }
12657
12658    /// §8.2 ack point after a pull round on a live connection: the replica
12659    /// cursor while it is at least `0`.
12660    fn ack_after_pull(&mut self, transport: &mut dyn Transport) {
12661        if !self.realtime_connected() {
12662            return;
12663        }
12664        if let Ok(cursor @ 0..) = self.stored(replica_cursor(&self.conn)) {
12665            let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
12666            let _ = transport.realtime_send(&ack);
12667        }
12668    }
12669}
12670
12671#[cfg(test)]
12672mod previous_version_wiring_tests;
12673
12674#[cfg(test)]
12675mod identity_tests;
12676
12677#[cfg(test)]
12678mod insert_intent_tests;
12679
12680#[cfg(test)]
12681mod robustness_tests;