Skip to main content

syncular_client/
client.rs

1//! The Syncular v2 Rust client core (SPEC.md client-behavior contract):
2//! rusqlite local storage, §3.2/§3.3 effective-scope persistence + purge,
3//! §4 pull/cursor/bootstrap (§4.7 resume, §5.6 segment application), §6
4//! push with outbox order, §7 optimistic apply / rollback / replay-on-top,
5//! §2.3 clientCommitId idempotency, §8 realtime client rules, §10 errors.
6//!
7//! Built from `SPEC.md` and the committed `ssp2` codec alone — no
8//! reference to the v1 Rust tree or the v2 TypeScript client.
9
10use crate::{ProgressObserver, ProgressPhase, ProgressState};
11use ssp2::encode_message;
12use std::borrow::Cow;
13#[cfg(test)]
14use std::cell::Cell;
15use std::cell::RefCell;
16use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
17use std::sync::Arc;
18
19use rusqlite::types::{ToSqlOutput, Value as SqlValue, ValueRef};
20use rusqlite::{Connection, OpenFlags, OptionalExtension};
21use serde::{Deserialize, Serialize};
22use serde_json::{json, Map, Value};
23use sha2::{Digest, Sha256};
24use ssp2::decode::WIRE_VERSION;
25use ssp2::model::{
26    FencedRecord, Frame, LiveStatus, MediaType, Message, MsgKind, Op, OpResult, PushStatus,
27    SubStatus,
28};
29use ssp2::primitives::RawJson;
30use ssp2::registration::canonical_scope_json;
31use ssp2::segment::{decode_rows_segment, Column, ColumnType, ColumnValue, Row, RowsSegment};
32use ssp2::{decode_message, encode_presence_publish, parse_control, ControlMessage, PresenceKind};
33
34use crate::api::{
35    AppliedServerCommit, AppliedServerCommitSource, ClientChangeBatch, ClientDiagnosticsHost,
36    ClientDiagnosticsLease, ClientDiagnosticsReplica, ClientDiagnosticsSchema,
37    ClientDiagnosticsSnapshot, ClientDiagnosticsStorage, ClientError, ClientLimits, CommandEffects,
38    CommitDelivery, CommitOperation, CommitOperationOutcome, CommitOutcome, CommitOutcomeQuery,
39    CommitOutcomeResolution, CommitOutcomeStatus, ConflictRecord, CoverageSnapshot,
40    CurrentBaseProof, CurrentBaseProofReason, DiagnosticLastChange, DiagnosticLastRound,
41    DiagnosticQueryFailure, DiagnosticRoundCounters, DiagnosticUnits, EchoRecord, FetchedBlob,
42    LeaseState, LocalDataPurgeInput, LocalDataPurgeResult, LocalDataPurgeTarget,
43    LocalDataRebootstrapInput, LocalDataRebootstrapResult, Mutation, PresencePeer,
44    PreviousVersionStatus, QueryOwner, QueryReadFailure, QueryRow, QuerySnapshot, QueryValue,
45    RejectionDetails, RejectionRecord, ResolveCommitOutcomeInput, RetainedCommitRow,
46    RetainedUniqueConflict, RowState, SchemaFloor, SnapshotRead, SyncIntent, SyncOutcome,
47    SyncReport, SyncStatusSnapshot, TableChange, UnitCatchup, UnitRecord, UnitScopes, UnitStatus,
48    WindowBase, WindowChange, WindowCoverage, WindowState, WindowUnitRef,
49    CLIENT_DIAGNOSTICS_VERSION, MAX_DIAGNOSTIC_QUERY_FAILURES,
50};
51use crate::api::{RealtimePolicy, RealtimeState, REALTIME_UNAVAILABLE_CODE};
52#[cfg(feature = "bench-internals")]
53use crate::bench::{Phase, Recorder};
54use crate::previous_version::{
55    capture_previous_version_from_replica, first_incompatibility,
56    reconcile_previous_version_at_boot, set_local_schema_descriptor,
57    sweep_previous_version_container, PendingCommitAudit, PreviousVersionContextConfig,
58    PreviousVersionLifecycle, PreviousVersionReadSpec, PreviousVersionSnapshot,
59};
60use crate::schema::{parse_schema_json, ClientSchema, FtsIndexSchema, TableSchema};
61use crate::transport::{BlobDownload, SegmentRequest, Transport, TransportError};
62use crate::values::{
63    bytes_to_hex, column_value_to_json, decode_row_bytes, encode_sparse_row_json, full_row_values,
64    is_canonical_bytes_envelope, json_to_column_value, json_to_scope_map, normalize_values_casing,
65    render_row_id_json, scope_map_to_json, SealedRow,
66};
67
68/// §4.2 default: rows and SQLite images, both applied in committed chunks.
69/// Bit 3 (signed URLs, §5.4) is
70/// added per transport capability at request-build time.
71const DEFAULT_ACCEPT: u8 = 0b0111;
72const ACCEPT_INLINE_ROWS: u8 = 1 << 0;
73const ACCEPT_EXTERNAL_ROWS: u8 = 1 << 1;
74const ACCEPT_SQLITE: u8 = 1 << 2;
75const ACCEPT_SIGNED_URLS: u8 = 1 << 3;
76const MAX_DIAGNOSTIC_DOMAINS: usize = 256;
77
78/// §10.1 `recommendedAction` carried by a server HTTP error body.
79fn recommended_action_of(details: &Option<Value>) -> Option<String> {
80    details
81        .as_ref()?
82        .get("recommendedAction")?
83        .as_str()
84        .map(str::to_owned)
85}
86
87/// §7.4.1 persisted local schema-version marker (`_syncular_meta` key).
88pub(crate) const LOCAL_SCHEMA_VERSION_KEY: &str = "localSchemaVersion";
89const LOCAL_REVISION_KEY: &str = "localRevision";
90/// §4.5 replica cursor; absent means `-1`.
91const REPLICA_CURSOR_KEY: &str = "replicaCursor";
92/// Hex of the last pull `LIVE_END.echoDigest` (§4.2); absent means empty.
93const ECHO_DIGEST_KEY: &str = "echoDigest";
94/// Hex of the registration digest of the list the server stores (§4.1).
95const REGISTERED_DIGEST_KEY: &str = "registrationDigest";
96/// §7.4.3: table name → generation, persisted with the declaration evidence.
97const TABLE_GENERATIONS_KEY: &str = "localTableGenerations";
98/// §7.1: the identity of the outbox sequence.
99const OUTBOX_ID_KEY: &str = "outboxId";
100const CLIENT_ID_KEY: &str = "clientId";
101const LEASE_STATE_KEY: &str = "leaseState";
102const SCHEMA_FLOOR_KEY: &str = "schemaFloor";
103pub(crate) const LOG_EPOCH_KEY: &str = "logEpoch";
104/// Persisted fail-closed quarantine marker: present while a security preflight
105/// is pending and `activateSecurity` has yet to run. Storing it in the replica
106/// keeps the gate with the data it protects, so a rebuilt host handle reopening
107/// the same file returns in preflight even though its in-memory state is fresh.
108const SECURITY_PREFLIGHT_PENDING_KEY: &str = "securityPreflightPending";
109const LOCAL_REBOOTSTRAP_RECEIPT_VERSION: u8 = 2;
110const MAX_JS_SAFE_INTEGER: u64 = 9_007_199_254_740_991;
111/// §7.4.4 client-local code: a pending outbox commit cannot re-encode under
112/// the new schema after a bump. Never a wire code (§10.3).
113const OUTBOX_INCOMPATIBLE_CODE: &str = "sync.outbox_incompatible";
114/// §5.11 client-local code: an encode at the push seam resolved no usable key
115/// id or named an unknown key. Never a wire code (§10.3).
116const ENCRYPT_FAILED_CODE: &str = "client.encrypt_failed";
117const INVALID_REQUEST_CODE: &str = "sync.invalid_request:";
118/// §7.4.4: static text for a persisted commit the current codec refuses.
119const LEGACY_VALUE_REFUSAL_MESSAGE: &str =
120    "the persisted commit carries values the current codec refuses";
121/// §7.4.4: bounded reason on every intrinsic value drop.
122const INVALID_STORED_VALUES_REASON: &str = "invalid_stored_values";
123
124#[derive(Debug, Serialize, Deserialize)]
125#[serde(rename_all = "camelCase", deny_unknown_fields)]
126struct PersistedLocalDataRebootstrapReceipt {
127    version: u8,
128    retained_commits: u64,
129    reset_subscriptions: u64,
130}
131
132fn invalid_local_rebootstrap_receipt() -> String {
133    "sync.local_corrupt: persisted local rebootstrap receipt is invalid".to_owned()
134}
135
136fn encode_local_rebootstrap_receipt(
137    retained_commits: usize,
138    reset_subscriptions: usize,
139) -> Result<String, String> {
140    let retained_commits =
141        u64::try_from(retained_commits).map_err(|_| invalid_local_rebootstrap_receipt())?;
142    let reset_subscriptions =
143        u64::try_from(reset_subscriptions).map_err(|_| invalid_local_rebootstrap_receipt())?;
144    if retained_commits > MAX_JS_SAFE_INTEGER || reset_subscriptions > MAX_JS_SAFE_INTEGER {
145        return Err(invalid_local_rebootstrap_receipt());
146    }
147    serde_json::to_string(&PersistedLocalDataRebootstrapReceipt {
148        version: LOCAL_REBOOTSTRAP_RECEIPT_VERSION,
149        retained_commits,
150        reset_subscriptions,
151    })
152    .map_err(|_| invalid_local_rebootstrap_receipt())
153}
154
155fn decode_local_rebootstrap_receipt(value: &str) -> Result<(usize, usize), String> {
156    // Pre-0.15.36 markers proved application but did not retain the receipt.
157    if value == "v1" {
158        return Ok((0, 0));
159    }
160    let receipt: PersistedLocalDataRebootstrapReceipt =
161        serde_json::from_str(value).map_err(|_| invalid_local_rebootstrap_receipt())?;
162    if receipt.version != LOCAL_REBOOTSTRAP_RECEIPT_VERSION
163        || receipt.retained_commits > MAX_JS_SAFE_INTEGER
164        || receipt.reset_subscriptions > MAX_JS_SAFE_INTEGER
165    {
166        return Err(invalid_local_rebootstrap_receipt());
167    }
168    Ok((
169        usize::try_from(receipt.retained_commits)
170            .map_err(|_| invalid_local_rebootstrap_receipt())?,
171        usize::try_from(receipt.reset_subscriptions)
172            .map_err(|_| invalid_local_rebootstrap_receipt())?,
173    ))
174}
175
176#[cfg(test)]
177mod observation_tests;
178
179/// Internal state for hosts that perform network I/O outside the client owner.
180#[doc(hidden)]
181pub struct CurrentBasePreparation {
182    client_commit_id: String,
183    sequence: i64,
184    units: Vec<UnitRecord>,
185    transport_epoch: u64,
186    registration_epoch: u64,
187    security_epoch: u64,
188}
189
190#[cfg(test)]
191mod units_tests;
192
193#[derive(Debug, Clone)]
194struct OutboxOp {
195    upsert: bool,
196    table: String,
197    row_id: String,
198    base_version: Option<i64>,
199    /// Schema-agnostic local form (§0): driver JSON values, encoded with
200    /// the current codec at send time. A key's presence IS the §6.1 sparse
201    /// presence set — both the overlay and the wire payload derive from it.
202    values: Option<Map<String, Value>>,
203    /// §7.1 insert intent: `mutate` recorded this upsert while the row was
204    /// absent from the visible replica. Replay and send-time encoding present
205    /// every current-schema column; a column `values` lacks is NULL.
206    insert: bool,
207}
208
209impl OutboxOp {
210    /// The values this upsert presents under the current `table` (§7.1). An
211    /// insert-intent operation presents every column, a missing one as NULL;
212    /// `None` when a missing column is non-nullable, which leaves the commit
213    /// unencodable (§7.4.4), or when the operation carries no values.
214    fn presented_values(&self, table: &TableSchema) -> Option<Cow<'_, Map<String, Value>>> {
215        let values = self.values.as_ref()?;
216        if !self.insert || table.columns.iter().all(|c| values.contains_key(&c.name)) {
217            return Some(Cow::Borrowed(values));
218        }
219        full_row_values(table, values.clone()).ok().map(Cow::Owned)
220    }
221}
222
223impl From<CommitOperation> for OutboxOp {
224    fn from(operation: CommitOperation) -> Self {
225        Self {
226            upsert: operation.op == "upsert",
227            table: operation.table,
228            row_id: operation.row_id,
229            base_version: operation.base_version,
230            values: operation.values,
231            insert: operation.insert,
232        }
233    }
234}
235
236/// §7.5 static authoring refusal; the table and column travel in details.
237fn invalid_authoring(message: &str, details: Value) -> ClientError {
238    ClientError {
239        code: "sync.invalid_request".to_owned(),
240        message: message.to_owned(),
241        details: Some(details),
242        retryable: false,
243    }
244}
245
246/// §7.1 chained base: `later` repeats `earlier`'s positive base on the same
247/// row and presents only columns `earlier` presents.
248fn chains_on(earlier: &OutboxOp, later: &OutboxOp) -> bool {
249    // Only an upsert carries values, and it always carries its primary key.
250    earlier.row_id == later.row_id
251        && earlier.table == later.table
252        && later.base_version.is_some_and(|base| base > 0)
253        && earlier.base_version == later.base_version
254        && later.values.as_ref().is_some_and(|values| {
255            values.keys().all(|column| {
256                earlier
257                    .values
258                    .as_ref()
259                    .is_some_and(|earlier| earlier.contains_key(column))
260            })
261        })
262}
263
264impl From<&OutboxOp> for CommitOperation {
265    fn from(operation: &OutboxOp) -> Self {
266        Self {
267            table: operation.table.clone(),
268            row_id: operation.row_id.clone(),
269            op: if operation.upsert { "upsert" } else { "delete" }.to_owned(),
270            base_version: operation.base_version,
271            values: operation.values.clone(),
272            insert: operation.insert,
273        }
274    }
275}
276
277#[derive(Debug, Clone)]
278struct OutboxCommit {
279    client_commit_id: String,
280    ops: Vec<OutboxOp>,
281}
282
283/// One commit's `(table, value keys, insert intent)` per upsert operation — the classification
284/// shape the §7.4.4 send-time drop and the RFC 0005 D6 audit share, so the two
285/// can never disagree about which commits cannot re-encode.
286fn commit_audit_operations(commit: &OutboxCommit) -> Vec<(&str, Vec<&str>, bool)> {
287    commit
288        .ops
289        .iter()
290        .filter(|operation| operation.upsert)
291        .map(|operation| {
292            (
293                operation.table.as_str(),
294                operation
295                    .values
296                    .as_ref()
297                    .map(|values| values.keys().map(String::as_str).collect())
298                    .unwrap_or_default(),
299                operation.insert,
300            )
301        })
302        .collect()
303}
304
305type FailedCommit = (OutboxCommit, Map<String, Value>);
306
307#[derive(Debug, Clone)]
308struct CompiledLocalDataPurgeTarget {
309    table: String,
310    selectors: Vec<(String, Vec<String>)>,
311}
312
313struct StoredCommitOutcomeRow {
314    sequence: i64,
315    client_commit_id: String,
316    status: String,
317    recorded_at_ms: i64,
318    results_json: String,
319    operations_json: Option<String>,
320    resolution: String,
321    resolved_at_ms: Option<i64>,
322    replacement_client_commit_id: Option<String>,
323}
324
325impl StoredCommitOutcomeRow {
326    fn from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result<Self> {
327        Ok(Self {
328            sequence: row.get(0)?,
329            client_commit_id: row.get(1)?,
330            status: row.get(2)?,
331            recorded_at_ms: row.get(3)?,
332            results_json: row.get(4)?,
333            operations_json: row.get(5)?,
334            resolution: row.get(6)?,
335            resolved_at_ms: row.get(7)?,
336            replacement_client_commit_id: row.get(8)?,
337        })
338    }
339}
340
341/// Section outcome distinguishing the §5.6 unit-local fail-closed
342/// path from a round-aborting failure (§1.4 rule 5).
343/// §4.4 a unit section's `SUB_START`.
344struct SectionStart {
345    id: String,
346    status: SubStatus,
347    reason_code: String,
348    bootstrap: bool,
349    fenced: Vec<FencedRecord>,
350}
351
352enum SectionError {
353    FailClosed,
354    /// §5.5: the server refused a segment with `sync.forbidden`.
355    Refused,
356    /// §5.3 client memory: the round deferred this image's body to a later
357    /// pull of the same resume token.
358    Deferred,
359    Abort(String, String),
360    Transport(TransportError),
361}
362
363#[derive(Debug)]
364pub(crate) struct RequestMeta {
365    pushed_ids: Vec<String>,
366    /// Hex of the registration digest this pull carried; the server stores
367    /// that list once the round completes (§4.1).
368    registration_digest: String,
369    pub(crate) accept: u8,
370    /// §6.1 splitBatch: outbox commits held back from THIS request because
371    /// the running operation count reached the push cap — the next round
372    /// pushes them (`sync_needed` stays set while any remain).
373    deferred_commits: usize,
374    /// §5.6 applied-segment record per unit: segment id → rows applied at
375    /// the unit's current cursor and token.
376    pub(crate) applied: BTreeMap<String, BTreeMap<String, i64>>,
377    /// §5.3 client memory: (unit, segment id) of the first image whose body
378    /// the round did not fetch because the batch already holds two images.
379    /// That unit's section stops there without persisting `SUB_END`.
380    pub(crate) deferred: BTreeSet<(String, String)>,
381}
382
383#[derive(Default)]
384struct ChangeAccumulator {
385    /// None means table-wide; Some is the exact scoped domain.
386    tables: BTreeMap<String, Option<BTreeSet<String>>>,
387    windows: BTreeMap<String, BTreeSet<String>>,
388    status: bool,
389    conflicts: bool,
390    rejections: bool,
391    outcomes: bool,
392}
393
394impl ChangeAccumulator {
395    fn table(&mut self, table: &str) {
396        self.tables.insert(table.to_owned(), None);
397    }
398
399    fn scope(&mut self, table: &str, key: String) {
400        match self.tables.get_mut(table) {
401            Some(None) => {}
402            Some(Some(keys)) => {
403                keys.insert(key);
404            }
405            None => {
406                self.tables
407                    .insert(table.to_owned(), Some(BTreeSet::from([key])));
408            }
409        }
410    }
411
412    fn window(&mut self, base_key: &str, value: &str) {
413        self.windows
414            .entry(base_key.to_owned())
415            .or_default()
416            .insert(value.to_owned());
417    }
418
419    fn touched(&self) -> bool {
420        !self.tables.is_empty()
421            || !self.windows.is_empty()
422            || self.status
423            || self.conflicts
424            || self.rejections
425            || self.outcomes
426    }
427}
428
429/// §6.1: the server caps total operations per request (reference default
430/// 500) and rejects the whole batch with `sync.too_many_operations`; the
431/// client "splits and retries". Splitting happens at build time: commits are
432/// included IN ORDER until the operation budget is spent, the rest wait for
433/// the next round.
434const PUSH_OPS_PER_REQUEST: usize = 500;
435const MAX_LOCAL_PURGE_TARGETS: usize = 64;
436const MAX_LOCAL_PURGE_SELECTORS: usize = 8;
437const MAX_LOCAL_PURGE_VALUES: usize = 128;
438const MAX_LOCAL_PURGE_VALUE_LENGTH: usize = 256;
439pub const SECURITY_PREFLIGHT_REQUIRED_CODE: &str = "client.security_preflight_required";
440
441/// Dirty visible tables; `None` means a whole-replica recovery/reset.
442struct OverlayDirty(RefCell<Option<BTreeSet<String>>>);
443
444impl OverlayDirty {
445    fn clean() -> Self {
446        Self(RefCell::new(Some(BTreeSet::new())))
447    }
448    fn get(&self) -> bool {
449        self.0
450            .borrow()
451            .as_ref()
452            .is_none_or(|tables| !tables.is_empty())
453    }
454    fn set(&self, dirty: bool) {
455        *self.0.borrow_mut() = if dirty { None } else { Some(BTreeSet::new()) };
456    }
457    fn table(&self, table: &str) {
458        if let Some(tables) = self.0.borrow_mut().as_mut() {
459            tables.insert(table.to_owned());
460        }
461    }
462    fn snapshot(&self) -> Option<BTreeSet<String>> {
463        self.0.borrow().clone()
464    }
465    fn restore(&self, tables: Option<BTreeSet<String>>) {
466        *self.0.borrow_mut() = tables;
467    }
468}
469
470/// A copy-on-write in-memory mirror: a [`Rollback`] shares it, and the first
471/// write after a capture copies it once.
472#[derive(Debug, Clone, Default)]
473struct Shared<T>(Arc<T>);
474
475impl<T> std::ops::Deref for Shared<T> {
476    type Target = T;
477    fn deref(&self) -> &T {
478        &self.0
479    }
480}
481
482impl<T: Clone> std::ops::DerefMut for Shared<T> {
483    fn deref_mut(&mut self) -> &mut T {
484        Arc::make_mut(&mut self.0)
485    }
486}
487
488impl<T> From<T> for Shared<T> {
489    fn from(value: T) -> Self {
490        Self(Arc::new(value))
491    }
492}
493
494/// The in-memory mirrors of durable state, captured before a local
495/// transaction and restored when it rolls back. Queues only grow inside a
496/// transaction, so their lengths suffice.
497#[derive(Debug, Clone)]
498struct Rollback {
499    outbox: Shared<Vec<Arc<OutboxCommit>>>,
500    failed_commits: Shared<Vec<FailedCommit>>,
501    conflicts: Shared<Vec<ConflictRecord>>,
502    rejections: Shared<Vec<RejectionRecord>>,
503    schema_floor: Option<SchemaFloor>,
504    lease_state: Option<LeaseState>,
505    stopped: bool,
506    upgrading: bool,
507    sync_needed: bool,
508    active_round: Option<(uuid::Uuid, i64, u64)>,
509    overlay_dirty: Option<BTreeSet<String>>,
510    change_queue: usize,
511    sync_intent_queue: usize,
512    last_change: Option<DiagnosticLastChange>,
513}
514
515pub struct SyncClient {
516    #[cfg(feature = "bench-internals")]
517    benchmark_phases: Recorder,
518    conn: Connection,
519    schema: ClientSchema,
520    client_id: String,
521    limits: ClientLimits,
522    outbox: Shared<Vec<Arc<OutboxCommit>>>,
523    failed_commits: Shared<Vec<FailedCommit>>,
524    retain_failed_commits: bool,
525    conflicts: Shared<Vec<ConflictRecord>>,
526    rejections: Shared<Vec<RejectionRecord>>,
527    schema_floor: Option<SchemaFloor>,
528    /// §7.3.5: the opaque auth-lease state (from LEASE frames + lease errors).
529    lease_state: Option<LeaseState>,
530    /// §1.6: the schema-floor response stops syncing until an upgrade.
531    stopped: bool,
532    /// §7.4.5: true while a schema-bump reset + first re-bootstrap is in flight.
533    upgrading: bool,
534    /// §8.4 coalesced sync-needed signal.
535    sync_needed: bool,
536    /// §8.8: how sync rounds treat the realtime binding, and its exact
537    /// availability state with the evidence behind a lost or refused one.
538    realtime_policy: RealtimePolicy,
539    realtime_state: RealtimeState,
540    realtime_reason_code: Option<String>,
541    realtime_retry_delay_ms: Option<u64>,
542    /// §8.6 presence: scopeKey → (`actorId clientId` peer key → peer).
543    presence: HashMap<String, HashMap<String, PresencePeer>>,
544    /// Client clock (epoch ms) for the §5.4 `urlExpiresAtMs` check and the
545    /// §7.8 drive deadline; the host may pin it and move the pinned value
546    /// (conformance runs on a virtual clock).
547    now_ms: Option<std::sync::Arc<std::sync::atomic::AtomicI64>>,
548    /// §5.11 client-side encryption keys (`keyId → key bytes`). Empty ⇒ E2EE
549    /// off. The encrypt/decrypt seam (`values.rs`) is compiled only under the
550    /// `e2ee` feature; without it, a schema with encrypted columns fails loud.
551    encryption: crate::values::EncryptionConfig,
552    /// Fail-closed host bootstrap gate. While set, command hosts permit only
553    /// status/lifecycle inspection and an exact authorized local purge.
554    security_preflight: bool,
555    security_epoch: u64,
556    transport_enabled: bool,
557    transport_epoch: u64,
558    /// Raised by every registration change; a current-base proof spanning
559    /// one reports missing coverage.
560    registration_epoch: u64,
561    /// RFC 0005 D2/D8: the previous-version capture config. `None` (default)
562    /// means the feature is off; the descriptor write and the container orphan
563    /// sweep still happen.
564    previous_version: Option<PreviousVersionContextConfig>,
565    /// Per-table primary-key upsert SQL, built once per (full table name) —
566    /// the row write path runs per row during bootstrap (§5.6), so the SQL
567    /// string (and, via `prepare_cached`, its compiled statement) is reused
568    /// instead of being rebuilt and re-prepared per row. Cleared on a §7.4.3
569    /// schema reset (the column lists may have changed).
570    insert_sql: RefCell<HashMap<String, String>>,
571    /// §7.1: reconcile only tables whose base/outbox changed. Whole-replica
572    /// reset and restart explicitly invalidate every table.
573    overlay_dirty: OverlayDirty,
574    // First SQLite failure of the current operation, before savepoint cleanup.
575    storage_failure: RefCell<Option<QueryReadFailure>>,
576    /// Test-only structural performance signal: response processing must not
577    /// turn a batch of acknowledgements into one full overlay rebuild each.
578    #[cfg(test)]
579    overlay_rebuild_count: Cell<usize>,
580    #[cfg(test)]
581    acknowledged_replay_count: Cell<usize>,
582    /// Test-only structural performance signal: outcome retention is enforced
583    /// once per response rather than once per acknowledgement.
584    #[cfg(test)]
585    outcome_prune_count: Cell<usize>,
586    /// Test-only pinned duration of every §5.3 image chunk, in ms.
587    #[cfg(test)]
588    image_chunk_ms: Cell<Option<f64>>,
589    /// Exact observer-transaction output drained by command/FFI hosts.
590    progress: ProgressObserver,
591    /// §7.6: random per core instance, never persisted or sent.
592    epoch: String,
593    change_queue: VecDeque<ClientChangeBatch>,
594    sync_intent_queue: VecDeque<SyncIntent>,
595    /// §8.7 `scopes` events received since the last drain.
596    scopes_changed: usize,
597    /// Explicit exponential retry policy for transient transport failures.
598    retry_delay_ms: u64,
599    /// §7.7: the background retry the current round scheduled, if any.
600    round_retry_delay_ms: Option<u64>,
601    /// §5.3: a table's next image chunk size, carried from one range image
602    /// of the table to the next.
603    image_chunk_rows: HashMap<String, u32>,
604    /// A round whose response never completed (§1.5): the client keeps its
605    /// code and reports it on every following request until a round receives a
606    /// complete, decodable response. The server already recorded the round, so
607    /// nothing else can tell it the round failed on this side.
608    round_failure: Option<String>,
609    /// The in-flight round: its id, `started_at_ms` and progress attempt.
610    active_round: Option<(uuid::Uuid, i64, u64)>,
611    last_round: Option<DiagnosticLastRound>,
612    last_change: Option<DiagnosticLastChange>,
613    /// §7.6: latest failed owned snapshot read per owner id, oldest first.
614    query_failures: Vec<DiagnosticQueryFailure>,
615    /// SPEC §1.4 rule 4: the open coalesced apply transaction of a response.
616    coalesced: Option<CoalescedApply>,
617    /// A coalesced apply transaction of the current response failed; every
618    /// later observation of that response fails until it ends.
619    coalesced_failure: Option<String>,
620}
621
622/// SPEC §1.4 rule 4: the frames and blocks of one response share one local
623/// write transaction of at most `COALESCED_APPLY_ROWS` rows. The in-memory
624/// state captured at its `BEGIN` is restored when its `COMMIT` fails.
625struct CoalescedApply {
626    rows: usize,
627    rollback: Rollback,
628}
629
630const COALESCED_APPLY_ROWS: usize = 1024;
631
632pub(crate) fn quote_ident(name: &str) -> String {
633    format!("\"{}\"", name.replace('"', "\"\""))
634}
635/// §7.4.1 `_syncular_meta` helpers. Free functions so RFC 0005's
636/// previous-version module reads and writes the SAME records as the client
637/// instead of growing a parallel meta layer.
638pub(crate) fn meta_get(conn: &Connection, key: &str) -> rusqlite::Result<Option<String>> {
639    conn.query_row(
640        "SELECT value FROM _syncular_meta WHERE key = ?1",
641        rusqlite::params![key],
642        |row| row.get::<_, String>(0),
643    )
644    .optional()
645}
646
647pub(crate) fn meta_set(conn: &Connection, key: &str, value: &str) -> rusqlite::Result<()> {
648    conn.execute(
649        "INSERT OR REPLACE INTO _syncular_meta (key, value) VALUES (?1, ?2)",
650        rusqlite::params![key, value],
651    )
652    .map(|_| ())
653}
654
655pub(crate) fn meta_delete(conn: &Connection, key: &str) -> rusqlite::Result<()> {
656    conn.execute(
657        "DELETE FROM _syncular_meta WHERE key = ?1",
658        rusqlite::params![key],
659    )
660    .map(|_| ())
661}
662
663/// Seek by the existing key while retaining the exact legacy text match.
664fn row_id_predicate(table: &TableSchema) -> String {
665    let key = quote_ident(&table.primary_key);
666    let text_match = format!("CAST({key} AS TEXT) = ?1");
667    match table.columns[table.pk_index].ty {
668        ColumnType::String | ColumnType::Json => {
669            format!("{key} = ?1 AND {text_match}")
670        }
671        // Native columns have no declared SQLite affinity. Match the stored
672        // integer bind type explicitly. Unary + removes expression affinity so
673        // SQLite can seek the typeless key; the text check preserves exact IDs.
674        ColumnType::Integer | ColumnType::Boolean => {
675            format!("{key} = +CAST(?1 AS INTEGER) AND {text_match}")
676        }
677        // REAL-to-text conversion can round distinct values to the same text.
678        // Narrowing those matches by numeric equality would change behavior.
679        _ => text_match,
680    }
681}
682
683/// §3.3/§4.8/§5.6 row ownership: rows whose local scope columns match
684/// `effective` and that no entry of `owners` matches. `owners` holds the
685/// pair scopes of every owner of the table (§3.3 row ownership); a scoped delete never removes a row one of them still covers.
686/// Scope values bind by the column's declared type, as `row_id_predicate`
687/// does. `Ok(None)` matches nothing (an empty map or value list); `Err(())`
688/// is a missing local scope-column mapping (the fail-closed case).
689fn local_scope_predicate(
690    table: &TableSchema,
691    effective: &[(String, Vec<String>)],
692    owners: &[Vec<(String, Vec<String>)>],
693) -> Result<Option<(String, Vec<SqlValue>)>, ()> {
694    if effective.is_empty() || effective.iter().any(|(_, values)| values.is_empty()) {
695        return Ok(None);
696    }
697    let column = |variable: &str| {
698        let name = table.scope_column(variable).ok_or(())?;
699        let column = table
700            .columns
701            .iter()
702            .find(|column| column.name == name)
703            .ok_or(())?;
704        Ok((quote_ident(name), column.ty == ColumnType::Integer))
705    };
706    let mut clauses = Vec::new();
707    let mut params = Vec::new();
708    for (variable, values) in effective {
709        let (key, integer) = column(variable)?;
710        let holes = vec!["?"; values.len()].join(", ");
711        clauses.push(if integer {
712            params.extend(values.iter().cloned().map(SqlValue::Text));
713            let seek = vec!["+CAST(? AS INTEGER)"; values.len()].join(", ");
714            format!("{key} IN ({seek}) AND CAST({key} AS TEXT) IN ({holes})")
715        } else {
716            format!("{key} IN ({holes})")
717        });
718        params.extend(values.iter().cloned().map(SqlValue::Text));
719    }
720    // Owners bind as one JSON parameter per variable set, so the statement
721    // keeps its size at any owner count; owners of one variable bind the
722    // union of their values. An owner holding other values of a variable
723    // `effective` names owns none of its rows and drops out. Values compare
724    // as text, the exact semantics of the effective clauses.
725    let mut groups: BTreeMap<Vec<&str>, Vec<Vec<&[String]>>> = BTreeMap::new();
726    for owner in owners {
727        if owner.is_empty()
728            || owner.iter().any(|(variable, values)| {
729                values.is_empty()
730                    || effective.iter().any(|(name, wanted)| {
731                        name == variable && wanted.iter().all(|value| !values.contains(value))
732                    })
733            })
734        {
735            continue;
736        }
737        let mut owner: Vec<_> = owner.iter().collect();
738        owner.sort_by(|a, b| a.0.cmp(&b.0));
739        groups
740            .entry(
741                owner
742                    .iter()
743                    .map(|(variable, _)| variable.as_str())
744                    .collect(),
745            )
746            .or_default()
747            .push(owner.iter().map(|(_, values)| values.as_slice()).collect());
748    }
749    let within = |variable: &str, list: &str| {
750        let (key, integer) = column(variable)?;
751        let key = if integer {
752            format!("CAST({key} AS TEXT)")
753        } else {
754            key
755        };
756        Ok(format!("{key} IN (SELECT value FROM json_each({list}))"))
757    };
758    for (variables, tuples) in groups {
759        if let [variable] = variables[..] {
760            let values: BTreeSet<&String> = tuples.iter().flatten().copied().flatten().collect();
761            params.push(SqlValue::Text(
762                Value::from_iter(values.into_iter().cloned()).to_string(),
763            ));
764            clauses.push(format!("NOT {}", within(variable, "?")?));
765        } else {
766            params.push(SqlValue::Text(
767                serde_json::to_string(&tuples).expect("owner tuples serialize"),
768            ));
769            let matches = variables
770                .iter()
771                .enumerate()
772                .map(|(index, variable)| within(variable, &format!("o.value,'$[{index}]'")))
773                .collect::<Result<Vec<_>, ()>>()?;
774            clauses.push(format!(
775                "NOT EXISTS (SELECT 1 FROM json_each(?) o WHERE {})",
776                matches.join(" AND ")
777            ));
778        }
779    }
780    Ok(Some((clauses.join(" AND "), params)))
781}
782
783/// [`local_scope_predicate`] over decoded row values (retained rows).
784fn owns_local_row(
785    table: &TableSchema,
786    effective: &[(String, Vec<String>)],
787    owners: &[Vec<(String, Vec<String>)>],
788    values: &Map<String, Value>,
789) -> bool {
790    let matches = |scopes: &[(String, Vec<String>)]| {
791        !scopes.is_empty()
792            && scopes.iter().all(|(variable, allowed)| {
793                table
794                    .scope_column(variable)
795                    .and_then(|column| values.get(column))
796                    .filter(|value| !value.is_null())
797                    .is_some_and(|value| {
798                        allowed.contains(
799                            &value
800                                .as_str()
801                                .map_or_else(|| value.to_string(), str::to_owned),
802                        )
803                    })
804            })
805    };
806    matches(effective) && !owners.iter().any(|owner| matches(owner))
807}
808
809fn is_local_operation_code_like(value: &str) -> bool {
810    let bytes = value.as_bytes();
811    bytes.first().is_some_and(u8::is_ascii_alphanumeric)
812        && bytes
813            .iter()
814            .all(|byte| byte.is_ascii_alphanumeric() || matches!(*byte, b'.' | b'_' | b':' | b'-'))
815}
816
817fn base_table(name: &str) -> String {
818    quote_ident(&format!("_syncular_base_{name}"))
819}
820
821/// §4.8 deferred eviction record: (unit id, table, effective scope map).
822type PendingEvict = (String, String, Vec<(String, Vec<String>)>);
823
824/// §4.8: a stable key for a window base: variable + canonical fixed scopes.
825fn window_base_key(base: &WindowBase) -> String {
826    format!(
827        "{}\0{}",
828        base.variable,
829        ssp2::registration::canonical_unit_json(&scope_pairs(&base.fixed_scopes))
830    )
831}
832
833/// The unit of one window value: the fixed scopes plus the value.
834fn window_unit_scopes(base: &WindowBase, value: &str) -> UnitScopes {
835    let mut scopes = base.fixed_scopes.clone();
836    scopes.insert(base.variable.clone(), value.to_owned());
837    scopes
838}
839
840/// A unit's scope map as the codec and the digests take it.
841fn scope_pairs(scopes: &UnitScopes) -> Vec<(String, String)> {
842    let mut pairs: Vec<(String, String)> = scopes
843        .iter()
844        .map(|(variable, value)| (variable.clone(), value.clone()))
845        .collect();
846    pairs.sort_by(|a, b| a.0.encode_utf16().cmp(b.0.encode_utf16()));
847    pairs
848}
849
850/// §4.1: the unit covers every table declaring all of its variables.
851fn covers(table: &TableSchema, scopes: &UnitScopes) -> bool {
852    scopes
853        .keys()
854        .all(|variable| table.scope_column(variable).is_some())
855}
856
857/// §4.4: the effective scopes of the (unit, table) pair under `echo`: the
858/// scope map as one-element lists plus the fenced lists. `None` when the
859/// fence revoked the pair.
860fn pair_scopes(
861    scopes: &UnitScopes,
862    table: &str,
863    echo: Option<&[EchoRecord]>,
864) -> Option<Vec<(String, Vec<String>)>> {
865    let fenced = echo.and_then(|echo| echo.iter().find(|record| record.table == table));
866    if fenced.is_some_and(|record| record.status == "revoked") {
867        return None;
868    }
869    let mut effective: BTreeMap<String, Vec<String>> = fenced
870        .map(|record| record.scopes.clone())
871        .unwrap_or_default();
872    for (variable, value) in scopes {
873        effective.insert(variable.clone(), vec![value.clone()]);
874    }
875    Some(effective.into_iter().collect())
876}
877
878/// §3.3 row ownership: the pair scopes of every owner of `table` except
879/// `except_id`. An owner is active with a persisted echo and live or
880/// post-bootstrap, or bootstrapping with a segment of the table in its
881/// current bootstrap.
882fn unit_owners(
883    units: &[UnitRecord],
884    table: &TableSchema,
885    except_id: &str,
886) -> Vec<Vec<(String, Vec<String>)>> {
887    units
888        .iter()
889        .filter(|unit| {
890            unit.id != except_id
891                && unit.status == UnitStatus::Active
892                && unit.echo.is_some()
893                && (!unit.bootstrapping() || unit.seeded.contains(&table.name))
894                && covers(table, &unit.scopes)
895        })
896        .filter_map(|unit| pair_scopes(&unit.scopes, &table.name, unit.echo.as_deref()))
897        .collect()
898}
899
900/// §5.5 / §5.6: the unit's scope map, each value a one-element list.
901fn unit_scope_lists(scopes: &UnitScopes) -> Vec<(String, Vec<String>)> {
902    scopes
903        .iter()
904        .map(|(variable, value)| (variable.clone(), vec![value.clone()]))
905        .collect()
906}
907
908/// §7.1 acknowledged intent of the rows `?1` names (`[[table, id], ...]`).
909/// It joins through the (tbl, id) index: a scan per acknowledgement would
910/// make draining a batch quadratic.
911const ACKED_ROWS_BY_ROW: &str = "SELECT a.op_json,a.intent_json FROM json_each(?1) k JOIN _syncular_acked_rows a ON a.tbl=json_extract(k.value,'$[0]') AND a.id=json_extract(k.value,'$[1]') WHERE ?2 IS NULL OR a.tbl IN(SELECT value FROM json_each(?2)) ORDER BY a.commit_seq,a.idx";
912
913const UNIT_COLUMNS: &str =
914    "id, scopes, explicit, status, reason_code, cursor, bootstrap_state, seeded, echo";
915
916/// A unit row; a field that does not parse is a conversion failure, which
917/// both read paths classify as `sync.local_corrupt`.
918fn unit_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result<UnitRecord> {
919    fn json<T: serde::de::DeserializeOwned>(index: usize, raw: &str) -> rusqlite::Result<T> {
920        serde_json::from_str(raw).map_err(|error| {
921            rusqlite::Error::FromSqlConversionFailure(
922                index,
923                rusqlite::types::Type::Text,
924                Box::new(error),
925            )
926        })
927    }
928    Ok(UnitRecord {
929        id: row.get(0)?,
930        scopes: json(1, &row.get::<_, String>(1)?)?,
931        explicit: row.get::<_, i64>(2)? == 1,
932        status: json(
933            3,
934            &serde_json::to_string(&row.get::<_, String>(3)?).expect("text"),
935        )?,
936        reason_code: row.get(4)?,
937        cursor: row.get(5)?,
938        bootstrap_state: row.get(6)?,
939        seeded: json(7, &row.get::<_, String>(7)?)?,
940        echo: row
941            .get::<_, Option<String>>(8)?
942            .map(|echo| json::<Vec<EchoRecord>>(8, &echo))
943            .transpose()?
944            .map(|echo| {
945                if echo
946                    .iter()
947                    .all(|record| record.status == "active" || record.status == "revoked")
948                {
949                    Ok(echo)
950                } else {
951                    Err(rusqlite::Error::FromSqlConversionFailure(
952                        8,
953                        rusqlite::types::Type::Text,
954                        "echo status is neither active nor revoked".into(),
955                    ))
956                }
957            })
958            .transpose()?,
959    })
960}
961
962/// Every unit in registration order (§4.1).
963fn load_units(conn: &Connection) -> rusqlite::Result<Vec<UnitRecord>> {
964    conn.prepare_cached(&format!(
965        "SELECT {UNIT_COLUMNS} FROM _syncular_units ORDER BY rowid"
966    ))?
967    .query_map([], unit_from_row)?
968    .collect()
969}
970
971fn get_unit(conn: &Connection, id: &str) -> rusqlite::Result<Option<UnitRecord>> {
972    conn.query_row(
973        &format!("SELECT {UNIT_COLUMNS} FROM _syncular_units WHERE id = ?1"),
974        [id],
975        unit_from_row,
976    )
977    .optional()
978}
979
980/// Insert or update in place: an update keeps the registration order. The
981/// §5.6 applied-segment record ends when the cursor or the token changes.
982fn save_unit(conn: &Connection, unit: &UnitRecord) -> rusqlite::Result<()> {
983    conn.prepare_cached(&format!(
984        "INSERT INTO _syncular_units({UNIT_COLUMNS}) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9)
985         ON CONFLICT(id) DO UPDATE SET explicit=excluded.explicit, status=excluded.status,
986           reason_code=excluded.reason_code, cursor=excluded.cursor,
987           bootstrap_state=excluded.bootstrap_state, seeded=excluded.seeded, echo=excluded.echo,
988           applied=CASE WHEN cursor IS excluded.cursor
989             AND bootstrap_state IS excluded.bootstrap_state THEN applied END"
990    ))?
991    .execute(rusqlite::params![
992        unit.id,
993        serde_json::to_string(&unit.scopes).expect("scope map serializes"),
994        i64::from(unit.explicit),
995        unit.status.name(),
996        unit.reason_code,
997        unit.cursor,
998        unit.bootstrap_state,
999        serde_json::to_string(&unit.seeded).expect("table list serializes"),
1000        unit.echo
1001            .as_ref()
1002            .map(|echo| serde_json::to_string(echo).expect("echo serializes")),
1003    ])
1004    .map(|_| ())
1005}
1006
1007/// §5.6 applied-segment record of unit `id`: segment id → rows applied at
1008/// its current cursor and token.
1009fn applied_rows(conn: &Connection, id: &str) -> rusqlite::Result<BTreeMap<String, i64>> {
1010    conn.query_row(
1011        "SELECT applied FROM _syncular_units WHERE id = ?1",
1012        [id],
1013        |row| row.get::<_, Option<String>>(0),
1014    )
1015    .optional()?
1016    .flatten()
1017    .map_or(Ok(BTreeMap::new()), |raw| {
1018        serde_json::from_str(&raw).map_err(|error| {
1019            rusqlite::Error::FromSqlConversionFailure(
1020                0,
1021                rusqlite::types::Type::Text,
1022                Box::new(error),
1023            )
1024        })
1025    })
1026}
1027
1028/// §5.6: `rows` rows of `segment_id` applied at unit `id`'s position. Runs
1029/// in the transaction that commits them.
1030fn record_applied_rows(
1031    conn: &Connection,
1032    id: &str,
1033    segment_id: &str,
1034    rows: i64,
1035) -> rusqlite::Result<()> {
1036    conn.prepare_cached(
1037        "UPDATE _syncular_units SET applied=json_patch(coalesce(applied,'{}'),?1) WHERE id=?2",
1038    )?
1039    .execute(rusqlite::params![
1040        serde_json::json!({ segment_id: rows }).to_string(),
1041        id
1042    ])
1043    .map(|_| ())
1044}
1045
1046/// §4.1 references: every window registry entry `(base, value)` holding a unit.
1047fn window_refs(conn: &Connection, unit_id: &str) -> rusqlite::Result<Vec<(String, String)>> {
1048    conn.prepare_cached("SELECT base, unit FROM _syncular_windows WHERE sub_id = ?1")?
1049        .query_map([unit_id], |row| Ok((row.get(0)?, row.get(1)?)))?
1050        .collect()
1051}
1052
1053/// §7.4.3 reset, §2.1 log reset and local rebootstrap: keep the unit
1054/// registrations and discard the synced state of every unit outside `keep`,
1055/// so the next round sends the list and fresh-bootstraps those units. The
1056/// replica cursor survives only while a kept unit is live. The caller owns
1057/// the transaction.
1058fn reset_units(conn: &Connection, keep: &BTreeSet<String>) -> rusqlite::Result<()> {
1059    conn.execute(
1060        "UPDATE _syncular_units SET status='active', reason_code=NULL, cursor=-1,
1061           bootstrap_state=NULL, seeded='[]', echo=NULL, applied=NULL
1062         WHERE id NOT IN (SELECT value FROM json_each(?1))",
1063        [serde_json::to_string(keep).expect("unit ids serialize")],
1064    )?;
1065    conn.execute_batch(
1066        "DELETE FROM _syncular_meta WHERE key IN ('echoDigest','registrationDigest');
1067         DELETE FROM _syncular_meta WHERE key='replicaCursor'
1068           AND NOT EXISTS (SELECT 1 FROM _syncular_units WHERE cursor IS NULL);",
1069    )
1070}
1071
1072/// §4.5: the replica cursor, `-1` while no unit is live.
1073fn replica_cursor(conn: &Connection) -> rusqlite::Result<i64> {
1074    meta_get(conn, REPLICA_CURSOR_KEY)?.map_or(Ok(-1), |raw| {
1075        raw.parse().map_err(|error| {
1076            rusqlite::Error::FromSqlConversionFailure(
1077                0,
1078                rusqlite::types::Type::Text,
1079                Box::new(error),
1080            )
1081        })
1082    })
1083}
1084
1085fn visible_table(name: &str) -> String {
1086    quote_ident(name)
1087}
1088
1089const FTS_SOURCE_ID_COLUMN: &str = "_syncular_source_id";
1090
1091/// §7.4.3: is a `sqlite_master` table name a synced table (visible or base),
1092/// i.e. NOT one of the durable bookkeeping tables the reset preserves?
1093fn is_synced_table_name(name: &str) -> bool {
1094    if name.starts_with("sqlite_") {
1095        return false;
1096    }
1097    if name.starts_with("_syncular_base_") {
1098        return true; // the base half of a synced table pair
1099    }
1100    // Bookkeeping: outbox, units, meta, blob cache/uploads.
1101    !name.starts_with("_syncular_")
1102}
1103
1104/// `"sha256:" + hex` of the bytes — the content address (§5.9.1).
1105fn blob_id_for(bytes: &[u8]) -> String {
1106    let digest = Sha256::digest(bytes);
1107    format!("sha256:{}", bytes_to_hex(&digest))
1108}
1109
1110/// One [`SyncClient::write_row`] bind parameter, borrowing the row-codec
1111/// value it wraps — strings/JSON/bytes bind as borrowed TEXT/BLOB (no copy
1112/// per row on the §5.6 bootstrap path), scalars bind owned.
1113enum RowParam<'a> {
1114    Cell(&'a Option<ColumnValue>),
1115    Version(i64),
1116    Text(Option<&'a str>),
1117}
1118
1119impl rusqlite::ToSql for RowParam<'_> {
1120    fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
1121        Ok(match self {
1122            RowParam::Version(v) => ToSqlOutput::Owned(SqlValue::Integer(*v)),
1123            RowParam::Text(text) => match text {
1124                Some(text) => ToSqlOutput::Borrowed(ValueRef::Text(text.as_bytes())),
1125                None => ToSqlOutput::Owned(SqlValue::Null),
1126            },
1127            RowParam::Cell(cell) => match cell {
1128                None => ToSqlOutput::Owned(SqlValue::Null),
1129                Some(ColumnValue::String(s)) => ToSqlOutput::Borrowed(ValueRef::Text(s.as_bytes())),
1130                Some(ColumnValue::Integer(i)) => ToSqlOutput::Owned(SqlValue::Integer(*i)),
1131                Some(ColumnValue::Float(f)) => ToSqlOutput::Owned(SqlValue::Real(*f)),
1132                Some(ColumnValue::Boolean(b)) => {
1133                    ToSqlOutput::Owned(SqlValue::Integer(i64::from(*b)))
1134                }
1135                Some(ColumnValue::Json(raw)) | Some(ColumnValue::BlobRef(raw)) => {
1136                    ToSqlOutput::Borrowed(ValueRef::Text(raw.0.as_bytes()))
1137                }
1138                // §5.10: crdt bytes store as BLOB, like bytes.
1139                Some(ColumnValue::Bytes(b)) | Some(ColumnValue::Crdt(b)) => {
1140                    ToSqlOutput::Borrowed(ValueRef::Blob(b))
1141                }
1142            },
1143        })
1144    }
1145}
1146
1147// Retain a typed primary key across the current import chunk without copying row payloads.
1148fn owned_sql_value(param: impl rusqlite::ToSql) -> Result<SqlValue, String> {
1149    match param.to_sql().map_err(|error| error.to_string())? {
1150        ToSqlOutput::Owned(value) => Ok(value),
1151        ToSqlOutput::Borrowed(value) => {
1152            SqlValue::try_from(value).map_err(|error| error.to_string())
1153        }
1154        _ => Err("sync.invalid_request: unsupported SQLite key parameter".into()),
1155    }
1156}
1157
1158/// §5.3 image cell → bind parameter, strict per the declared column type
1159/// (`boolean` from INTEGER 0/1, `json` from its raw TEXT, NULL only when
1160/// nullable). Mismatches are image-producer violations. Returns the cell
1161/// borrowed when the stored representation already matches what the row
1162/// codec would write, or the normalized scalar (boolean → 0/1, float from
1163/// INTEGER → REAL) otherwise — the local write is byte-identical to the
1164/// old convert-then-insert path without allocating per cell.
1165fn image_cell_param<'a>(column: &Column, value: ValueRef<'a>) -> Result<ToSqlOutput<'a>, String> {
1166    use ssp2::segment::ColumnType;
1167    let mismatch = || {
1168        Err(format!(
1169            "image column {:?} holds a value of the wrong type",
1170            column.name
1171        ))
1172    };
1173    match value {
1174        ValueRef::Null => {
1175            if !column.nullable {
1176                return Err(format!(
1177                    "image column {:?} is NULL but not nullable",
1178                    column.name
1179                ));
1180            }
1181            Ok(ToSqlOutput::Owned(SqlValue::Null))
1182        }
1183        ValueRef::Integer(i) => match column.ty {
1184            ColumnType::Integer => Ok(ToSqlOutput::Borrowed(value)),
1185            ColumnType::Boolean => Ok(ToSqlOutput::Owned(SqlValue::Integer(i64::from(i != 0)))),
1186            ColumnType::Float => Ok(ToSqlOutput::Owned(SqlValue::Real(i as f64))),
1187            _ => mismatch(),
1188        },
1189        ValueRef::Real(_) => match column.ty {
1190            ColumnType::Float => Ok(ToSqlOutput::Borrowed(value)),
1191            _ => mismatch(),
1192        },
1193        ValueRef::Text(t) => {
1194            std::str::from_utf8(t)
1195                .map_err(|_| format!("image column {:?} is not UTF-8", column.name))?;
1196            match column.ty {
1197                ColumnType::String | ColumnType::Json | ColumnType::BlobRef => {
1198                    Ok(ToSqlOutput::Borrowed(value))
1199                }
1200                _ => mismatch(),
1201            }
1202        }
1203        ValueRef::Blob(_) => match column.ty {
1204            // §5.10: a crdt column stores its opaque bytes as BLOB, like bytes.
1205            ColumnType::Bytes | ColumnType::Crdt => Ok(ToSqlOutput::Borrowed(value)),
1206            _ => mismatch(),
1207        },
1208    }
1209}
1210
1211pub(crate) fn sql_ref_to_json(column: &Column, value: rusqlite::types::ValueRef<'_>) -> Value {
1212    use rusqlite::types::ValueRef;
1213    match value {
1214        ValueRef::Null => Value::Null,
1215        ValueRef::Integer(i) => match column.ty {
1216            ssp2::segment::ColumnType::Boolean => Value::Bool(i != 0),
1217            ssp2::segment::ColumnType::Float => {
1218                serde_json::Number::from_f64(i as f64).map_or(Value::Null, Value::Number)
1219            }
1220            _ => Value::from(i),
1221        },
1222        ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
1223        ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
1224        ValueRef::Blob(b) => {
1225            let mut map = Map::new();
1226            map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
1227            Value::Object(map)
1228        }
1229    }
1230}
1231
1232/// Bind a driver JSON value form as a rusqlite parameter for [`SyncClient::query`].
1233/// Objects are accepted in lossless `{"$bytes": hex}` and
1234/// `{"$bigint": decimal}` envelope forms.
1235fn json_param_to_sql(value: &Value) -> Result<SqlValue, String> {
1236    Ok(match value {
1237        Value::Null => SqlValue::Null,
1238        Value::Bool(b) => SqlValue::Integer(i64::from(*b)),
1239        Value::Number(n) => {
1240            if let Some(i) = n.as_i64() {
1241                SqlValue::Integer(i)
1242            } else if let Some(f) = n.as_f64() {
1243                SqlValue::Real(f)
1244            } else {
1245                return Err(format!("query param number {n} is out of range"));
1246            }
1247        }
1248        Value::String(s) => SqlValue::Text(s.clone()),
1249        Value::Object(_) => {
1250            if let Some(hex) = value.get("$bytes").and_then(Value::as_str) {
1251                SqlValue::Blob(crate::values::hex_to_bytes(hex)?)
1252            } else if let Some(decimal) = value.get("$bigint").and_then(Value::as_str) {
1253                SqlValue::Integer(decimal.parse::<i64>().map_err(|_| {
1254                    format!("query bigint param {decimal:?} is outside SQLite's i64 range")
1255                })?)
1256            } else {
1257                return Err(
1258                    "query object param must be a {$bytes: hex} or {$bigint: decimal} value"
1259                        .to_owned(),
1260                );
1261            }
1262        }
1263        Value::Array(_) => return Err("query array params are not supported".to_owned()),
1264    })
1265}
1266
1267/// Map a rusqlite value with no schema column to consult (arbitrary query
1268/// output): integers/reals/text pass through by stored affinity, blobs ride
1269/// as `{"$bytes": hex}`. Distinct from [`sql_ref_to_json`], which uses the
1270/// schema column type to recover booleans/floats/json.
1271fn sql_ref_to_json_dynamic(value: rusqlite::types::ValueRef<'_>) -> Value {
1272    use rusqlite::types::ValueRef;
1273    match value {
1274        ValueRef::Null => Value::Null,
1275        ValueRef::Integer(i) => {
1276            // JSON/Tauri IPC cannot represent every SQLite i64 exactly. Keep
1277            // ordinary UI-sized integers ergonomic and envelope only values
1278            // beyond JavaScript's safe range.
1279            const MAX_SAFE_INTEGER: i64 = 9_007_199_254_740_991;
1280            if (-MAX_SAFE_INTEGER..=MAX_SAFE_INTEGER).contains(&i) {
1281                Value::from(i)
1282            } else {
1283                let mut map = Map::new();
1284                map.insert("$bigint".to_owned(), Value::from(i.to_string()));
1285                Value::Object(map)
1286            }
1287        }
1288        ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
1289        ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
1290        ValueRef::Blob(b) => {
1291            let mut map = Map::new();
1292            map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
1293            Value::Object(map)
1294        }
1295    }
1296}
1297
1298/// §7.5: SQLite failures classify by result code into [`QueryReadFailure`].
1299fn query_connection(
1300    conn: &Connection,
1301    sql: &str,
1302    params: &[QueryValue],
1303) -> Result<Vec<QueryRow>, QueryReadFailure> {
1304    let lowered_sql = crate::query_guard::lower_public_query_sql(sql);
1305    let mut stmt = crate::query_guard::prepare_read_only(conn, &lowered_sql)?;
1306    let bound: Vec<SqlValue> = params
1307        .iter()
1308        .map(json_param_to_sql)
1309        .collect::<Result<_, _>>()?;
1310    // §2.4: an unaliased physical version column stays engine-internal, so a
1311    // `SELECT *` row feeds straight back into mutation values. An explicit
1312    // alias (`_sync_version AS v`) returns it as ordinary query data. The
1313    // §5.11 sealed columns follow the same rule.
1314    let columns: Vec<(usize, String)> = stmt
1315        .column_names()
1316        .enumerate()
1317        .filter(|(_, name)| {
1318            !matches!(
1319                *name,
1320                "_syncular_version" | "_sync_sealed_key" | "_sync_sealed"
1321            )
1322        })
1323        .map(|(i, name)| (i, name.to_owned()))
1324        .collect();
1325    let bound_refs: Vec<&dyn rusqlite::ToSql> =
1326        bound.iter().map(|v| v as &dyn rusqlite::ToSql).collect();
1327    let mut sql_rows = stmt.query(bound_refs.as_slice())?;
1328    let mut out = Vec::new();
1329    while let Some(row) = sql_rows.next()? {
1330        let mut record = Map::new();
1331        for (i, name) in &columns {
1332            let value = row.get_ref(*i)?;
1333            record.insert(name.clone(), sql_ref_to_json_dynamic(value));
1334        }
1335        out.push(record);
1336    }
1337    Ok(out)
1338}
1339
1340fn persisted_window_state(
1341    conn: &Connection,
1342    base: &WindowBase,
1343) -> Result<WindowState, QueryReadFailure> {
1344    let mut stmt = conn.prepare(
1345        "SELECT windows.unit, units.status, units.cursor, units.bootstrap_state
1346           FROM _syncular_windows AS windows
1347           LEFT JOIN _syncular_units AS units ON units.id = windows.sub_id
1348          WHERE windows.base = ?1
1349          ORDER BY windows.unit ASC",
1350    )?;
1351    let rows = stmt.query_map(rusqlite::params![window_base_key(base)], |row| {
1352        Ok((
1353            row.get::<_, String>(0)?,
1354            row.get::<_, Option<String>>(1)?,
1355            row.get::<_, Option<i64>>(2)?,
1356            row.get::<_, Option<String>>(3)?,
1357        ))
1358    })?;
1359    let mut units = Vec::new();
1360    let mut pending = Vec::new();
1361    for row in rows {
1362        let (unit, status, cursor, bootstrap_state) = row?;
1363        // §4.8: complete iff registered, active and not bootstrapping.
1364        if status.as_deref() != Some("active")
1365            || cursor.is_some_and(|cursor| cursor < 0 || bootstrap_state.is_some())
1366        {
1367            pending.push(unit.clone());
1368        }
1369        units.push(unit);
1370    }
1371    Ok(WindowState { units, pending })
1372}
1373
1374/// One read-only statement in a §7.5 snapshot read.
1375#[derive(Debug, Clone, Copy)]
1376pub struct SnapshotStatement<'a> {
1377    pub sql: &'a str,
1378    pub params: &'a [Value],
1379}
1380
1381/// A §7.5 atomic read request: read-only statements, window coverage, unit
1382/// catch-up states, and commit deliveries, all resolved from one SQLite read
1383/// transaction and one revision.
1384#[derive(Debug, Clone, Default)]
1385pub struct SnapshotReadRequest<'a> {
1386    pub statements: Vec<SnapshotStatement<'a>>,
1387    pub coverage: &'a [WindowCoverage],
1388    pub units: Vec<String>,
1389    pub commit_ids: Vec<String>,
1390}
1391
1392#[cfg(test)]
1393thread_local! {
1394    /// Test-only rendezvous that runs between the statements of one snapshot
1395    /// read. A test uses it to commit on another connection mid-read and prove
1396    /// the read transaction's snapshot and revision stay stable.
1397    static SNAPSHOT_READ_HOOK: std::cell::RefCell<Option<Box<dyn FnOnce()>>> =
1398        std::cell::RefCell::new(None);
1399}
1400
1401#[cfg(test)]
1402fn run_snapshot_read_hook() {
1403    SNAPSHOT_READ_HOOK.with(|slot| {
1404        if let Some(hook) = slot.borrow_mut().take() {
1405            hook();
1406        }
1407    });
1408}
1409
1410/// §7.5: resolve a request over one SQLite read transaction. The revision,
1411/// every statement's rows, coverage, catch-up states, and deliveries are
1412/// captured under one `SAVEPOINT`; any statement's failure rolls the whole
1413/// read back and releases the connection.
1414fn snapshot_read_connection(
1415    conn: &Connection,
1416    request: &SnapshotReadRequest<'_>,
1417) -> Result<SnapshotRead, QueryReadFailure> {
1418    conn.execute_batch("SAVEPOINT syncular_snapshot_read")?;
1419    let corrupt = |message: &str| QueryReadFailure {
1420        code: Some("sync.local_corrupt"),
1421        sqlite_code: None,
1422        sqlite_message: None,
1423        rollback_failure: None,
1424        message: message.to_owned(),
1425    };
1426    let stored_failure = |error: rusqlite::Error, message: &str| match error {
1427        rusqlite::Error::Utf8Error(..)
1428        | rusqlite::Error::InvalidColumnType(..)
1429        | rusqlite::Error::FromSqlConversionFailure(..)
1430        | rusqlite::Error::IntegralValueOutOfRange(..) => corrupt(message),
1431        _ => QueryReadFailure::from(error),
1432    };
1433    let result: Result<SnapshotRead, QueryReadFailure> = (|| {
1434        // The durable revision as a canonical decimal `u64`; a missing marker
1435        // is the legacy zero, and any other non-canonical value is corrupt.
1436        let revision: Option<String> = conn
1437            .query_row(
1438                "SELECT value FROM _syncular_meta WHERE key = ?1",
1439                rusqlite::params![LOCAL_REVISION_KEY],
1440                |row| row.get(0),
1441            )
1442            .optional()
1443            .map_err(|error| stored_failure(error, "persisted local revision is invalid"))?;
1444        let revision = match revision {
1445            None => "0".to_owned(),
1446            Some(value)
1447                if value == "0"
1448                    || (matches!(value.as_bytes().first(), Some(b'1'..=b'9'))
1449                        && value.bytes().all(|b| b.is_ascii_digit())
1450                        && value.parse::<u64>().is_ok()) =>
1451            {
1452                value
1453            }
1454            Some(_) => return Err(corrupt("persisted local revision is invalid")),
1455        };
1456        let mut queries = Vec::with_capacity(request.statements.len());
1457        for statement in &request.statements {
1458            queries.push(query_connection(conn, statement.sql, statement.params)?);
1459            #[cfg(test)]
1460            run_snapshot_read_hook();
1461        }
1462        let mut pending = Vec::new();
1463        let mut missing = Vec::new();
1464        for requested in request.coverage {
1465            let base_key = window_base_key(&requested.base);
1466            let state = persisted_window_state(conn, &requested.base)?;
1467            for unit in BTreeSet::from_iter(requested.units.iter().cloned()) {
1468                let reference = WindowUnitRef {
1469                    base_key: base_key.clone(),
1470                    unit: unit.clone(),
1471                };
1472                if !state.units.iter().any(|held| held == &unit) {
1473                    missing.push(reference);
1474                } else if state.pending.iter().any(|held| held == &unit) {
1475                    pending.push(reference);
1476                }
1477            }
1478        }
1479        let mut units = Vec::with_capacity(request.units.len());
1480        for id in &request.units {
1481            let exists: bool = conn.query_row(
1482                "SELECT EXISTS(SELECT 1 FROM _syncular_units WHERE id = ?1)",
1483                [id],
1484                |row| row.get(0),
1485            )?;
1486            if !exists {
1487                units.push(UnitCatchup::Unknown { id: id.clone() });
1488                continue;
1489            }
1490            // Every persisted field is required to be well typed; a corrupt
1491            // record fails typed instead of inventing a state.
1492            let unit = get_unit(conn, id)
1493                .map_err(|error| stored_failure(error, "persisted unit state is invalid"))?
1494                .ok_or_else(|| corrupt("persisted unit state is invalid"))?;
1495            let cursor = match unit.cursor {
1496                Some(cursor) => cursor,
1497                None => replica_cursor(conn)
1498                    .map_err(|error| stored_failure(error, "persisted unit state is invalid"))?,
1499            };
1500            if cursor.unsigned_abs() > MAX_JS_SAFE_INTEGER {
1501                return Err(corrupt("persisted unit state is invalid"));
1502            }
1503            let active = unit.status == UnitStatus::Active;
1504            let pending = unit.bootstrapping();
1505            units.push(UnitCatchup::Known {
1506                id: id.clone(),
1507                status: unit.status,
1508                live: unit.cursor.is_none(),
1509                cursor,
1510                has_resume_token: unit.bootstrap_state.is_some(),
1511                bootstrap_complete: active && !pending,
1512                known_pending_pages: active && pending,
1513                reason_code: unit.reason_code,
1514            });
1515        }
1516        let mut deliveries = Vec::with_capacity(request.commit_ids.len());
1517        for id in &request.commit_ids {
1518            let pending: bool = conn.query_row(
1519                "SELECT EXISTS(SELECT 1 FROM _syncular_outbox WHERE commit_id = ?1)",
1520                rusqlite::params![id],
1521                |row| row.get(0),
1522            )?;
1523            if pending {
1524                deliveries.push(CommitDelivery::Pending {
1525                    client_commit_id: id.clone(),
1526                });
1527                continue;
1528            }
1529            let row = conn
1530                .query_row(
1531                    "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
1532                            resolution, resolved_at_ms, replacement_client_commit_id
1533                       FROM _syncular_commit_outcomes WHERE client_commit_id = ?1",
1534                    rusqlite::params![id],
1535                    StoredCommitOutcomeRow::from_row,
1536                )
1537                .optional()
1538                .map_err(|error| stored_failure(error, "persisted commit outcome is invalid"))?;
1539            deliveries.push(match row {
1540                Some(row) => CommitDelivery::Known {
1541                    client_commit_id: id.clone(),
1542                    outcome: SyncClient::outcome_from_row(row)
1543                        .map_err(|_| corrupt("persisted commit outcome is invalid"))?,
1544                },
1545                None => CommitDelivery::Unknown {
1546                    client_commit_id: id.clone(),
1547                },
1548            });
1549        }
1550        Ok(SnapshotRead {
1551            revision,
1552            queries,
1553            coverage: CoverageSnapshot {
1554                complete: pending.is_empty() && missing.is_empty(),
1555                pending,
1556                missing,
1557            },
1558            units,
1559            deliveries,
1560        })
1561    })();
1562    match result {
1563        Ok(read) => {
1564            conn.execute_batch("RELEASE syncular_snapshot_read")?;
1565            Ok(read)
1566        }
1567        Err(mut error) => {
1568            if let Err(rollback) = conn
1569                .execute_batch("ROLLBACK TO syncular_snapshot_read; RELEASE syncular_snapshot_read")
1570            {
1571                error.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
1572            }
1573            Err(error)
1574        }
1575    }
1576}
1577
1578fn snapshot_connection(
1579    conn: &Connection,
1580    sql: &str,
1581    params: &[Value],
1582    coverage: &[WindowCoverage],
1583) -> Result<QuerySnapshot, QueryReadFailure> {
1584    let read = snapshot_read_connection(
1585        conn,
1586        &SnapshotReadRequest {
1587            statements: vec![SnapshotStatement { sql, params }],
1588            coverage,
1589            units: Vec::new(),
1590            commit_ids: Vec::new(),
1591        },
1592    )?;
1593    Ok(QuerySnapshot {
1594        revision: read.revision,
1595        rows: read
1596            .queries
1597            .into_iter()
1598            .next()
1599            .expect("single-query snapshot returns one result"),
1600        coverage: read.coverage,
1601    })
1602}
1603
1604/// A long-lived read-only SQLite sidecar for latency-critical native views.
1605/// Network rounds stay serialized on the mutable core owner, while atomic
1606/// query snapshots use this independent connection and therefore never queue
1607/// behind HTTP/WebSocket latency.
1608pub struct FileQuerySnapshotReader {
1609    path: String,
1610    conn: Option<Connection>,
1611}
1612
1613impl FileQuerySnapshotReader {
1614    #[must_use]
1615    pub fn new(path: impl Into<String>) -> Self {
1616        Self {
1617            path: path.into(),
1618            conn: None,
1619        }
1620    }
1621
1622    fn connection(&mut self) -> Result<&Connection, QueryReadFailure> {
1623        if self.conn.is_none() {
1624            let conn = Connection::open_with_flags(
1625                &self.path,
1626                OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
1627            )
1628            .map_err(QueryReadFailure::from)?;
1629            conn.busy_timeout(std::time::Duration::from_millis(250))
1630                .map_err(QueryReadFailure::from)?;
1631            self.conn = Some(conn);
1632        }
1633        self.conn
1634            .as_ref()
1635            .ok_or_else(|| QueryReadFailure::from("read sidecar connection missing".to_owned()))
1636    }
1637
1638    /// One atomic read over the shared primitive: every statement, coverage,
1639    /// catch-up, and delivery under one `SAVEPOINT` and one revision.
1640    pub fn snapshot_read(
1641        &mut self,
1642        request: &SnapshotReadRequest<'_>,
1643    ) -> Result<SnapshotRead, QueryReadFailure> {
1644        snapshot_read_connection(self.connection()?, request)
1645    }
1646
1647    /// The current durable local revision. A host can skip a read when it is
1648    /// unchanged; a corrupt or unreadable marker fails typed.
1649    pub fn local_revision(&mut self) -> Result<String, QueryReadFailure> {
1650        Ok(self
1651            .snapshot_read(&SnapshotReadRequest::default())?
1652            .revision)
1653    }
1654
1655    /// One unit's persisted catch-up state (see the §7.5 contract).
1656    pub fn unit_catchup(&mut self, id: &str) -> Result<UnitCatchup, QueryReadFailure> {
1657        let read = self.snapshot_read(&SnapshotReadRequest {
1658            units: vec![id.to_owned()],
1659            ..Default::default()
1660        })?;
1661        Ok(read
1662            .units
1663            .into_iter()
1664            .next()
1665            .expect("one requested catch-up"))
1666    }
1667
1668    /// One client commit's delivery status: pending, its retained outcome, or
1669    /// an explicit unknown (never held, or pruned by retention).
1670    pub fn commit_delivery(&mut self, id: &str) -> Result<CommitDelivery, QueryReadFailure> {
1671        let read = self.snapshot_read(&SnapshotReadRequest {
1672            commit_ids: vec![id.to_owned()],
1673            ..Default::default()
1674        })?;
1675        Ok(read
1676            .deliveries
1677            .into_iter()
1678            .next()
1679            .expect("one requested delivery"))
1680    }
1681
1682    /// Hosts report an owned failure, and the first success after it, to the
1683    /// owning core through [`SyncClient::record_query_read`] (§7.6).
1684    pub fn query_snapshot(
1685        &mut self,
1686        sql: &str,
1687        params: &[Value],
1688        coverage: &[WindowCoverage],
1689    ) -> Result<QuerySnapshot, QueryReadFailure> {
1690        snapshot_connection(self.connection()?, sql, params, coverage)
1691    }
1692}
1693
1694fn validate_authority_reads(
1695    schema: &ClientSchema,
1696    declarations: &[crate::api::AuthorityReadDeclaration],
1697) -> Result<(), String> {
1698    let mut seen = BTreeSet::new();
1699    for read in declarations {
1700        let table = schema
1701            .table(&read.table)
1702            .ok_or("client.authority_read_forbidden: authority table is not declared")?;
1703        let plain = |name: &str| {
1704            table.columns.iter().enumerate().any(|(i, c)| {
1705                c.name == name
1706                    && name != "authority_version"
1707                    && !name.starts_with("_sync")
1708                    && !table.encrypted_columns.iter().any(|e| e.index == i)
1709                    && matches!(
1710                        c.ty,
1711                        ssp2::segment::ColumnType::String
1712                            | ssp2::segment::ColumnType::Integer
1713                            | ssp2::segment::ColumnType::Float
1714                            | ssp2::segment::ColumnType::Boolean
1715                            | ssp2::segment::ColumnType::Json
1716                    )
1717            })
1718        };
1719        if !seen.insert(&read.table)
1720            || read.columns.is_empty()
1721            || read.columns.iter().collect::<BTreeSet<_>>().len() != read.columns.len()
1722            || !read.columns.contains(&table.primary_key)
1723            || !read.columns.iter().all(|c| plain(c))
1724            || read.scopes.is_empty()
1725            || !read.scopes.iter().all(|(v, values)| {
1726                table
1727                    .scope_column(v)
1728                    .is_some_and(|c| read.columns.iter().any(|column| column == c) && plain(c))
1729                    && !values.is_empty()
1730                    && values.iter().all(|v| !v.is_empty() && v != "*")
1731            })
1732        {
1733            return Err("client.authority_read_forbidden: authority reads require plain columns, a primary key and nonempty plain scope selectors".into());
1734        }
1735    }
1736    Ok(())
1737}
1738
1739// §7.4.2: every open and recreation checks before any persistent writes.
1740/// A marker read that hit a recognized SQLite storage failure keeps that
1741/// classification (BUSY/LOCKED stay retryable); only an unclassified read
1742/// failure is the invalid-marker `sync.local_corrupt`.
1743fn schema_marker_read_failure(error: rusqlite::Error) -> String {
1744    let failure = QueryReadFailure::from(error);
1745    if failure.code.is_some() {
1746        failure.to_string()
1747    } else {
1748        "sync.local_corrupt: persisted local schema marker is unreadable or invalid".to_owned()
1749    }
1750}
1751
1752/// §7.1: reject an invalid configured client limit before any storage is
1753/// opened or written. Zero would wedge the push queue and an out-of-range
1754/// value would differ per target; the TS core enforces the same bounds.
1755fn validate_client_limits(limits: &crate::api::ClientLimits) -> Result<(), String> {
1756    if limits.outcome_retention_max_entries == Some(0) {
1757        return Err("sync.invalid_request: outcomeRetentionMaxEntries must be positive".to_owned());
1758    }
1759    if [
1760        limits.max_push_commits_per_request,
1761        limits.max_push_operations_per_request,
1762        limits.max_push_request_bytes,
1763    ]
1764    .into_iter()
1765    .flatten()
1766    .any(|value| value == 0 || value > u32::MAX as usize)
1767    {
1768        return Err(
1769            "sync.invalid_request: push limit must be an integer in 1..=4294967295".to_owned(),
1770        );
1771    }
1772    Ok(())
1773}
1774
1775fn read_local_schema_version(conn: &Connection, requested: i32) -> Result<Option<i32>, String> {
1776    let corrupt =
1777        || "sync.local_corrupt: persisted local schema marker is unreadable or invalid".to_owned();
1778    let table_type: Option<String> = conn
1779        .query_row(
1780            "SELECT type FROM sqlite_master WHERE name = '_syncular_meta'",
1781            [],
1782            |row| row.get(0),
1783        )
1784        .optional()
1785        .map_err(schema_marker_read_failure)?;
1786    let Some(table_type) = table_type else {
1787        return Ok(None);
1788    };
1789    if table_type != "table" {
1790        return Err(corrupt());
1791    }
1792    // §7.4.2: a duplicated marker cannot be resolved. `query_row` would take
1793    // whichever row SQLite returns first, and an older one would drive a
1794    // destructive reset over a newer replica.
1795    let mut statement = conn
1796        .prepare("SELECT value FROM _syncular_meta WHERE key = ?1")
1797        .map_err(schema_marker_read_failure)?;
1798    let mut rows = statement
1799        .query([LOCAL_SCHEMA_VERSION_KEY])
1800        .map_err(schema_marker_read_failure)?;
1801    let value: Option<String> = rows
1802        .next()
1803        .map_err(schema_marker_read_failure)?
1804        .map(|row| row.get(0))
1805        .transpose()
1806        .map_err(schema_marker_read_failure)?;
1807    if rows.next().map_err(schema_marker_read_failure)?.is_some() {
1808        return Err(corrupt());
1809    }
1810    let Some(value) = value else {
1811        let has_descriptor = conn
1812            .query_row(
1813                "SELECT 1 FROM _syncular_meta WHERE key = ?1 LIMIT 1",
1814                [crate::previous_version::LOCAL_SCHEMA_DESCRIPTOR_KEY],
1815                |_| Ok(()),
1816            )
1817            .optional()
1818            .map_err(schema_marker_read_failure)?
1819            .is_some();
1820        if has_descriptor {
1821            return Err(corrupt());
1822        }
1823        return Ok(None);
1824    };
1825    if !matches!(value.as_bytes().first(), Some(b'1'..=b'9'))
1826        || !value.bytes().all(|b| b.is_ascii_digit())
1827    {
1828        return Err(corrupt());
1829    }
1830    let version = value.parse::<i32>().map_err(|_| corrupt())?;
1831    if version > requested {
1832        return Err(
1833            "client.schema_downgrade: persisted local schema is newer than the requested schema"
1834                .to_owned(),
1835        );
1836    }
1837    Ok(Some(version))
1838}
1839
1840/// Open a replica file. On a browser worker the file must recover a hot
1841/// journal before any statement reads it: `wasm_vfs` replaces the installed
1842/// OPFS VFS's reserved lock callback before the connection opens its file. A
1843/// new browser replica uses 64 KiB pages (§5.3), because every OPFS call costs
1844/// the same whatever its size; a database with tables keeps its page size. Its
1845/// page cache holds 64 MiB, because SQLite's 2,000 KiB default is 31 such pages
1846/// and every cache miss is an OPFS call.
1847fn open_replica(path: &str) -> Result<Connection, String> {
1848    let browser = cfg!(all(target_arch = "wasm32", target_os = "unknown"))
1849        && !path.is_empty()
1850        && path != ":memory:";
1851    #[cfg(all(target_arch = "wasm32", target_os = "unknown"))]
1852    if browser {
1853        crate::wasm_vfs::disable_foreign_lock_check()?;
1854    }
1855    let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
1856    if browser {
1857        conn.execute_batch("PRAGMA page_size = 65536; PRAGMA cache_size = -65536")
1858            .map_err(|e| format!("open db {path:?}: {e}"))?;
1859    }
1860    Ok(conn)
1861}
1862
1863impl SyncClient {
1864    pub fn new_with_identity(
1865        client_id: Option<String>,
1866        schema_json: &Value,
1867        limits: ClientLimits,
1868    ) -> Result<Self, String> {
1869        let resolved = client_id.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
1870        let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
1871        Self::with_connection(resolved, schema_json, limits, conn)
1872    }
1873
1874    pub fn new(
1875        client_id: String,
1876        schema_json: &Value,
1877        limits: ClientLimits,
1878    ) -> Result<Self, String> {
1879        let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
1880        Self::with_connection(client_id, schema_json, limits, conn)
1881    }
1882
1883    /// Build a client backed by an on-disk SQLite database at `path` — the
1884    /// seam a native host (Tauri plugin, FFI file-DB variant) uses to persist
1885    /// across process restarts. `create_tables` runs `IF NOT EXISTS`, so
1886    /// re-opening the same file reuses the persisted rows. Keeps rusqlite out
1887    /// of the command router's dependency set (the router only holds a path).
1888    pub fn open_path(
1889        client_id: String,
1890        schema_json: &Value,
1891        limits: ClientLimits,
1892        path: &str,
1893    ) -> Result<Self, String> {
1894        validate_client_limits(&limits)?;
1895        parse_schema_json(schema_json)?;
1896        let conn = open_replica(path)?;
1897        Self::with_connection(client_id, schema_json, limits, conn)
1898    }
1899
1900    /// Open a file replica and resolve its client id. Native targets require
1901    /// WAL for the independent snapshot reader; `wasm32-unknown-unknown`
1902    /// accepts the sqlite-wasm worker journal instead.
1903    pub fn open_path_with_identity(
1904        client_id: Option<String>,
1905        schema_json: &Value,
1906        limits: ClientLimits,
1907        path: &str,
1908    ) -> Result<Self, String> {
1909        Self::open_path_with_identity_journal(
1910            client_id,
1911            schema_json,
1912            limits,
1913            path,
1914            !cfg!(all(target_arch = "wasm32", target_os = "unknown")),
1915        )
1916    }
1917
1918    fn open_path_with_identity_journal(
1919        client_id: Option<String>,
1920        schema_json: &Value,
1921        limits: ClientLimits,
1922        path: &str,
1923        require_wal: bool,
1924    ) -> Result<Self, String> {
1925        validate_client_limits(&limits)?;
1926        // §7.4.2: an invalid requested version is refused before the replica is
1927        // even created, so a rejected constructor leaves no file behind.
1928        parse_schema_json(schema_json)?;
1929        let conn = open_replica(path)?;
1930        // A file-backed native client uses an independent read connection for
1931        // latency-critical snapshots. WAL is SQLite's intended reader/writer
1932        // concurrency mode: a view read never holds a rollback-journal lock
1933        // that delays the client's next commit. The busy timeout must precede
1934        // this replica's own startup transaction; WAL is applied only after
1935        // startup succeeded, because a refused open must leave the replica's
1936        // journal mode and contents untouched (§7.4.2).
1937        conn.busy_timeout(std::time::Duration::from_millis(250))
1938            .map_err(|error| format!("configure db {path:?} busy timeout: {error}"))?;
1939        // Durability is the caller's contract here, not the VFS default: a
1940        // rollback-journal commit needs FULL before it writes the database
1941        // pages, or a killed worker can leave a half-written image behind.
1942        // WAL below inherits the same setting.
1943        conn.pragma_update(None, "synchronous", "FULL")
1944            .map_err(|error| format!("configure db {path:?} synchronous: {error}"))?;
1945        let client = Self::with_connection_identity(client_id, schema_json, limits, conn)?;
1946        // A browser worker's sqlite-wasm VFS has no WAL mode, and the wasm
1947        // host has no independent snapshot reader. The WAL requirement exists
1948        // for native file replicas, where it is the reader/writer boundary.
1949        if require_wal {
1950            let journal_mode: String = client
1951                .conn
1952                .pragma_update_and_check(None, "journal_mode", "WAL", |row| row.get(0))
1953                .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
1954            if journal_mode != "wal" {
1955                return Err(
1956                    "sync.invalid_request: file-backed client requires WAL journal mode".to_owned(),
1957                );
1958            }
1959        }
1960        Ok(client)
1961    }
1962
1963    /// Delete a file replica at `path`, including its `-journal`, `-wal`, and
1964    /// `-shm` sidecars, through the VFS that owns it. Use this after
1965    /// `client.storage_corrupt`: an OPFS SAH pool stores those files under
1966    /// opaque names, so deleting the pool directory behind SQLite bypasses the
1967    /// pool's slot accounting.
1968    ///
1969    /// Close the client and every [`FileQuerySnapshotReader`] on `path` first;
1970    /// a VFS cannot delete a file another handle still owns. A file that does
1971    /// not exist counts as deleted, so a cleanly closed replica without
1972    /// sidecars resets. The next open
1973    /// creates a fresh replica (new client id) and re-bootstraps from the
1974    /// server. A memory replica has nothing to delete and fails with
1975    /// `sync.invalid_request`.
1976    pub fn reset_replica(path: &str) -> Result<(), String> {
1977        if path.is_empty() || path == ":memory:" {
1978            return Err("sync.invalid_request: a memory replica has nothing to reset".to_owned());
1979        }
1980        // SAFETY: a null VFS name asks SQLite for its process-wide default VFS,
1981        // the same one `Connection::open(path)` above selects. `xDelete` is
1982        // that VFS's own function pointer, and `c_name` outlives the call.
1983        let vfs = unsafe { rusqlite::ffi::sqlite3_vfs_find(std::ptr::null()) };
1984        if vfs.is_null() {
1985            return Err("client.storage_io: no SQLite VFS is registered".to_owned());
1986        }
1987        let delete = unsafe { (*vfs).xDelete }
1988            .ok_or_else(|| "client.storage_io: the SQLite VFS cannot delete files".to_owned())?;
1989        for suffix in ["-wal", "-shm", "-journal", ""] {
1990            let file_name = format!("{path}{suffix}");
1991            let c_name = std::ffi::CString::new(file_name.as_str()).map_err(|_| {
1992                format!("sync.invalid_request: replica path {file_name:?} contains a NUL byte")
1993            })?;
1994            // SAFETY: `vfs` and its `xDelete` pointer come from the live
1995            // SQLite library; `c_name` is a valid NUL-terminated path.
1996            let code = unsafe { delete(vfs, c_name.as_ptr(), 1) };
1997            // A cleanly closed WAL replica has no sidecars; a missing file is
1998            // already deleted.
1999            if code != rusqlite::ffi::SQLITE_OK && code != rusqlite::ffi::SQLITE_IOERR_DELETE_NOENT
2000            {
2001                return Err(format!(
2002                    "client.storage_io: deleting replica file {file_name:?} failed with SQLite code {code}"
2003                ));
2004            }
2005        }
2006        Ok(())
2007    }
2008
2009    #[must_use]
2010    pub fn client_id(&self) -> &str {
2011        &self.client_id
2012    }
2013
2014    /// Rotate the caller's client id (§1.5) and return the new one. Use it
2015    /// after a `sync.invalid_client_id` refusal (`recommendedAction:
2016    /// resetClientId`): the next round carries the new id. A live realtime
2017    /// socket is closed so the next connection registers with the new id.
2018    /// The replica and its outbox stay in place; wipe them yourself when the
2019    /// stored data belongs to the account that signed out.
2020    pub fn reset_client_id(&mut self, transport: &mut dyn Transport) -> Result<String, String> {
2021        self.disconnect_realtime(transport);
2022        let client_id = uuid::Uuid::new_v4().to_string();
2023        self.set_meta(CLIENT_ID_KEY, &client_id)?;
2024        self.client_id.clone_from(&client_id);
2025        self.round_failure = None;
2026        Ok(client_id)
2027    }
2028
2029    /// Build a client over a caller-supplied rusqlite connection — the seam a
2030    /// native host (Tauri plugin, FFI file-DB variant) uses to back the core
2031    /// with an on-disk database (`Connection::open(path)`) rather than the
2032    /// default `:memory:`. The connection MUST be fresh (no pre-existing
2033    /// syncular tables); `create_tables` runs `IF NOT EXISTS`, so re-opening
2034    /// the same file across process restarts reuses the persisted rows.
2035    pub fn with_connection(
2036        client_id: String,
2037        schema_json: &Value,
2038        limits: ClientLimits,
2039        conn: Connection,
2040    ) -> Result<Self, String> {
2041        Self::with_connection_identity(Some(client_id), schema_json, limits, conn)
2042    }
2043
2044    /// [`Self::with_connection`] for `open_path_with_identity`: no requested id
2045    /// means a durable replica's persisted `clientId` wins over a fresh one.
2046    fn with_connection_identity(
2047        client_id: Option<String>,
2048        schema_json: &Value,
2049        limits: ClientLimits,
2050        conn: Connection,
2051    ) -> Result<Self, String> {
2052        validate_client_limits(&limits)?;
2053        let schema = parse_schema_json(schema_json)?;
2054        validate_authority_reads(&schema, &limits.authority_reads)?;
2055        // RFC 0005 D8: resolve the previous-version config before the opening
2056        // reset runs — the capture happens inside it. Bad bounds fail loud.
2057        let previous_version = match limits.previous_version_context {
2058            Some(config) => {
2059                config.validate()?;
2060                Some(config)
2061            }
2062            None => None,
2063        };
2064        let mut client = SyncClient {
2065            #[cfg(feature = "bench-internals")]
2066            benchmark_phases: Recorder::default(),
2067            conn,
2068            schema,
2069            client_id: client_id
2070                .clone()
2071                .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()),
2072            limits,
2073            outbox: Shared::default(),
2074            failed_commits: Shared::default(),
2075            retain_failed_commits: false,
2076            conflicts: Shared::default(),
2077            rejections: Shared::default(),
2078            schema_floor: None,
2079            lease_state: None,
2080            stopped: false,
2081            upgrading: false,
2082            sync_needed: false,
2083            realtime_policy: RealtimePolicy::Optional,
2084            realtime_state: RealtimeState::Disconnected,
2085            realtime_reason_code: None,
2086            realtime_retry_delay_ms: None,
2087            presence: HashMap::new(),
2088            now_ms: None,
2089            encryption: crate::values::EncryptionConfig::default(),
2090            security_preflight: false,
2091            security_epoch: 0,
2092            transport_enabled: true,
2093            transport_epoch: 0,
2094            registration_epoch: 0,
2095            previous_version,
2096            insert_sql: RefCell::new(HashMap::new()),
2097            overlay_dirty: OverlayDirty::clean(),
2098            storage_failure: RefCell::new(None),
2099            #[cfg(test)]
2100            overlay_rebuild_count: Cell::new(0),
2101            #[cfg(test)]
2102            acknowledged_replay_count: Cell::new(0),
2103            #[cfg(test)]
2104            outcome_prune_count: Cell::new(0),
2105            #[cfg(test)]
2106            image_chunk_ms: Cell::new(None),
2107            progress: ProgressObserver::default(),
2108            epoch: uuid::Uuid::new_v4().to_string(),
2109            change_queue: VecDeque::new(),
2110            sync_intent_queue: VecDeque::new(),
2111            scopes_changed: 0,
2112            retry_delay_ms: 250,
2113            round_retry_delay_ms: None,
2114            image_chunk_rows: HashMap::new(),
2115            round_failure: None,
2116            active_round: None,
2117            last_round: None,
2118            last_change: None,
2119            query_failures: Vec::new(),
2120            coalesced: None,
2121            coalesced_failure: None,
2122        };
2123        // The row write path leans on the prepared-statement cache (two
2124        // insert statements per synced table, plus the bookkeeping
2125        // statements); size it so a multi-table schema never thrashes.
2126        client
2127            .conn
2128            .set_prepared_statement_cache_capacity(64.max(client.schema.tables.len() * 4));
2129        // §7.4.2: one write transaction (SQLite `BEGIN IMMEDIATE`, a reserved
2130        // writer) covers the marker read, the identity check, and every
2131        // bookkeeping/schema write, so a concurrent open that upgraded the
2132        // replica cannot interleave between the guard and the writes, and a
2133        // refusal leaves the replica unchanged.
2134        client
2135            .conn
2136            .execute_batch("BEGIN IMMEDIATE")
2137            .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
2138        let startup = (|| -> Result<(), String> {
2139            // The marker guard runs inside the write transaction, so the version
2140            // it validates is the version every write below commits under.
2141            let marker = read_local_schema_version(&client.conn, client.schema.version)?;
2142            // New app indexes may reference columns that only exist after a
2143            // version-bump reset.
2144            client.create_bookkeeping_tables()?;
2145            match client.get_meta(CLIENT_ID_KEY)? {
2146                Some(existing) => match client_id {
2147                    Some(requested) if existing != requested => {
2148                        return Err(format!(
2149                            "client.identity_mismatch: this database belongs to {existing:?}; refusing to rebind it to {requested:?}"
2150                        ))
2151                    }
2152                    _ => client.client_id = existing,
2153                },
2154                None => client.set_meta(CLIENT_ID_KEY, &client.client_id)?,
2155            }
2156            client.restore_persisted_state()?;
2157            client.failed_commits = client.load_failed_commits()?.into();
2158            match marker {
2159                None => {
2160                    client.create_synced_tables()?;
2161                    client.persist_schema_version()?;
2162                    client.save_unit_scope_schema()?;
2163                }
2164                Some(version) if version == client.schema.version => {
2165                    client.create_synced_tables()?;
2166                    // RFC 0005 D1: the same-version open backfills the descriptor
2167                    // for a database first opened by an unaware binary, so the
2168                    // NEXT bump can capture without needing a schema bump first.
2169                    set_local_schema_descriptor(&client.conn, &client.schema)
2170                        .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
2171                    client.save_unit_scope_schema()?;
2172                }
2173                Some(_) => client.run_schema_reset()?,
2174            }
2175            // RFC 0005 D9: the reset sweep cannot see a container whose capture
2176            // committed in its OWN file while the replica's savepoint never
2177            // released — a crash at that seam leaves a container beside a matching
2178            // OLD marker, and the next open at that version runs no reset. Discard
2179            // it before anything can read it. No-op when no container exists.
2180            client.reconcile_previous_version_at_boot()?;
2181            client.clear_satisfied_persisted_schema_floor()?;
2182            client.prune_unknown_units(false)?;
2183            // §7.1/§7.4.4: recover intrinsically codec-invalid persisted values at
2184            // the full startup/reset reconciliation boundary, inside this open
2185            // transaction and after the schema and identity guards, so a commit
2186            // an earlier version accepted is dropped with durable rejection
2187            // evidence before any replay. A table the current schema removed
2188            // keeps the §7.4.4 send-time classification, and an unresolved
2189            // encryption key stays with the send seam because hosts configure
2190            // keys after open.
2191            client.drop_unencodable_outbox(false, None)?;
2192            if marker == Some(client.schema.version)
2193                && (!client.outbox.is_empty()
2194                    || !client.failed_commits.is_empty()
2195                    || client.has_acknowledged_rows()?)
2196            {
2197                // Reconstruct the visible optimistic overlay from the durable
2198                // base plus outbox instead of trusting a process-interrupted
2199                // mirror.
2200                client.overlay_dirty.set(true);
2201                client.rebuild_overlay()?;
2202            }
2203            // Every persisted active unit needs one catch-up pull on
2204            // open: realtime only covers changes after connection, while an
2205            // idempotent setWindow correctly creates no fresh command effect.
2206            // Pending outbox work has the same restart requirement. The core
2207            // owns this intent so native hosts never poll or require an
2208            // application-issued sync().
2209            client.enqueue_startup_sync_if_needed()?;
2210            Ok(())
2211        })();
2212        match startup {
2213            Ok(()) => match client.conn.execute_batch("COMMIT") {
2214                // §4.8: an eviction a crash interrupted finishes before any
2215                // read, with or without a transport; a replica reopened in
2216                // preflight finishes it at activation.
2217                Ok(()) => {
2218                    if !client.security_preflight {
2219                        client.drain_pending_evictions()?;
2220                    }
2221                    Ok(client)
2222                }
2223                Err(error) => {
2224                    let failure = Self::sqlite_failure(&client.storage_failure, error);
2225                    if let Err(rollback) = client.conn.execute_batch("ROLLBACK") {
2226                        // Keep a failed rollback's typed storage classification.
2227                        let _ = Self::sqlite_failure(&client.storage_failure, rollback);
2228                    }
2229                    Err(failure)
2230                }
2231            },
2232            Err(error) => {
2233                if let Err(rollback) = client.conn.execute_batch("ROLLBACK") {
2234                    // Keep a failed rollback's typed storage classification.
2235                    let _ = Self::sqlite_failure(&client.storage_failure, rollback);
2236                }
2237                Err(error)
2238            }
2239        }
2240    }
2241
2242    /// Pin the client clock (epoch ms) — the §5.4 expiry check runs
2243    /// against this instead of system time (conformance virtual clock).
2244    pub fn set_now_ms(&mut self, now_ms: i64) {
2245        self.now_ms = Some(std::sync::Arc::new(now_ms.into()));
2246    }
2247
2248    /// The pinned clock, shared: a host that stores into it moves the client
2249    /// clock while a call runs (a virtual clock that advances per request).
2250    pub fn pinned_clock(&self) -> Option<std::sync::Arc<std::sync::atomic::AtomicI64>> {
2251        self.now_ms.clone()
2252    }
2253
2254    /// §5.11: install the client-side encryption keys (`keyId → key bytes`).
2255    /// The command router parses these from the `create` command's
2256    /// `encryption` config (keys as `{$bytes: hex}`).
2257    pub fn set_encryption(&mut self, encryption: crate::values::EncryptionConfig) {
2258        self.encryption = encryption;
2259    }
2260
2261    #[must_use]
2262    pub fn security_lifecycle(&self) -> &'static str {
2263        if self.security_preflight {
2264            "preflight"
2265        } else {
2266            "active"
2267        }
2268    }
2269
2270    #[must_use]
2271    pub fn security_preflight(&self) -> bool {
2272        self.security_preflight
2273    }
2274
2275    /// Host-owned, non-persisted network gate. Local operations remain available.
2276    pub fn set_transport_enabled(&mut self, transport: &mut dyn Transport, enabled: bool) {
2277        if self.transport_enabled != enabled {
2278            self.transport_enabled = enabled;
2279            self.transport_epoch = self.transport_epoch.wrapping_add(1);
2280            self.sync_intent_queue.clear();
2281            self.sync_intent_queue
2282                .push_back(if enabled && !self.security_preflight {
2283                    SyncIntent::Interactive
2284                } else {
2285                    SyncIntent::None
2286                });
2287        }
2288        // A captured reply keeps its normal apply/revocation semantics.
2289        if !enabled && self.active_round.is_none() {
2290            self.disconnect_realtime(transport);
2291        }
2292    }
2293
2294    pub fn transport_enabled(&self) -> bool {
2295        self.transport_enabled
2296    }
2297
2298    /// Quarantine this replica: enter the fail-closed gate, release all
2299    /// core-owned key material, and record the gate in the database so
2300    /// reopening re-enters preflight until `activate_security` clears it.
2301    ///
2302    /// Use [`SyncClient::seal_security_on_teardown`] for the shutdown barrier,
2303    /// which must NOT leave that durable mark.
2304    pub fn begin_security_preflight(&mut self) -> Result<(), String> {
2305        let entering = !self.security_preflight;
2306        self.seal_security_on_teardown();
2307        // Persist the quarantine so it survives handle teardown and restart:
2308        // reopening this replica re-enters preflight until activation clears it.
2309        self.set_meta(SECURITY_PREFLIGHT_PENDING_KEY, "1")?;
2310        if entering {
2311            self.emit_status_batch()?;
2312        }
2313        Ok(())
2314    }
2315
2316    /// One change batch carrying the status snapshot (§5.11 lifecycle).
2317    fn emit_status_batch(&mut self) -> Result<(), String> {
2318        self.begin_observation("syncular_status")?;
2319        let batch = ChangeAccumulator {
2320            status: true,
2321            ..ChangeAccumulator::default()
2322        };
2323        self.finish_observation("syncular_status", batch)
2324            .inspect_err(|_| self.rollback_observation("syncular_status"))
2325    }
2326
2327    /// Release core-owned key material as a host tears the client down,
2328    /// WITHOUT recording a quarantine.
2329    ///
2330    /// Shutting an activated client down is not a quarantine event. Persisting
2331    /// the gate here marks every cleanly closed replica as pending, and the
2332    /// next plain `create` is then refused permanently: the reopen path
2333    /// restores the flag from the marker and the create guard rejects it. The
2334    /// in-memory flag still closes the gate for anything still holding this
2335    /// instance, which is all a teardown barrier needs.
2336    pub fn seal_security_on_teardown(&mut self) {
2337        self.cancel_sync_round();
2338        self.security_preflight = true;
2339        self.security_epoch = self.security_epoch.wrapping_add(1);
2340        self.encryption = crate::values::EncryptionConfig::default();
2341        self.sync_intent_queue.clear();
2342    }
2343
2344    /// Install the post-authentication keyring and release the host loop.
2345    pub fn activate_security(
2346        &mut self,
2347        encryption: crate::values::EncryptionConfig,
2348    ) -> Result<(), String> {
2349        if !self.security_preflight {
2350            return Err(
2351                "sync.invalid_request: activateSecurity requires security preflight".to_owned(),
2352            );
2353        }
2354        // §4.8: an interrupted eviction finishes before protected reads open.
2355        self.drain_pending_evictions()?;
2356        self.enqueue_startup_sync_if_needed()?;
2357        self.delete_meta(SECURITY_PREFLIGHT_PENDING_KEY)?;
2358        self.encryption = encryption;
2359        self.security_preflight = false;
2360        // §5.11: rows sealed under a key this keyring now holds open at once.
2361        self.reveal_sealed_rows()
2362            .map_err(|error| format!("{}: {}", error.code, error.message))?;
2363        self.emit_status_batch()
2364    }
2365
2366    fn clock_now_ms(&self) -> i64 {
2367        self.now_ms
2368            .as_ref()
2369            .map_or_else(crate::wall_clock_ms, |clock| {
2370                clock.load(std::sync::atomic::Ordering::Relaxed)
2371            })
2372    }
2373
2374    fn create_bookkeeping_tables(&self) -> Result<(), String> {
2375        let exists = |name: &str| {
2376            self.conn
2377                .query_row(
2378                    "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?1)",
2379                    [name],
2380                    |row| row.get::<_, bool>(0),
2381                )
2382                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2383        };
2384        let outbox_existed = exists("_syncular_outbox")?;
2385        // Durable client bookkeeping (outbox + units + meta).
2386        self.conn
2387            .execute_batch(
2388                "CREATE TABLE IF NOT EXISTS _syncular_outbox (
2389                   seq INTEGER PRIMARY KEY AUTOINCREMENT,
2390                   commit_id TEXT NOT NULL UNIQUE, ops_json TEXT NOT NULL);
2391                 CREATE TABLE IF NOT EXISTS _syncular_row_deliveries (
2392                   tbl TEXT NOT NULL, id TEXT NOT NULL, commit_seq INTEGER NOT NULL, PRIMARY KEY(tbl,id));
2393                 CREATE TABLE IF NOT EXISTS _syncular_acked_rows (
2394                   commit_id TEXT NOT NULL, idx INTEGER NOT NULL, tbl TEXT NOT NULL, id TEXT NOT NULL,
2395                   commit_seq INTEGER NOT NULL, op_json TEXT NOT NULL, intent_json TEXT,
2396                   PRIMARY KEY(commit_id, idx));
2397                 CREATE INDEX IF NOT EXISTS _syncular_acked_rows_row
2398                   ON _syncular_acked_rows(tbl, id);
2399                 CREATE TABLE IF NOT EXISTS _syncular_failed_commits (
2400                   seq INTEGER PRIMARY KEY AUTOINCREMENT, client_commit_id TEXT NOT NULL UNIQUE,
2401                   operations_json TEXT NOT NULL, initial_json TEXT NOT NULL);
2402                 CREATE TABLE IF NOT EXISTS _syncular_commit_outcomes (
2403                   seq INTEGER PRIMARY KEY AUTOINCREMENT,
2404                   client_commit_id TEXT NOT NULL UNIQUE,
2405                   status TEXT NOT NULL CHECK(status IN ('applied', 'cached', 'conflict', 'rejected')),
2406                   recorded_at_ms INTEGER NOT NULL,
2407                   results_json TEXT NOT NULL,
2408                   operations_json TEXT,
2409                   resolution TEXT NOT NULL DEFAULT 'active'
2410                     CHECK(resolution IN ('active', 'resolved_keep_server', 'superseded', 'dismissed')),
2411                   resolved_at_ms INTEGER,
2412                   replacement_client_commit_id TEXT);
2413                 CREATE INDEX IF NOT EXISTS _syncular_commit_outcomes_resolution_seq
2414                   ON _syncular_commit_outcomes(resolution, seq);
2415                 CREATE TABLE IF NOT EXISTS _syncular_units (
2416                   id TEXT PRIMARY KEY, scopes TEXT NOT NULL,
2417                   explicit INTEGER NOT NULL DEFAULT 0,
2418                   status TEXT NOT NULL DEFAULT 'active', reason_code TEXT,
2419                   cursor INTEGER DEFAULT -1, bootstrap_state TEXT,
2420                   seeded TEXT NOT NULL DEFAULT '[]', echo TEXT, applied TEXT);
2421                 CREATE TABLE IF NOT EXISTS _syncular_meta (
2422                   key TEXT PRIMARY KEY, value TEXT NOT NULL);
2423                 CREATE TABLE IF NOT EXISTS _syncular_windows (
2424                   base TEXT NOT NULL, unit TEXT NOT NULL, sub_id TEXT NOT NULL,
2425                   PRIMARY KEY (base, unit));
2426                 CREATE INDEX IF NOT EXISTS _syncular_windows_unit ON _syncular_windows(sub_id);
2427                 CREATE TABLE IF NOT EXISTS _syncular_pending_evictions (
2428                   unit_id TEXT NOT NULL, tbl TEXT NOT NULL,
2429                   effective_scopes TEXT NOT NULL, PRIMARY KEY (unit_id, tbl));
2430                 CREATE TABLE IF NOT EXISTS _syncular_blob_commit_refs(
2431                   commit_id TEXT NOT NULL, blob_id TEXT NOT NULL,
2432                   PRIMARY KEY(commit_id, blob_id));
2433                 CREATE INDEX IF NOT EXISTS _syncular_blob_commit_refs_body
2434                   ON _syncular_blob_commit_refs(blob_id);",
2435            )
2436            .map_err(|e| e.to_string())?;
2437        // §5.6 applied-segment record, added after the units table shipped.
2438        if !self
2439            .conn
2440            .prepare("SELECT 1 FROM pragma_table_info('_syncular_units') WHERE name = 'applied'")
2441            .and_then(|mut stmt| stmt.exists([]))
2442            .map_err(|e| e.to_string())?
2443        {
2444            self.conn
2445                .execute_batch("ALTER TABLE _syncular_units ADD COLUMN applied TEXT")
2446                .map_err(|e| e.to_string())?;
2447        }
2448        // Migrate an outcome journal created before failed aggregate
2449        // envelopes were retained. Historical rows intentionally stay NULL.
2450        let _ = self
2451            .conn
2452            .execute_batch("ALTER TABLE _syncular_commit_outcomes ADD COLUMN operations_json TEXT");
2453        if self.get_meta(LOCAL_REVISION_KEY)?.is_none() {
2454            self.set_meta(LOCAL_REVISION_KEY, "0")?;
2455        }
2456        // §7.1: `outboxId` names the outbox sequence. A new outbox table starts
2457        // a new sequence, so it replaces the id in the same transaction.
2458        if !outbox_existed || self.get_meta(OUTBOX_ID_KEY)?.is_none() {
2459            self.set_meta(OUTBOX_ID_KEY, &uuid::Uuid::new_v4().to_string())?;
2460        }
2461        // §9: a replica written under wire version 3 keeps its outbox, journal,
2462        // rows and pending evictions and registers its units fresh. Its
2463        // subscriptions, cursors, echoes, resume tokens and window entries go;
2464        // units left from an earlier version 4 life restart.
2465        if exists("_syncular_window_pending_evict")? {
2466            let invalid: bool = self
2467                .conn
2468                .query_row(
2469                    "SELECT EXISTS(SELECT 1 FROM _syncular_window_pending_evict
2470                       WHERE NOT json_valid(effective_scopes)
2471                          OR json_type(effective_scopes) <> 'object'
2472                          OR EXISTS(SELECT 1 FROM json_each(effective_scopes)
2473                               WHERE type <> 'array'
2474                                  OR EXISTS(SELECT 1 FROM json_each(value) AS v
2475                                       WHERE v.type <> 'text')))",
2476                    [],
2477                    |row| row.get(0),
2478                )
2479                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
2480            if invalid {
2481                return Err(
2482                    "sync.local_corrupt: a version 3 pending eviction cannot be upgraded"
2483                        .to_owned(),
2484                );
2485            }
2486            self.conn
2487                .execute_batch(
2488                    "INSERT OR REPLACE INTO _syncular_pending_evictions
2489                       SELECT sub_id, tbl, effective_scopes FROM _syncular_window_pending_evict;
2490                     DROP TABLE _syncular_window_pending_evict;
2491                     DROP TABLE IF EXISTS _syncular_subscriptions;
2492                     DELETE FROM _syncular_windows;",
2493                )
2494                .and_then(|()| reset_units(&self.conn, &BTreeSet::new()))
2495                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
2496        }
2497        // §5.9.7 blob cache. Mutable upload state stays off the body row so
2498        // completing a large upload does not rewrite its BLOB record.
2499        if self.schema_has_blobs() {
2500            if exists("_syncular_blobs")? {
2501                let mut statement = self
2502                    .conn
2503                    .prepare("PRAGMA table_info(_syncular_blobs)")
2504                    .map_err(|e| e.to_string())?;
2505                let mut columns = statement
2506                    .query_map([], |row| row.get::<_, String>(1))
2507                    .map_err(|e| e.to_string())?
2508                    .collect::<Result<Vec<_>, _>>()
2509                    .map_err(|e| e.to_string())?;
2510                columns.sort();
2511                let mut expected = vec![
2512                    "blob_id",
2513                    "byte_length",
2514                    "bytes",
2515                    "created_at_ms",
2516                    "media_type",
2517                ];
2518                expected.sort_unstable();
2519                if columns != expected {
2520                    return Err(
2521                        "sync.schema_mismatch: Local blob schema is incompatible".to_owned()
2522                    );
2523                }
2524            } else {
2525                self.conn
2526                    .execute_batch(
2527                        "CREATE TABLE _syncular_blobs(
2528                           blob_id TEXT PRIMARY KEY,
2529                           bytes BLOB NOT NULL,
2530                           byte_length INTEGER NOT NULL,
2531                           media_type TEXT,
2532                           created_at_ms INTEGER NOT NULL);",
2533                    )
2534                    .map_err(|e| e.to_string())?;
2535            }
2536            self.conn
2537                .execute_batch(
2538                    "CREATE TABLE IF NOT EXISTS _syncular_blob_uploads(
2539                       blob_id TEXT PRIMARY KEY,
2540                       media_type TEXT,
2541                       created_at_ms INTEGER NOT NULL);",
2542                )
2543                .map_err(|e| e.to_string())?;
2544        }
2545        Ok(())
2546    }
2547
2548    /// True iff any synced table declares a `blob_ref` column (§5.9).
2549    fn schema_has_blobs(&self) -> bool {
2550        self.schema
2551            .tables
2552            .iter()
2553            .any(|t| t.columns.iter().any(|c| c.ty == ColumnType::BlobRef))
2554    }
2555
2556    // -- meta (§7.4.1 marker, bookkeeping) ------------------------------------
2557
2558    fn get_meta(&self, key: &str) -> Result<Option<String>, String> {
2559        meta_get(&self.conn, key)
2560            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2561    }
2562
2563    // The caller holds the startup/reset transaction. Either both schema
2564    // records persist or the enclosing operation fails and rolls back.
2565    fn persist_schema_version(&self) -> Result<(), String> {
2566        self.conn
2567            .execute(
2568                "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
2569                rusqlite::params![LOCAL_SCHEMA_VERSION_KEY, self.schema.version.to_string()],
2570            )
2571            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
2572        set_local_schema_descriptor(&self.conn, &self.schema)
2573            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2574    }
2575
2576    pub(crate) fn set_meta(&self, key: &str, value: &str) -> Result<(), String> {
2577        meta_set(&self.conn, key, value)
2578            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2579    }
2580
2581    fn delete_meta(&self, key: &str) -> Result<(), String> {
2582        meta_delete(&self.conn, key)
2583            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2584    }
2585
2586    /// §7.1: the durable outbox in FIFO order, in schema-agnostic local form.
2587    fn load_outbox(&self) -> Result<Vec<Arc<OutboxCommit>>, String> {
2588        let mut stmt = self
2589            .conn
2590            .prepare("SELECT commit_id, ops_json FROM _syncular_outbox ORDER BY seq ASC")
2591            .map_err(|error| error.to_string())?;
2592        let rows = stmt
2593            .query_map([], |row| {
2594                Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
2595            })
2596            .map_err(|error| error.to_string())?;
2597        let mut commits = Vec::new();
2598        for row in rows {
2599            let (client_commit_id, raw) = row.map_err(|error| error.to_string())?;
2600            let entries: Vec<Value> = serde_json::from_str(&raw).map_err(|error| {
2601                format!("invalid persisted outbox {client_commit_id:?}: {error}")
2602            })?;
2603            let mut ops = Vec::with_capacity(entries.len());
2604            for entry in entries {
2605                let op = entry.get("op").and_then(Value::as_str).unwrap_or("delete");
2606                ops.push(OutboxOp {
2607                    upsert: op == "upsert",
2608                    table: entry
2609                        .get("table")
2610                        .and_then(Value::as_str)
2611                        .ok_or_else(|| "persisted outbox operation missing table".to_owned())?
2612                        .to_owned(),
2613                    row_id: entry
2614                        .get("rowId")
2615                        .and_then(Value::as_str)
2616                        .ok_or_else(|| "persisted outbox operation missing rowId".to_owned())?
2617                        .to_owned(),
2618                    base_version: entry.get("baseVersion").and_then(Value::as_i64),
2619                    // A legacy entry's `changedFields` key is ignored:
2620                    // the values map is the presence set (§6.1).
2621                    values: entry.get("values").and_then(Value::as_object).cloned(),
2622                    insert: entry.get("insert") == Some(&Value::Bool(true)),
2623                });
2624            }
2625            commits.push(Arc::new(OutboxCommit {
2626                client_commit_id,
2627                ops,
2628            }));
2629        }
2630        Ok(commits)
2631    }
2632
2633    fn restore_persisted_state(&mut self) -> Result<(), String> {
2634        self.outbox = self.load_outbox()?.into();
2635
2636        self.prune_commit_outcomes()?;
2637        let active = self.commit_outcomes(CommitOutcomeQuery {
2638            active_only: true,
2639            ..CommitOutcomeQuery::default()
2640        })?;
2641        self.conflicts = active
2642            .iter()
2643            .flat_map(|outcome| outcome.results.iter())
2644            .filter_map(|result| match result {
2645                CommitOperationOutcome::Conflict { conflict } => Some(conflict.clone()),
2646                _ => None,
2647            })
2648            .collect::<Vec<_>>()
2649            .into();
2650        self.rejections = active
2651            .iter()
2652            .flat_map(|outcome| outcome.results.iter())
2653            .filter_map(|result| match result {
2654                CommitOperationOutcome::Error { rejection } => Some(rejection.clone()),
2655                _ => None,
2656            })
2657            .collect::<Vec<_>>()
2658            .into();
2659
2660        self.lease_state = self
2661            .get_meta(LEASE_STATE_KEY)?
2662            .map(|raw| serde_json::from_str(&raw))
2663            .transpose()
2664            .map_err(|error| format!("invalid persisted lease state: {error}"))?;
2665        self.schema_floor = self
2666            .get_meta(SCHEMA_FLOOR_KEY)?
2667            .map(|raw| serde_json::from_str(&raw))
2668            .transpose()
2669            .map_err(|error| format!("invalid persisted schema floor: {error}"))?;
2670        self.stopped = self.schema_floor.is_some();
2671        // A replica left in unactivated preflight reopens gated: the quarantine
2672        // decision persists with the data across handle teardown and restart.
2673        if self.get_meta(SECURITY_PREFLIGHT_PENDING_KEY)?.as_deref() == Some("1") {
2674            self.security_preflight = true;
2675        }
2676        Ok(())
2677    }
2678
2679    fn save_unit_scope_schema(&self) -> Result<(), String> {
2680        let declarations: BTreeMap<_, BTreeMap<_, _>> = self
2681            .schema
2682            .tables
2683            .iter()
2684            .map(|table| {
2685                (
2686                    table.name.clone(),
2687                    table
2688                        .scope_variables
2689                        .iter()
2690                        .map(|scope| {
2691                            (
2692                                scope.variable.clone(),
2693                                [scope.prefix.clone(), scope.column.clone()],
2694                            )
2695                        })
2696                        .collect(),
2697                )
2698            })
2699            .collect();
2700        self.set_meta(
2701            "subscriptionScopeSchema",
2702            &serde_json::to_string(&declarations).expect("scope schema serialization"),
2703        )?;
2704        let generations: BTreeMap<_, _> = self
2705            .schema
2706            .tables
2707            .iter()
2708            .filter_map(|table| Some((&table.name, table.generation.as_ref()?)))
2709            .collect();
2710        self.set_meta(
2711            TABLE_GENERATIONS_KEY,
2712            &serde_json::to_string(&generations).expect("generations serialize"),
2713        )
2714    }
2715
2716    /// Drop units that cover no table of the running schema, with their window
2717    /// registry rows and pending evictions; never translate scopes. At a bump
2718    /// (§7.4.3) a unit also needs every variable unchanged in prefix and column
2719    /// on every table it covered before.
2720    fn prune_unknown_units(&mut self, bump: bool) -> Result<(), String> {
2721        let previous: Option<BTreeMap<String, BTreeMap<String, [String; 2]>>> = if bump {
2722            self.get_meta("subscriptionScopeSchema")?
2723                .map(|raw| serde_json::from_str(&raw))
2724                .transpose()
2725                .map_err(|_| "sync.local_corrupt: invalid persisted scope schema".to_owned())?
2726        } else {
2727            None
2728        };
2729        // A table keeps the variables' meaning when each has the same prefix
2730        // and column as before. An old binary stored no declaration evidence;
2731        // matching names cannot prove unchanged meaning, so the app registers
2732        // again.
2733        let unchanged = |name: &str, variables: &[&String]| {
2734            let (Some(old), Some(table)) = (
2735                previous.as_ref().and_then(|previous| previous.get(name)),
2736                self.schema.table(name),
2737            ) else {
2738                return false;
2739            };
2740            variables.iter().all(|variable| {
2741                table
2742                    .scope_variables
2743                    .iter()
2744                    .find(|scope| &scope.variable == *variable)
2745                    .is_some_and(|scope| {
2746                        old.get(*variable) == Some(&[scope.prefix.clone(), scope.column.clone()])
2747                    })
2748            })
2749        };
2750        let compatible = |scopes: &UnitScopes| {
2751            if !self.schema.tables.iter().any(|table| covers(table, scopes)) {
2752                return false;
2753            }
2754            if !bump {
2755                return true;
2756            }
2757            let Some(previous) = &previous else {
2758                return false;
2759            };
2760            previous.iter().all(|(name, old)| {
2761                !scopes.keys().all(|variable| old.contains_key(variable))
2762                    || unchanged(name, &scopes.keys().collect::<Vec<_>>())
2763            })
2764        };
2765        let mut stale: BTreeSet<String> = self
2766            .stored(load_units(&self.conn))?
2767            .into_iter()
2768            .filter(|unit| !compatible(&unit.scopes))
2769            .map(|unit| unit.id)
2770            .collect();
2771        // A shrunk window's deferred eviction outlives its unit row and keeps
2772        // its per-table scopes. A table the schema no longer has cannot be
2773        // evicted, and at a bump a table whose scope declarations changed must
2774        // not be evicted by the outdated rule.
2775        for (id, table, effective) in self.load_pending_evictions()? {
2776            if self.schema.table(&table).is_none()
2777                || (bump
2778                    && !unchanged(
2779                        &table,
2780                        &effective
2781                            .iter()
2782                            .map(|(variable, _)| variable)
2783                            .collect::<Vec<_>>(),
2784                    ))
2785            {
2786                stale.insert(id);
2787            }
2788        }
2789        for id in &stale {
2790            for sql in [
2791                "DELETE FROM _syncular_windows WHERE sub_id = ?1",
2792                "DELETE FROM _syncular_pending_evictions WHERE unit_id = ?1",
2793                "DELETE FROM _syncular_units WHERE id = ?1",
2794            ] {
2795                self.conn
2796                    .execute(sql, rusqlite::params![id])
2797                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
2798            }
2799        }
2800        Ok(())
2801    }
2802
2803    pub fn local_revision(&self) -> Result<u64, String> {
2804        Ok(self
2805            .get_meta(LOCAL_REVISION_KEY)?
2806            .and_then(|value| value.parse().ok())
2807            .unwrap_or(0))
2808    }
2809
2810    /// §7.5: resolve a snapshot read on the mutable owner connection through
2811    /// the same primitive the read-only sidecar uses, so the command surface
2812    /// and the file sidecar return one revision, rows, coverage, catch-up, and
2813    /// delivery shape. The structured [`QueryReadFailure`] is preserved; an
2814    /// `owner` records its failure (and its next success) in diagnostics.
2815    pub fn snapshot_read(
2816        &mut self,
2817        request: &SnapshotReadRequest<'_>,
2818        owner: Option<QueryOwner<'_>>,
2819    ) -> Result<SnapshotRead, QueryReadFailure> {
2820        let result = snapshot_read_connection(&self.conn, request);
2821        if let Some(owner) = owner {
2822            self.record_query_read(owner, result.as_ref().err());
2823        }
2824        result
2825    }
2826
2827    #[must_use]
2828    pub fn status_snapshot(&self) -> SyncStatusSnapshot {
2829        SyncStatusSnapshot {
2830            current_schema_version: self.schema.version,
2831            outbox: self.outbox.len(),
2832            upgrading: self.upgrading,
2833            lease_state: self.lease_state.clone(),
2834            schema_floor: self.schema_floor.clone(),
2835            sync_needed: self.sync_needed,
2836            previous_version_context: self.previous_version_status(),
2837            security_lifecycle: self.security_lifecycle(),
2838        }
2839    }
2840
2841    /// RFC 0005 A2: container presence for a host that must act on it (Diego's
2842    /// rollback precondition), read without creating the file.
2843    fn previous_version_status(&self) -> PreviousVersionStatus {
2844        let Some(replica_path) = self.previous_version_replica_path() else {
2845            return PreviousVersionStatus {
2846                present: false,
2847                created_at_ms: None,
2848            };
2849        };
2850        match crate::previous_version::read_previous_version_record(&replica_path) {
2851            Ok(Some(record)) => PreviousVersionStatus {
2852                present: true,
2853                created_at_ms: Some(record.created_at_ms),
2854            },
2855            _ => PreviousVersionStatus {
2856                present: std::path::Path::new(
2857                    &crate::previous_version::previous_version_container_path(&replica_path),
2858                )
2859                .exists(),
2860                created_at_ms: None,
2861            },
2862        }
2863    }
2864
2865    /// RFC 0005 D7: read captured rows from the previous local schema. `state`
2866    /// is always `previousVersion`; the container never makes
2867    /// `query_snapshot()` coverage complete. Lease stop/expiry, scope
2868    /// revocation, coverage completion and the TTL DISCARD the container and
2869    /// close the read with their own reason.
2870    pub fn previous_version_snapshot(
2871        &self,
2872        spec: &PreviousVersionReadSpec,
2873    ) -> Result<PreviousVersionSnapshot, String> {
2874        let now_ms = self.clock_now_ms();
2875        crate::previous_version::previous_version_snapshot(
2876            &self.conn,
2877            self.previous_version_replica_path().as_deref(),
2878            self.previous_version.as_ref(),
2879            PreviousVersionLifecycle {
2880                lease_inactive: self.previous_version_lease_inactive(now_ms),
2881                scope_revoked: self
2882                    .stored(load_units(&self.conn))?
2883                    .iter()
2884                    .any(|unit| unit.status == UnitStatus::Revoked),
2885                coverage_complete: self.previous_version_coverage_complete()?,
2886            },
2887            now_ms,
2888            self.schema.version,
2889            spec,
2890        )
2891    }
2892
2893    /// RFC 0005 D6: the advisory pre-reset compatibility audit.
2894    pub fn previous_version_audit(
2895        &self,
2896    ) -> Result<Option<crate::previous_version::PreviousVersionAudit>, String> {
2897        crate::previous_version::stored_previous_version_audit(&self.conn)
2898            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2899    }
2900
2901    /// RFC 0005 A2/D9: the executable downgrade step. Drop the container file
2902    /// and both metadata records, and report whether anything was present.
2903    /// Idempotent: absence is a successful no-op.
2904    pub fn previous_version_discard(
2905        &self,
2906    ) -> Result<crate::previous_version::PreviousVersionDiscardOutcome, String> {
2907        self.drop_previous_version()
2908    }
2909
2910    fn drop_previous_version(
2911        &self,
2912    ) -> Result<crate::previous_version::PreviousVersionDiscardOutcome, String> {
2913        let Some(replica_path) = self.previous_version_replica_path() else {
2914            return Ok(crate::previous_version::PreviousVersionDiscardOutcome {
2915                present: false,
2916                discarded: false,
2917            });
2918        };
2919        crate::previous_version::discard_previous_version(&self.conn, &replica_path)
2920    }
2921
2922    /// RFC 0005 D7: `leaseState.errorCode` or an expired lease.
2923    fn previous_version_lease_inactive(&self, now_ms: i64) -> bool {
2924        let Some(lease) = &self.lease_state else {
2925            return false;
2926        };
2927        if lease.error_code.is_some() {
2928            return true;
2929        }
2930        lease.expires_at_ms.is_some_and(|expires| expires <= now_ms)
2931    }
2932
2933    /// RFC 0005 D7: §7.4.5 replacement coverage — no active unit is
2934    /// bootstrapping. An empty active set is not completion (mirrors the TS
2935    /// predicate exactly).
2936    fn previous_version_coverage_complete(&self) -> Result<bool, String> {
2937        let units = self.stored(load_units(&self.conn))?;
2938        let mut active = units
2939            .iter()
2940            .filter(|unit| unit.status == UnitStatus::Active)
2941            .peekable();
2942        Ok(active.peek().is_some() && !active.any(UnitRecord::bootstrapping))
2943    }
2944
2945    /// RFC 0005 D7: lifetime triggers evaluated once per sync round, after the
2946    /// apply. A container that no longer has a reason to exist is discarded so
2947    /// the rows do not outlive the window they were kept for.
2948    fn previous_version_lifetime_check(&self) {
2949        let Some(config) = self.previous_version.filter(|config| config.enabled) else {
2950            return;
2951        };
2952        let now_ms = self.clock_now_ms();
2953        if self.previous_version_lease_inactive(now_ms) {
2954            let _ = self.drop_previous_version();
2955            return;
2956        }
2957        let Some(replica_path) = self.previous_version_replica_path() else {
2958            return;
2959        };
2960        let Ok(Some(record)) = crate::previous_version::read_previous_version_record(&replica_path)
2961        else {
2962            return;
2963        };
2964        let Ok(units) = load_units(&self.conn) else {
2965            return;
2966        };
2967        if units.iter().any(|unit| unit.status == UnitStatus::Revoked) {
2968            let _ = self.drop_previous_version();
2969            return;
2970        }
2971        if now_ms - record.created_at_ms > config.max_age_ms {
2972            let _ = self.drop_previous_version();
2973            return;
2974        }
2975        if self.previous_version_coverage_complete() == Ok(true) {
2976            let _ = self.drop_previous_version();
2977        }
2978    }
2979
2980    pub fn diagnostics_snapshot(&self) -> Result<ClientDiagnosticsSnapshot, String> {
2981        let registered = self.stored(load_units(&self.conn))?;
2982        // Both meta values in one statement: the FFI host recomputes
2983        // diagnostics after every command.
2984        let (local_revision, replica_cursor): (Option<String>, Option<String>) = self.stored(
2985            self.conn
2986                .prepare_cached(
2987                    "SELECT (SELECT value FROM _syncular_meta WHERE key = ?1),
2988                            (SELECT value FROM _syncular_meta WHERE key = ?2)",
2989                )
2990                .and_then(|mut statement| {
2991                    statement.query_row([LOCAL_REVISION_KEY, REPLICA_CURSOR_KEY], |row| {
2992                        Ok((row.get(0)?, row.get(1)?))
2993                    })
2994                }),
2995        )?;
2996        let mut units = DiagnosticUnits::default();
2997        for unit in &registered {
2998            *match unit.status {
2999                UnitStatus::Revoked => &mut units.revoked,
3000                UnitStatus::Failed => &mut units.failed,
3001                UnitStatus::Active if !unit.bootstrapping() => &mut units.complete,
3002                UnitStatus::Active
3003                    if unit.reason_code.as_deref() == Some("sync.cursor_expired") =>
3004                {
3005                    &mut units.reset
3006                }
3007                UnitStatus::Active => &mut units.bootstrapping,
3008            } += 1;
3009        }
3010        units.reason_codes = registered
3011            .iter()
3012            .filter_map(|unit| unit.reason_code.as_deref().map(Self::diagnostic_code))
3013            .collect::<BTreeSet<_>>()
3014            .into_iter()
3015            .collect();
3016        let captured_at_ms = self.clock_now_ms();
3017        let lease = if let Some(error_code) = self
3018            .lease_state
3019            .as_ref()
3020            .and_then(|state| state.error_code.clone())
3021        {
3022            ClientDiagnosticsLease {
3023                state: "stopped".to_owned(),
3024                expires_at_ms: self
3025                    .lease_state
3026                    .as_ref()
3027                    .and_then(|state| state.expires_at_ms),
3028                error_code: Some(Self::diagnostic_code(&error_code)),
3029            }
3030        } else if let Some(expires_at_ms) = self
3031            .lease_state
3032            .as_ref()
3033            .and_then(|state| state.expires_at_ms)
3034        {
3035            ClientDiagnosticsLease {
3036                state: if expires_at_ms <= captured_at_ms {
3037                    "expired".to_owned()
3038                } else {
3039                    "active".to_owned()
3040                },
3041                expires_at_ms: Some(expires_at_ms),
3042                error_code: None,
3043            }
3044        } else {
3045            ClientDiagnosticsLease {
3046                state: "none".to_owned(),
3047                expires_at_ms: None,
3048                error_code: None,
3049            }
3050        };
3051        let connectivity = match self.last_round.as_ref() {
3052            _ if !self.transport_enabled && self.active_round.is_none() => "offline",
3053            Some(round) if round.status == "succeeded" => "online",
3054            Some(round)
3055                if round.status == "failed"
3056                    && matches!(
3057                        round.cause_kind,
3058                        Some(
3059                            crate::TransportCauseKind::Network | crate::TransportCauseKind::Connect
3060                        )
3061                    ) =>
3062            {
3063                "offline"
3064            }
3065            Some(round) if round.status == "failed" && round.http_status.is_some() => "online",
3066            _ => "unknown",
3067        };
3068        Ok(ClientDiagnosticsSnapshot {
3069            version: CLIENT_DIAGNOSTICS_VERSION,
3070            captured_at_ms,
3071            host: ClientDiagnosticsHost {
3072                kind: "direct".to_owned(),
3073                role: "single".to_owned(),
3074                connectivity: connectivity.to_owned(),
3075                realtime: self.realtime_state,
3076                realtime_policy: self.realtime_policy,
3077                realtime_reason_code: self.realtime_reason_code.clone(),
3078                realtime_retry_delay_ms: self.realtime_retry_delay_ms,
3079            },
3080            security_lifecycle: self.security_lifecycle().to_owned(),
3081            schema: ClientDiagnosticsSchema {
3082                current_version: self.schema.version,
3083                upgrading: self.upgrading,
3084                required_version: self
3085                    .schema_floor
3086                    .as_ref()
3087                    .and_then(|floor| floor.required_schema_version),
3088                latest_version: self
3089                    .schema_floor
3090                    .as_ref()
3091                    .and_then(|floor| floor.latest_schema_version),
3092            },
3093            replica: ClientDiagnosticsReplica {
3094                local_revision: local_revision
3095                    .and_then(|value| value.parse::<u64>().ok())
3096                    .unwrap_or(0)
3097                    .to_string(),
3098                sync_needed: self.sync_needed,
3099                pending_outbox: self.outbox.len(),
3100            },
3101            lease,
3102            active_units: registered
3103                .iter()
3104                .filter(|unit| unit.status == UnitStatus::Active)
3105                .count(),
3106            units,
3107            replica_cursor: replica_cursor.map_or(Ok(-1), |raw| {
3108                raw.parse()
3109                    .map_err(|_| "sync.local_corrupt: persisted unit state is invalid".to_owned())
3110            })?,
3111            last_round: self.last_round.clone(),
3112            last_change: self.last_change.clone(),
3113            storage: self.diagnostics_storage(),
3114            query_failures: self.query_failures.clone(),
3115        })
3116    }
3117
3118    fn diagnostics_storage(&self) -> ClientDiagnosticsStorage {
3119        let read = || -> Result<ClientDiagnosticsStorage, rusqlite::Error> {
3120            let page_count: i64 = self
3121                .conn
3122                .query_row("PRAGMA page_count", [], |row| row.get(0))?;
3123            let page_size: i64 = self
3124                .conn
3125                .query_row("PRAGMA page_size", [], |row| row.get(0))?;
3126            // One cached aggregate, since the FFI host recomputes diagnostics
3127            // after every command; every invocation still reads current storage.
3128            let (outbox_bytes, outcome_entries, outcome_bytes): (i64, i64, i64) = self
3129                .conn
3130                .prepare_cached(
3131                    "SELECT (SELECT COALESCE(SUM(LENGTH(ops_json)), 0) FROM _syncular_outbox),
3132                            COUNT(*), COALESCE(SUM(LENGTH(results_json) + COALESCE(LENGTH(operations_json), 0)), 0)
3133                       FROM _syncular_commit_outcomes",
3134                )?
3135                .query_row([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))?;
3136            let blob_bytes = if self.schema_has_blobs() {
3137                self.conn
3138                    .prepare_cached("SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs")?
3139                    .query_row([], |row| row.get(0))?
3140            } else {
3141                0
3142            };
3143            let pressure = self
3144                .limits
3145                .blob_cache_max_bytes
3146                .is_some_and(|limit| blob_bytes > limit);
3147            Ok(ClientDiagnosticsStorage {
3148                status: if pressure { "pressure" } else { "healthy" }.to_owned(),
3149                database_bytes_approx: Some(page_count.saturating_mul(page_size).max(0)),
3150                pending_outbox_bytes_approx: Some(outbox_bytes.max(0)),
3151                retained_outcome_bytes_approx: Some(outcome_bytes.max(0)),
3152                retained_outcome_entries: Some(outcome_entries.max(0)),
3153                blob_cache_bytes_approx: Some(blob_bytes.max(0)),
3154                pressure_reason_code: pressure.then(|| "client.blob_cache_over_limit".to_owned()),
3155            })
3156        };
3157        read().unwrap_or_else(|_| ClientDiagnosticsStorage {
3158            status: "unreadable".to_owned(),
3159            database_bytes_approx: None,
3160            pending_outbox_bytes_approx: None,
3161            retained_outcome_bytes_approx: None,
3162            retained_outcome_entries: None,
3163            blob_cache_bytes_approx: None,
3164            pressure_reason_code: None,
3165        })
3166    }
3167
3168    pub fn drain_change_batches(&mut self) -> Vec<ClientChangeBatch> {
3169        self.change_queue.drain(..).collect()
3170    }
3171
3172    pub fn drain_sync_intents(&mut self) -> Vec<SyncIntent> {
3173        let intents = self.sync_intent_queue.drain(..).collect();
3174        if self.transport_enabled {
3175            intents
3176        } else {
3177            vec![SyncIntent::None]
3178        }
3179    }
3180
3181    /// §8.7 host announcement: the number of `scopes` events received since
3182    /// the last drain. The event changes no sync state; the host surfaces each
3183    /// one to the app, whose discovery registers the newly readable units.
3184    pub fn drain_scopes_changed(&mut self) -> usize {
3185        std::mem::take(&mut self.scopes_changed)
3186    }
3187
3188    fn schedule_background_retry(&mut self) -> u64 {
3189        let delay_ms = self.retry_delay_ms;
3190        self.sync_intent_queue
3191            .push_back(SyncIntent::Background { delay_ms });
3192        self.retry_delay_ms = (self.retry_delay_ms * 2).min(30_000);
3193        self.round_retry_delay_ms = Some(delay_ms);
3194        delay_ms
3195    }
3196
3197    fn reset_background_retry(&mut self) {
3198        self.retry_delay_ms = 250;
3199    }
3200
3201    fn retryable_transport_code(code: &str) -> bool {
3202        code == "transport.failed"
3203            || code == "transport.timeout"
3204            || code == "transport.unavailable"
3205            || code == "sync.transport_failed"
3206    }
3207
3208    /// §1.8: the transport reported a refused or unreachable endpoint. The
3209    /// native transport says `connect` (DNS failure, connection refused,
3210    /// reset, or aborted); a browser fetch transport says `network`. Repeating
3211    /// the request on a backoff repairs neither, so no retry is scheduled.
3212    fn unreachable_transport(details: &Option<serde_json::Value>) -> bool {
3213        matches!(
3214            details
3215                .as_ref()
3216                .and_then(|details| details.get("causeKind"))
3217                .and_then(serde_json::Value::as_str),
3218            Some("connect" | "network")
3219        )
3220    }
3221
3222    /// §1.5: a segment fetch that fails inside an otherwise complete round is
3223    /// invisible to the server (the round response arrived whole), so its code
3224    /// rides the following requests like a socket-round failure. A code the
3225    /// frame grammar rejects is not reportable and the previous code stays.
3226    fn segment_fetch_failed(&mut self, error: TransportError) -> SectionError {
3227        if ssp2::model::valid_round_failure_code(&error.code) {
3228            self.round_failure = Some(error.code.clone());
3229        }
3230        SectionError::Transport(error)
3231    }
3232
3233    /// A failed round worth a background retry: a transport failure, or a
3234    /// §10.2 code the catalog marks retryable (clients may hardcode the
3235    /// catalog metadata, §10.1). Matches the TypeScript core, which retries
3236    /// on the server's `retryable` flag.
3237    fn retryable_failure_code(code: &str) -> bool {
3238        Self::retryable_transport_code(code)
3239            || matches!(
3240                code,
3241                "sync.auth_required"
3242                    | "sync.auth_lease_required"
3243                    | "sync.auth_lease_revoked"
3244                    | "sync.internal_error"
3245                    | "sync.idempotency_cache_miss"
3246                    | "sync.schema_not_ready"
3247                    | "sync.segment_expired"
3248                    | "sync.rate_limited"
3249                    | "sync.websocket_connection_limit"
3250                    | "sync.idle_timeout"
3251            )
3252    }
3253
3254    fn diagnostic_code(code: &str) -> String {
3255        let valid = !code.is_empty()
3256            && code.len() <= 96
3257            && code.contains('.')
3258            && code
3259                .bytes()
3260                .next()
3261                .is_some_and(|byte| byte.is_ascii_lowercase())
3262            && code.bytes().all(|byte| {
3263                byte.is_ascii_lowercase()
3264                    || byte.is_ascii_digit()
3265                    || matches!(byte, b'.' | b'_' | b'-')
3266            });
3267        if valid {
3268            code.to_owned()
3269        } else {
3270            "client.unknown_failure".to_owned()
3271        }
3272    }
3273
3274    fn set_sync_needed(&mut self, value: bool, interactive: bool) {
3275        if self.sync_needed != value {
3276            if self.begin_observation("syncular_status").is_ok() {
3277                self.sync_needed = value;
3278                let batch = ChangeAccumulator {
3279                    status: true,
3280                    ..ChangeAccumulator::default()
3281                };
3282                if self.finish_observation("syncular_status", batch).is_err() {
3283                    self.rollback_observation("syncular_status");
3284                }
3285            } else {
3286                self.sync_needed = value;
3287            }
3288        }
3289        if value && interactive {
3290            self.sync_intent_queue.push_back(SyncIntent::Interactive);
3291        }
3292    }
3293
3294    /// A unit-store result: a row that does not parse is `sync.local_corrupt`;
3295    /// every other SQLite error keeps its storage classification.
3296    fn stored<T>(&self, result: rusqlite::Result<T>) -> Result<T, String> {
3297        result.map_err(|error| match error {
3298            rusqlite::Error::FromSqlConversionFailure(..) => {
3299                "sync.local_corrupt: persisted unit state is invalid".to_owned()
3300            }
3301            error => Self::sqlite_failure(&self.storage_failure, error),
3302        })
3303    }
3304
3305    fn sqlite_failure(
3306        storage_failure: &RefCell<Option<QueryReadFailure>>,
3307        error: rusqlite::Error,
3308    ) -> String {
3309        let message = error.to_string();
3310        let failure = QueryReadFailure::from(error);
3311        let code = failure.code;
3312        if code.is_some() && storage_failure.borrow().is_none() {
3313            *storage_failure.borrow_mut() = Some(failure);
3314        }
3315        match code {
3316            Some(code) => format!("{code}: {message}"),
3317            None => message,
3318        }
3319    }
3320
3321    fn begin_observation(&self, name: &str) -> Result<(), String> {
3322        if let Some(failure) = &self.coalesced_failure {
3323            return Err(failure.clone());
3324        }
3325        // SQLite rolls a whole transaction back on some I/O and capacity
3326        // errors; later frames must not commit on their own after that.
3327        if self.coalesced.is_some() && self.conn.is_autocommit() {
3328            return Err("the coalesced apply transaction was rolled back".to_owned());
3329        }
3330        self.conn
3331            .execute_batch(&format!("SAVEPOINT {name}"))
3332            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
3333    }
3334
3335    /// SPEC §1.4 rule 4: open a coalesced apply transaction unless a caller
3336    /// already owns one.
3337    fn begin_coalesced_apply(&mut self) -> Result<(), String> {
3338        if self.coalesced.is_some() || !self.conn.is_autocommit() {
3339            return Ok(());
3340        }
3341        self.conn
3342            .execute_batch("BEGIN")
3343            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3344        self.coalesced = Some(CoalescedApply {
3345            rows: 0,
3346            rollback: self.capture(),
3347        });
3348        Ok(())
3349    }
3350
3351    fn capture(&self) -> Rollback {
3352        Rollback {
3353            outbox: self.outbox.clone(),
3354            failed_commits: self.failed_commits.clone(),
3355            conflicts: self.conflicts.clone(),
3356            rejections: self.rejections.clone(),
3357            schema_floor: self.schema_floor.clone(),
3358            lease_state: self.lease_state.clone(),
3359            stopped: self.stopped,
3360            upgrading: self.upgrading,
3361            sync_needed: self.sync_needed,
3362            active_round: self.active_round,
3363            overlay_dirty: self.overlay_dirty.snapshot(),
3364            change_queue: self.change_queue.len(),
3365            sync_intent_queue: self.sync_intent_queue.len(),
3366            last_change: self.last_change.clone(),
3367        }
3368    }
3369
3370    fn restore(&mut self, rollback: Rollback) {
3371        self.outbox = rollback.outbox;
3372        self.failed_commits = rollback.failed_commits;
3373        self.conflicts = rollback.conflicts;
3374        self.rejections = rollback.rejections;
3375        self.schema_floor = rollback.schema_floor;
3376        self.lease_state = rollback.lease_state;
3377        self.stopped = rollback.stopped;
3378        self.upgrading = rollback.upgrading;
3379        self.sync_needed = rollback.sync_needed;
3380        self.active_round = rollback.active_round;
3381        self.overlay_dirty.restore(rollback.overlay_dirty);
3382        self.change_queue.truncate(rollback.change_queue);
3383        self.sync_intent_queue.truncate(rollback.sync_intent_queue);
3384        self.last_change = rollback.last_change;
3385    }
3386
3387    /// Account the rows of the next frame or block. A unit that would take the
3388    /// coalesced transaction past the bound commits the earlier units first.
3389    fn coalesce_unit(&mut self, rows: usize) -> Result<(), String> {
3390        let Some(open) = &self.coalesced else {
3391            return Ok(());
3392        };
3393        if open.rows > 0 && open.rows + rows > COALESCED_APPLY_ROWS {
3394            self.commit_coalesced_apply()?;
3395            self.begin_coalesced_apply()?;
3396        }
3397        if let Some(open) = &mut self.coalesced {
3398            open.rows += rows;
3399        }
3400        Ok(())
3401    }
3402
3403    /// COMMIT the coalesced transaction. A failed COMMIT rolls every coalesced
3404    /// frame back, restores the in-memory state of its `BEGIN`, drops the
3405    /// change batches and sync intents queued since, and fails the rest of the
3406    /// response.
3407    fn commit_coalesced_apply(&mut self) -> Result<(), String> {
3408        let Some(open) = self.coalesced.take() else {
3409            return Ok(());
3410        };
3411        let committed = if self.conn.is_autocommit() {
3412            Err("the coalesced apply transaction was rolled back".to_owned())
3413        } else {
3414            self.conn
3415                .execute_batch("COMMIT")
3416                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
3417        };
3418        let Err(error) = committed else {
3419            return Ok(());
3420        };
3421        if !self.conn.is_autocommit() {
3422            if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
3423                let _ = Self::sqlite_failure(&self.storage_failure, rollback);
3424            }
3425        }
3426        self.restore(open.rollback);
3427        self.coalesced_failure = Some(error.clone());
3428        Err(error)
3429    }
3430
3431    /// End the response's coalescing: COMMIT what is open and clear a failure.
3432    fn finish_coalesced_apply(&mut self) -> Result<(), String> {
3433        let committed = self.commit_coalesced_apply();
3434        match self.coalesced_failure.take() {
3435            Some(failure) => Err(failure),
3436            None => committed,
3437        }
3438    }
3439
3440    fn rollback_observation(&self, name: &str) {
3441        if let Err(error) = self
3442            .conn
3443            .execute_batch(&format!("ROLLBACK TO {name}; RELEASE {name}"))
3444        {
3445            if let Some(failure) = self.storage_failure.borrow_mut().as_mut() {
3446                if failure.rollback_failure.is_none() {
3447                    failure.rollback_failure = Some(Box::new(QueryReadFailure::from(error)));
3448                }
3449            }
3450        }
3451    }
3452
3453    fn finish_observation(
3454        &mut self,
3455        name: &str,
3456        mut batch: ChangeAccumulator,
3457    ) -> Result<(), String> {
3458        #[cfg(feature = "bench-internals")]
3459        let _phase = self.benchmark_phases.start(Phase::ObservationCommit);
3460        if self.failed_commits.iter().any(|(commit, _)| {
3461            commit
3462                .ops
3463                .iter()
3464                .any(|operation| batch.tables.contains_key(&operation.table))
3465        }) {
3466            batch.outcomes = true;
3467        }
3468        if !batch.touched() {
3469            self.conn
3470                .execute_batch(&format!("RELEASE {name}"))
3471                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3472            return Ok(());
3473        }
3474        let revision = self
3475            .local_revision()?
3476            .checked_add(1)
3477            .ok_or_else(|| "local revision exhausted u64".to_owned())?;
3478        self.conn
3479            .execute(
3480                "INSERT OR REPLACE INTO _syncular_meta(key, value) VALUES (?1, ?2)",
3481                rusqlite::params![LOCAL_REVISION_KEY, revision.to_string()],
3482            )
3483            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3484        let status = batch.status.then(|| self.status_snapshot());
3485        let event = ClientChangeBatch {
3486            revision: revision.to_string(),
3487            tables: batch
3488                .tables
3489                .into_iter()
3490                .map(|(table, scope_keys)| TableChange {
3491                    table,
3492                    scope_keys: scope_keys.map(|keys| keys.into_iter().collect()),
3493                })
3494                .collect(),
3495            windows: batch
3496                .windows
3497                .into_iter()
3498                .map(|(base_key, units)| WindowChange {
3499                    base_key,
3500                    units: units.into_iter().collect(),
3501                })
3502                .collect(),
3503            status,
3504            conflicts_changed: batch.conflicts,
3505            rejections_changed: batch.rejections,
3506            outcomes_changed: batch.outcomes,
3507            applied_commits: Vec::new(),
3508        };
3509        self.conn
3510            .execute_batch(&format!("RELEASE {name}"))
3511            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3512        let mut diagnostic_tables = event
3513            .tables
3514            .iter()
3515            .map(|entry| entry.table.clone())
3516            .collect::<BTreeSet<_>>()
3517            .into_iter()
3518            .collect::<Vec<_>>();
3519        // §7.6: the selected variables of the changed window bases.
3520        let mut diagnostic_windows = event
3521            .windows
3522            .iter()
3523            .map(|entry| {
3524                entry
3525                    .base_key
3526                    .split('\0')
3527                    .next()
3528                    .unwrap_or_default()
3529                    .to_owned()
3530            })
3531            .collect::<BTreeSet<_>>()
3532            .into_iter()
3533            .collect::<Vec<_>>();
3534        let domains_truncated = diagnostic_tables.len() > MAX_DIAGNOSTIC_DOMAINS
3535            || diagnostic_windows.len() > MAX_DIAGNOSTIC_DOMAINS;
3536        diagnostic_tables.truncate(MAX_DIAGNOSTIC_DOMAINS);
3537        diagnostic_windows.truncate(MAX_DIAGNOSTIC_DOMAINS);
3538        self.last_change = Some(DiagnosticLastChange {
3539            revision: event.revision.clone(),
3540            recorded_at_ms: self.clock_now_ms(),
3541            tables: diagnostic_tables,
3542            windows: diagnostic_windows,
3543            domains_truncated,
3544            status_changed: event.status.is_some(),
3545            conflicts_changed: event.conflicts_changed,
3546            rejections_changed: event.rejections_changed,
3547            outcomes_changed: event.outcomes_changed,
3548        });
3549        self.change_queue.push_back(event);
3550        Ok(())
3551    }
3552
3553    fn record_scope_map(
3554        &self,
3555        batch: &mut ChangeAccumulator,
3556        table_name: &str,
3557        scopes: &[(String, Vec<String>)],
3558    ) {
3559        let Some(table) = self.schema.table(table_name) else {
3560            return;
3561        };
3562        // A bulk scoped delete whose map leaves a declared variable unnamed
3563        // cannot list the deleted rows' keys for it: the change is table-wide.
3564        if table.scope_variables.iter().any(|scope| {
3565            !scopes
3566                .iter()
3567                .any(|(variable, _)| *variable == scope.variable)
3568        }) {
3569            batch.table(table_name);
3570            return;
3571        }
3572        for (variable, values) in scopes {
3573            let Some(scope) = table
3574                .scope_variables
3575                .iter()
3576                .find(|scope| &scope.variable == variable)
3577            else {
3578                continue;
3579            };
3580            for value in values {
3581                batch.scope(table_name, format!("{}:{value}", scope.prefix));
3582            }
3583        }
3584    }
3585
3586    /// Record a row's current scope keys from the base or visible table.
3587    fn record_row_scopes(
3588        &self,
3589        batch: &mut ChangeAccumulator,
3590        table_name: &str,
3591        row_id: &str,
3592        base: bool,
3593    ) -> bool {
3594        let Some(table) = self.schema.table(table_name) else {
3595            return false;
3596        };
3597        if table.scope_variables.is_empty() {
3598            return false;
3599        }
3600        let columns = table
3601            .scope_variables
3602            .iter()
3603            .map(|scope| quote_ident(&scope.column))
3604            .collect::<Vec<_>>()
3605            .join(", ");
3606        let full_table = if base {
3607            base_table(table_name)
3608        } else {
3609            visible_table(table_name)
3610        };
3611        let sql = format!(
3612            "SELECT {columns} FROM {full_table} WHERE {} LIMIT 1",
3613            row_id_predicate(table)
3614        );
3615        let Ok(mut stmt) = self.conn.prepare_cached(&sql) else {
3616            return false;
3617        };
3618        let values = stmt.query_row(rusqlite::params![row_id], |row| {
3619            let mut values = Vec::with_capacity(table.scope_variables.len());
3620            for index in 0..table.scope_variables.len() {
3621                values.push(row.get::<_, Option<String>>(index)?);
3622            }
3623            Ok(values)
3624        });
3625        let Ok(values) = values else {
3626            return false;
3627        };
3628        let mut recorded = false;
3629        for (scope, value) in table.scope_variables.iter().zip(values) {
3630            if let Some(value) = value {
3631                batch.scope(table_name, format!("{}:{value}", scope.prefix));
3632                recorded = true;
3633            }
3634        }
3635        recorded
3636    }
3637
3638    fn record_commit_changes(
3639        &self,
3640        batch: &mut ChangeAccumulator,
3641        tables: &[String],
3642        changes: &[ssp2::model::Change],
3643    ) {
3644        #[cfg(feature = "bench-internals")]
3645        let _phase = self.benchmark_phases.start(Phase::ObservationPrepare);
3646        for change in changes {
3647            let Some(table_name) = tables.get(change.table_index as usize) else {
3648                continue;
3649            };
3650            let mut precise = self.record_row_scopes(batch, table_name, &change.row_id, true);
3651            if let Some(table) = self.schema.table(table_name) {
3652                for (variable, value) in &change.scopes {
3653                    if let Some(scope) = table
3654                        .scope_variables
3655                        .iter()
3656                        .find(|scope| &scope.variable == variable)
3657                    {
3658                        batch.scope(table_name, format!("{}:{value}", scope.prefix));
3659                        precise = true;
3660                    }
3661                }
3662            }
3663            if !precise {
3664                batch.table(table_name);
3665            }
3666        }
3667    }
3668
3669    /// Whether a base or visible row matches the scope predicate; `false`
3670    /// without a local scope-column mapping. A SQLite failure is an error.
3671    fn scoped_rows_exist(
3672        &self,
3673        table_name: &str,
3674        effective: &[(String, Vec<String>)],
3675        owners: &[Vec<(String, Vec<String>)>],
3676    ) -> Result<bool, String> {
3677        let Some(table) = self.schema.table(table_name) else {
3678            return Ok(false);
3679        };
3680        let Ok(Some((predicate, params))) = local_scope_predicate(table, effective, owners) else {
3681            return Ok(false);
3682        };
3683        for source in [base_table(table_name), visible_table(table_name)] {
3684            if self
3685                .conn
3686                .query_row(
3687                    &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {predicate})"),
3688                    rusqlite::params_from_iter(&params),
3689                    |row| row.get::<_, bool>(0),
3690                )
3691                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
3692            {
3693                return Ok(true);
3694            }
3695        }
3696        Ok(false)
3697    }
3698
3699    /// §5.6 first-page clear of `unit_id`'s pair: delete its rows except
3700    /// those another owner covers (§3.3), failing closed without a local
3701    /// scope-column mapping. Owners are read only when a row matches, inside
3702    /// the caller's transaction. Returns whether rows matched.
3703    fn clear_scope(
3704        &mut self,
3705        table: &TableSchema,
3706        effective: &[(String, Vec<String>)],
3707        unit_id: &str,
3708    ) -> Result<bool, SectionError> {
3709        let storage = |message| SectionError::Abort("storage.failed".to_owned(), message);
3710        let matched = self
3711            .scoped_rows_exist(&table.name, effective, &[])
3712            .map_err(storage)?;
3713        let owners = if matched {
3714            unit_owners(
3715                &self.stored(load_units(&self.conn)).map_err(storage)?,
3716                table,
3717                unit_id,
3718            )
3719        } else {
3720            Vec::new()
3721        };
3722        let changed = matched
3723            && (owners.is_empty()
3724                || self
3725                    .scoped_rows_exist(&table.name, effective, &owners)
3726                    .map_err(storage)?);
3727        if !self
3728            .purge_scope_rows(&table.name, effective, &owners)
3729            .map_err(storage)?
3730        {
3731            return Err(SectionError::FailClosed);
3732        }
3733        Ok(changed)
3734    }
3735
3736    /// §7.4.5: true while a schema-bump reset + first re-bootstrap runs.
3737    pub fn upgrading(&self) -> bool {
3738        self.upgrading
3739    }
3740
3741    /// §7.4.2 "app ships new code": swap to a NEW generated schema while
3742    /// keeping this client's local database (identity, outbox, tables). The
3743    /// §7.4.1 marker check then fires the wipe/re-bootstrap flow when the
3744    /// version increases. Recreation runs the same marker guard as opening
3745    /// a durable replica, holds the guard and every write it authorizes under
3746    /// one write transaction, and preserves the current client (compiled
3747    /// schema, outbox, tables) on refusal.
3748    pub fn recreate_with_schema(&mut self, schema_json: &Value) -> Result<(), String> {
3749        let new_schema = parse_schema_json(schema_json)?;
3750        validate_authority_reads(&new_schema, &self.limits.authority_reads)?;
3751        self.conn
3752            .execute_batch("BEGIN IMMEDIATE")
3753            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3754        let prior_schema = std::mem::replace(&mut self.schema, new_schema);
3755        // §7.4.3/§7.4.4: a reset inside this transaction mutates in-memory
3756        // state (dropped outbox commits, rejections, upgrade and floor flags,
3757        // overlay dirtiness) and appends change/intent batches. A COMMIT that
3758        // fails after the reset succeeded rolls the database back, so every
3759        // one of those fields is captured here and restored with it.
3760        let rollback = self.capture();
3761        let guarded = (|| -> Result<(), String> {
3762            // §7.4.2: the marker is re-read under the write lock the caller
3763            // holds, so a replica another process upgraded meanwhile refuses
3764            // this recreation instead of having its marker and tables
3765            // downgraded.
3766            let marker = read_local_schema_version(&self.conn, self.schema.version)?;
3767            self.create_bookkeeping_tables()?;
3768            match marker {
3769                Some(version) if version < self.schema.version => self.run_schema_reset()?,
3770                None => {
3771                    self.create_synced_tables()?;
3772                    self.persist_schema_version()?;
3773                }
3774                _ => {}
3775            }
3776            // §7.4.2: recreation IS the boot for the in-memory core, so reload the
3777            // durable outbox the same-version path is about to replay. Only the
3778            // outbox is reloaded: the recovery below reads it, and the caller's
3779            // rollback snapshot already covers this field.
3780            self.outbox = self.load_outbox()?.into();
3781            // §7.1/§7.4.4: recover intrinsically codec-invalid persisted values at
3782            // this guarded boundary once the schema and marker agree. A bump went
3783            // through `run_schema_reset`, which performs the same recovery after
3784            // it creates the new physical tables and before its final replay.
3785            self.drop_unencodable_outbox(false, None)?;
3786            if marker == Some(self.schema.version)
3787                && (!self.outbox.is_empty()
3788                    || !self.failed_commits.is_empty()
3789                    || self.has_acknowledged_rows()?)
3790            {
3791                self.overlay_dirty.set(true);
3792                self.rebuild_overlay()?;
3793            }
3794            self.prune_unknown_units(false)?;
3795            self.save_unit_scope_schema()?;
3796            // The conformance recreate is the in-memory equivalent of reopening
3797            // a durable client. Apply the same startup catch-up contract even
3798            // when the schema itself did not change.
3799            self.enqueue_startup_sync_if_needed()?;
3800            Ok(())
3801        })();
3802        let outcome = match guarded {
3803            Ok(()) => self
3804                .conn
3805                .execute_batch("COMMIT")
3806                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error)),
3807            Err(error) => Err(error),
3808        };
3809        if let Err(error) = outcome {
3810            if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
3811                // Keep a failed rollback's typed storage classification.
3812                let _ = Self::sqlite_failure(&self.storage_failure, rollback);
3813            }
3814            self.schema = prior_schema;
3815            self.restore(rollback);
3816            // Prepared insert statements were derived from the reverted schema.
3817            self.insert_sql.borrow_mut().clear();
3818            return Err(error);
3819        }
3820        if !self.security_preflight {
3821            self.drain_pending_evictions()?;
3822        }
3823        Ok(())
3824    }
3825
3826    fn enqueue_startup_sync_if_needed(&mut self) -> Result<(), String> {
3827        let startup_work = !self.stopped
3828            && (!self.outbox.is_empty()
3829                || self.has_acknowledged_rows()?
3830                || !self.load_pending_evictions()?.is_empty()
3831                || self
3832                    .stored(load_units(&self.conn))?
3833                    .iter()
3834                    .any(|unit| unit.status == UnitStatus::Active));
3835        if startup_work {
3836            self.sync_needed = true;
3837            self.sync_intent_queue.push_back(SyncIntent::Interactive);
3838        }
3839        Ok(())
3840    }
3841
3842    /// A native client persists schema-floor stops across process restarts.
3843    /// Once the running app already satisfies that floor, the persisted stop
3844    /// is only stale evidence from an older server round (for example, the
3845    /// server was restarted after catching up to an app that was ahead).
3846    /// Clear it and let the normal startup pull re-negotiate. If the server is
3847    /// still incompatible it will return the floor again in that first round.
3848    fn clear_satisfied_persisted_schema_floor(&mut self) -> Result<(), String> {
3849        let satisfied = self
3850            .schema_floor
3851            .as_ref()
3852            .and_then(|floor| floor.required_schema_version)
3853            .is_some_and(|required| self.schema.version >= required);
3854        if !satisfied {
3855            return Ok(());
3856        }
3857        self.delete_meta(SCHEMA_FLOOR_KEY)?;
3858        self.schema_floor = None;
3859        self.stopped = false;
3860        Ok(())
3861    }
3862
3863    /// RFC 0005 D1: discard an orphan or stale container at boot. The reset
3864    /// path already swept it; this covers a same-version or fresh-install open,
3865    /// which runs no reset and therefore no sweep — plus the D7 TTL, which only
3866    /// an aware binary can apply.
3867    fn reconcile_previous_version_at_boot(&self) -> Result<(), String> {
3868        let Some(replica_path) = self.previous_version_replica_path() else {
3869            return Ok(());
3870        };
3871        let max_age_ms = self
3872            .previous_version
3873            .filter(|config| config.enabled)
3874            .map(|config| config.max_age_ms);
3875        reconcile_previous_version_at_boot(
3876            &self.conn,
3877            &replica_path,
3878            self.schema.version,
3879            max_age_ms,
3880            self.clock_now_ms(),
3881        )
3882    }
3883
3884    /// RFC 0005 D3: the sibling container path, or `None` when the replica has
3885    /// no file — an in-memory replica returns an empty path, and a `file:` URI
3886    /// has no plain sibling path — so the feature resolves as off there.
3887    fn previous_version_replica_path(&self) -> Option<String> {
3888        let path = self.conn.path()?;
3889        if path.is_empty() || path.starts_with("file:") {
3890            return None;
3891        }
3892        Some(path.to_owned())
3893    }
3894
3895    /// §7.4.3 reset: whole-database local reset EXCEPT the outbox, clientId,
3896    /// and leaseState. Drops/recreates every synced table from the new
3897    /// schema, resets unit sync state (keeping registrations), clears
3898    /// the schema-floor stop state, rewrites the marker, drops outbox commits
3899    /// that cannot re-encode (§7.4.4), and replays the survivors on top.
3900    fn run_schema_reset(&mut self) -> Result<(), String> {
3901        self.begin_observation("syncular_schema_reset")?;
3902        let rollback = self.capture();
3903        let mut batch = ChangeAccumulator::default();
3904        let result = self
3905            .run_schema_reset_observed(&mut batch, true)
3906            .and_then(|()| self.finish_observation("syncular_schema_reset", batch));
3907        if let Err(error) = result {
3908            self.rollback_observation("syncular_schema_reset");
3909            self.restore(rollback);
3910            return Err(error);
3911        }
3912        // After the commit: a rolled-back reset keeps the in-flight round.
3913        self.cancel_sync_round();
3914        Ok(())
3915    }
3916
3917    fn run_log_epoch_reset(&mut self, log_epoch: &str) -> Result<Vec<String>, String> {
3918        if self.get_meta(LOG_EPOCH_KEY)?.is_none() {
3919            self.begin_observation("syncular_epoch_acquisition")?;
3920            let rollback = self.capture();
3921            if let Err(error) = self.conn.execute(
3922                "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
3923                rusqlite::params![LOG_EPOCH_KEY, log_epoch],
3924            ) {
3925                self.rollback_observation("syncular_epoch_acquisition");
3926                return Err(Self::sqlite_failure(&self.storage_failure, error));
3927            }
3928            self.sync_needed = true;
3929            let batch = ChangeAccumulator {
3930                status: true,
3931                ..ChangeAccumulator::default()
3932            };
3933            if let Err(error) = self.finish_observation("syncular_epoch_acquisition", batch) {
3934                self.rollback_observation("syncular_epoch_acquisition");
3935                self.restore(rollback);
3936                return Err(error);
3937            }
3938            self.sync_intent_queue.push_back(SyncIntent::Interactive);
3939            return Ok(Vec::new());
3940        }
3941        let resets = self
3942            .stored(load_units(&self.conn))?
3943            .into_iter()
3944            .map(|unit| unit.id)
3945            .collect();
3946        self.begin_observation("syncular_log_epoch_reset")?;
3947        let rollback = self.capture();
3948        let mut batch = ChangeAccumulator::default();
3949        let result = self
3950            .run_schema_reset_observed(&mut batch, false)
3951            .and_then(|()| {
3952                self.conn
3953                    .execute(
3954                        "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
3955                        rusqlite::params![LOG_EPOCH_KEY, log_epoch],
3956                    )
3957                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3958                self.sync_needed = true;
3959                batch.status = true;
3960                self.finish_observation("syncular_log_epoch_reset", batch)
3961            });
3962        if let Err(error) = result {
3963            self.rollback_observation("syncular_log_epoch_reset");
3964            self.restore(rollback);
3965            return Err(error);
3966        }
3967        self.cancel_sync_round();
3968        self.sync_intent_queue.push_back(SyncIntent::Interactive);
3969        Ok(resets)
3970    }
3971
3972    fn run_schema_reset_observed(
3973        &mut self,
3974        batch: &mut ChangeAccumulator,
3975        capture: bool,
3976    ) -> Result<(), String> {
3977        // §7.4.2: the reset is the only path that wipes local tables for a
3978        // version bump, and it rewrites the marker last. Re-read the marker
3979        // under this reset's transaction: a replica another process upgraded
3980        // meanwhile refuses here instead of being reset to this client's
3981        // older schema.
3982        read_local_schema_version(&self.conn, self.schema.version)?;
3983        // §7.4.3: a schema bump keeps every table whose generation equals the
3984        // persisted one; the log-epoch reset keeps nothing. The previous
3985        // declaration evidence decides which units cover only kept tables.
3986        let mut kept = BTreeSet::new();
3987        let mut previous_declarations: Option<BTreeMap<String, BTreeMap<String, Value>>> = None;
3988        if capture {
3989            let persisted: BTreeMap<String, String> = self
3990                .get_meta(TABLE_GENERATIONS_KEY)?
3991                .map(|raw| serde_json::from_str(&raw))
3992                .transpose()
3993                .map_err(|_| "sync.local_corrupt: invalid persisted table generations".to_owned())?
3994                .unwrap_or_default();
3995            kept = self
3996                .schema
3997                .tables
3998                .iter()
3999                .filter(|table| {
4000                    table.generation.is_some()
4001                        && table.generation.as_ref() == persisted.get(&table.name)
4002                })
4003                .map(|table| table.name.clone())
4004                .collect();
4005            previous_declarations = self
4006                .get_meta("subscriptionScopeSchema")?
4007                .map(|raw| serde_json::from_str(&raw))
4008                .transpose()
4009                .map_err(|_| "sync.local_corrupt: invalid persisted scope schema".to_owned())?;
4010        }
4011        // Kept tables keep both halves, their FTS projections with the FTS5
4012        // shadow tables and id mapping, and their row-level bookkeeping.
4013        let mut kept_objects = BTreeSet::new();
4014        for table in self
4015            .schema
4016            .tables
4017            .iter()
4018            .filter(|table| kept.contains(&table.name))
4019        {
4020            kept_objects.insert(table.name.clone());
4021            kept_objects.insert(format!("_syncular_base_{}", table.name));
4022            for index in &table.fts_indexes {
4023                kept_objects.insert(index.name.clone());
4024                for shadow in ["data", "idx", "content", "docsize", "config"] {
4025                    kept_objects.insert(format!("{}_{shadow}", index.name));
4026                }
4027            }
4028        }
4029        let kept_json = serde_json::to_string(&kept).expect("table names serialize");
4030        for sql in [
4031            "DELETE FROM _syncular_acked_rows WHERE tbl NOT IN (SELECT value FROM json_each(?1))",
4032            "DELETE FROM _syncular_row_deliveries WHERE tbl NOT IN (SELECT value FROM json_each(?1))",
4033        ] {
4034            self.conn
4035                .execute(sql, [&kept_json])
4036                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4037        }
4038        self.upgrading = true;
4039        batch.status = true;
4040        // RFC 0005 D6: classify the pending outbox against the NEW compiled
4041        // schema BEFORE the wipe, and record the advisory audit. This pass
4042        // drops nothing, so the audit sees the original pending intent; the
4043        // §7.4.4 send-time drop and the intrinsic value recovery later in this
4044        // reset are the paths that remove a commit. The classification is
4045        // shared with the send-time drop so the two cannot disagree about what
4046        // is incompatible.
4047        if self.previous_version.is_some_and(|config| config.enabled) {
4048            let previous_version = self
4049                .get_meta(LOCAL_SCHEMA_VERSION_KEY)?
4050                .and_then(|value| value.parse::<i32>().ok())
4051                .unwrap_or(self.schema.version);
4052            let pending = self
4053                .outbox
4054                .iter()
4055                .map(|commit| PendingCommitAudit {
4056                    commit_id: &commit.client_commit_id,
4057                    operations: commit_audit_operations(commit),
4058                })
4059                .collect::<Vec<_>>();
4060            crate::previous_version::write_previous_version_audit(
4061                &self.conn,
4062                &crate::previous_version::build_previous_version_audit(
4063                    &self.schema,
4064                    &pending,
4065                    previous_version,
4066                    self.schema.version,
4067                    self.clock_now_ms(),
4068                ),
4069            )?;
4070        }
4071        // RFC 0005 D5: the container lives in its own FILE, so the reset can
4072        // never reach it. Sweep first (unconditional — also on the log-epoch
4073        // reset, which captures nothing), then capture BEFORE the wipe drops
4074        // any row. A refused capture records its reason and leaves no file.
4075        if let Some(replica_path) = self.previous_version_replica_path() {
4076            sweep_previous_version_container(&self.conn, &replica_path)?;
4077            if capture {
4078                if let Some(config) = self.previous_version.filter(|config| config.enabled) {
4079                    let previous_version = self
4080                        .get_meta(LOCAL_SCHEMA_VERSION_KEY)?
4081                        .and_then(|value| value.parse::<i32>().ok())
4082                        .unwrap_or(self.schema.version);
4083                    capture_previous_version_from_replica(
4084                        &self.conn,
4085                        &replica_path,
4086                        previous_version,
4087                        self.schema.version,
4088                        &config,
4089                        self.clock_now_ms(),
4090                    )?;
4091                }
4092            }
4093        }
4094        for table in self
4095            .schema
4096            .tables
4097            .iter()
4098            .filter(|table| !kept.contains(&table.name))
4099        {
4100            batch.table(&table.name);
4101        }
4102        for (base_key, value) in self.load_window_registry()? {
4103            batch.window(&base_key, &value);
4104        }
4105        if capture {
4106            self.prune_unknown_units(true)?;
4107        }
4108        // The per-table insert SQL is derived from the OLD column lists.
4109        self.insert_sql.borrow_mut().clear();
4110        self.overlay_dirty.set(true);
4111        // Drop every synced table (base + visible) that currently exists —
4112        // discovered from sqlite_master so a bump that adds/removes tables is
4113        // handled. Bookkeeping tables (`_syncular_outbox/_units/_meta`
4114        // and the blob cache) are preserved; base tables are `_syncular_base_*`
4115        // so they are matched explicitly, not by the bookkeeping filter.
4116        // Drop virtual tables first so SQLite removes every FTS shadow table
4117        // atomically instead of the generic discovery tearing it apart.
4118        let virtual_tables: Vec<String> = {
4119            let mut stmt = self
4120                .conn
4121                .prepare(
4122                    "SELECT name FROM sqlite_master WHERE type = 'table' AND sql LIKE 'CREATE VIRTUAL TABLE%'",
4123                )
4124                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4125            let rows = stmt
4126                .query_map([], |row| row.get::<_, String>(0))
4127                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4128            rows.filter_map(Result::ok)
4129                .filter(|name| is_synced_table_name(name) && !kept_objects.contains(name))
4130                .collect()
4131        };
4132        for name in virtual_tables {
4133            self.conn
4134                .execute_batch(&format!(
4135                    "DROP TABLE IF EXISTS {}",
4136                    quote_ident(&format!("_syncular_fts_{name}"))
4137                ))
4138                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4139            self.conn
4140                .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
4141                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4142        }
4143        let existing: Vec<String> = {
4144            let mut stmt = self
4145                .conn
4146                .prepare("SELECT name FROM sqlite_master WHERE type = 'table'")
4147                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4148            let rows = stmt
4149                .query_map([], |row| row.get::<_, String>(0))
4150                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4151            rows.filter_map(Result::ok)
4152                .filter(|name| is_synced_table_name(name) && !kept_objects.contains(name))
4153                .collect()
4154        };
4155        for name in &existing {
4156            if let Some(table) = name.strip_prefix("_syncular_base_") {
4157                batch.table(table);
4158            } else if !name.starts_with("_syncular_") {
4159                batch.table(name);
4160            }
4161        }
4162        for name in existing {
4163            self.conn
4164                .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
4165                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4166        }
4167        // Recreate the synced tables from the NEW schema.
4168        self.create_synced_tables()?;
4169        // Reset the sync state of every unit that covers a table the reset
4170        // did not keep, under the previous declarations or the new schema,
4171        // keeping the registration.
4172        let keep_units = match &previous_declarations {
4173            Some(previous) if !kept.is_empty() => self
4174                .stored(load_units(&self.conn))?
4175                .into_iter()
4176                .filter(|unit| {
4177                    previous.iter().all(|(name, old)| {
4178                        kept.contains(name) || !unit.scopes.keys().all(|v| old.contains_key(v))
4179                    }) && self
4180                        .schema
4181                        .tables
4182                        .iter()
4183                        .all(|table| kept.contains(&table.name) || !covers(table, &unit.scopes))
4184                })
4185                .map(|unit| unit.id)
4186                .collect(),
4187            _ => BTreeSet::new(),
4188        };
4189        reset_units(&self.conn, &keep_units)
4190            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4191        // The stop state is over: this client now ships a servable schema.
4192        self.stopped = false;
4193        self.schema_floor = None;
4194        self.delete_meta(SCHEMA_FLOOR_KEY)?;
4195        // Rewrite the marker LAST so a crash mid-reset re-runs the reset.
4196        self.persist_schema_version()?;
4197        self.save_unit_scope_schema()?;
4198        // §7.1/§7.4.4: the previous-version capture and audit above saw the
4199        // original pending intent; the synced tables now carry the new schema, so
4200        // recover intrinsically codec-invalid persisted values here, before the
4201        // final replay. A pending upsert that names a column the new schema
4202        // removed stays pending for the send-time `drop_incompatible_outbox`
4203        // prepass. The reset wrapper snapshots and restores the outbox,
4204        // rejections, and change queues, so a failed commit rolls this recovery
4205        // back with the rest of the reset.
4206        // §7.4.4: the enclosing reset observation owns the publication, so the
4207        // recovery contributes its rejection, outcome, and table domains to it.
4208        self.drop_unencodable_outbox(false, Some(batch))?;
4209        // Re-apply the surviving outbox optimistically over the empty tables.
4210        self.rebuild_overlay()?;
4211        Ok(())
4212    }
4213
4214    /// §7.4.4 send-time prepass: a persisted upsert whose values reference a
4215    /// column the current schema lacks (or a removed table) cannot be encoded
4216    /// (`commit_audit_operations` inspects upserts, so a value-free delete stays
4217    /// encodable and is validated by the server), nor can an insert-intent
4218    /// upsert that lacks a non-nullable column (§7.1, the audit's
4219    /// `missing-required-column`). Drop the commit, undo
4220    /// its optimistic projection, and raise a client-local
4221    /// `sync.outbox_incompatible` rejection.
4222    fn drop_incompatible_outbox(&mut self) -> Result<bool, String> {
4223        let incompatible = self
4224            .outbox
4225            .iter()
4226            .filter(|commit| {
4227                first_incompatibility(&self.schema, &commit_audit_operations(commit)).is_some()
4228            })
4229            .cloned()
4230            .collect::<Vec<_>>();
4231        if incompatible.is_empty() {
4232            return Ok(false);
4233        }
4234        self.drop_outbox_commits(
4235            &incompatible,
4236            OUTBOX_INCOMPATIBLE_CODE,
4237            "the persisted commit cannot encode under the current schema",
4238        )?;
4239        Ok(true)
4240    }
4241
4242    /// §7.4.4 incompatible-commit path, also taken by a refused blob upload
4243    /// (§5.9.7) with the server's code: reject every operation of `commits`,
4244    /// remove them from the outbox, and undo their optimistic rows.
4245    fn drop_outbox_commits(
4246        &mut self,
4247        incompatible: &[Arc<OutboxCommit>],
4248        code: &str,
4249        message: &str,
4250    ) -> Result<(), String> {
4251        // §7.4.4: one observation covers the durable outcome writes, the outbox
4252        // removal, the in-memory drop, and the overlay rebuild, so a storage
4253        // fault leaves the replica and the client unchanged and publishes no
4254        // change; a successful drop publishes exactly one batch carrying the
4255        // rejections, outcomes, status, and the tables whose projection changed.
4256        let rollback = self.capture();
4257        let owns_transaction = self.conn.is_autocommit();
4258        self.begin_observation("syncular_outbox_drop")?;
4259        let mut batch = ChangeAccumulator::default();
4260        let dropped = (|| -> Result<(), String> {
4261            let mut rejections = Vec::new();
4262            for commit in incompatible {
4263                let results = commit
4264                    .ops
4265                    .iter()
4266                    .enumerate()
4267                    .map(|(op_index, operation)| {
4268                        let rejection = RejectionRecord {
4269                            client_commit_id: commit.client_commit_id.clone(),
4270                            op_index: op_index as i32,
4271                            code: code.to_owned(),
4272                            message: message.to_owned(),
4273                            retryable: false,
4274                            details: None,
4275                            operation: Some(CommitOperation::from(operation)),
4276                        };
4277                        rejections.push(rejection.clone());
4278                        CommitOperationOutcome::Error { rejection }
4279                    })
4280                    .collect::<Vec<_>>();
4281                self.persist_commit_outcome(
4282                    &commit.client_commit_id,
4283                    CommitOutcomeStatus::Rejected,
4284                    &results,
4285                    Some(&commit.ops),
4286                )?;
4287                self.delete_outbox_persisted(&commit.client_commit_id)?;
4288                // A removed table has no local projection to invalidate; every
4289                // surviving table the dropped commit touched does.
4290                for operation in &commit.ops {
4291                    if self.schema.table(&operation.table).is_some() {
4292                        batch.table(&operation.table);
4293                    }
4294                }
4295            }
4296            self.prune_commit_outcomes()?;
4297            let incompatible_ids = incompatible
4298                .iter()
4299                .map(|commit| commit.client_commit_id.as_str())
4300                .collect::<BTreeSet<_>>();
4301            self.outbox
4302                .retain(|commit| !incompatible_ids.contains(commit.client_commit_id.as_str()));
4303            self.rejections.extend(rejections);
4304            // The dropped commits' purely-optimistic rows are undone: the
4305            // visible projection is re-derived from the surviving base + outbox.
4306            self.overlay_dirty.set(true);
4307            self.rebuild_overlay()?;
4308            batch.status = true;
4309            batch.rejections = true;
4310            batch.outcomes = true;
4311            Ok(())
4312        })();
4313        let result = dropped.and_then(|()| self.finish_observation("syncular_outbox_drop", batch));
4314        if let Err(error) = result {
4315            // An outer RELEASE is the COMMIT. Roll back that transaction
4316            // directly: retrying RELEASE after ROLLBACK TO still needs the
4317            // reader's lock. A nested observation only rolls back its savepoint.
4318            if owns_transaction {
4319                if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
4320                    let mut failure = self.storage_failure.borrow_mut();
4321                    let failure =
4322                        failure.get_or_insert_with(|| QueryReadFailure::from(error.clone()));
4323                    failure.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
4324                }
4325            } else {
4326                self.rollback_observation("syncular_outbox_drop");
4327            }
4328            self.restore(rollback);
4329            return Err(error);
4330        }
4331        Ok(())
4332    }
4333
4334    /// §7.4.3: (re)create the base + visible table pair for every synced
4335    /// table in the CURRENT schema (idempotent — `IF NOT EXISTS`).
4336    fn create_synced_tables(&self) -> Result<(), String> {
4337        for table in &self.schema.tables {
4338            // The base half + the visible half form the synced-table pair. An
4339            // index name is global in SQLite, so the base half's indexes are
4340            // name-prefixed (`_syncular_base_<index>`) to stay distinct.
4341            for (full, index_prefix) in [
4342                (base_table(&table.name), "_syncular_base_"),
4343                (visible_table(&table.name), ""),
4344            ] {
4345                let mut cols: Vec<String> =
4346                    table.columns.iter().map(|c| quote_ident(&c.name)).collect();
4347                cols.push("\"_syncular_version\" INTEGER NOT NULL".to_owned());
4348                cols.extend(
4349                    table
4350                        .sealed_columns()
4351                        .iter()
4352                        .map(|name| format!("{} TEXT", quote_ident(name))),
4353                );
4354                let sql = format!(
4355                    "CREATE TABLE IF NOT EXISTS {full} ({} , PRIMARY KEY ({}))",
4356                    cols.join(", "),
4357                    quote_ident(&table.primary_key)
4358                );
4359                self.conn
4360                    .execute(&sql, [])
4361                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4362                // §5.11: a replica created before sealed apply gains the
4363                // reserved columns in place, on both halves in the same order,
4364                // so data, indexes and the FTS projection stay.
4365                for name in table.sealed_columns() {
4366                    let present: bool = self
4367                        .conn
4368                        .query_row(
4369                            "SELECT EXISTS(SELECT 1 FROM pragma_table_info(?1) WHERE name = ?2)",
4370                            rusqlite::params![format!("{index_prefix}{}", table.name), name],
4371                            |row| row.get(0),
4372                        )
4373                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4374                    if !present {
4375                        self.conn
4376                            .execute(
4377                                &format!(
4378                                    "ALTER TABLE {full} ADD COLUMN {} TEXT",
4379                                    quote_ident(name)
4380                                ),
4381                                [],
4382                            )
4383                            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4384                    }
4385                }
4386                // The reveal at every round start reads the sealed rows of the
4387                // base half through this partial index.
4388                if table.has_encrypted_columns() && !index_prefix.is_empty() {
4389                    self.conn
4390                        .execute(
4391                            &format!(
4392                                "CREATE INDEX IF NOT EXISTS {} ON {full} (\"_sync_sealed_key\") WHERE \"_sync_sealed_key\" IS NOT NULL",
4393                                quote_ident(&format!("_syncular_sealed_{}", table.name))
4394                            ),
4395                            [],
4396                        )
4397                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4398                }
4399                // Local secondary indexes (CREATE INDEX subset). Created on
4400                // both halves so mirror reads hit an index on either. Runs on
4401                // both the initial create and the §7.4.3 reset recreate path.
4402                for index in &table.indexes {
4403                    let unique = if index.unique { "UNIQUE " } else { "" };
4404                    let index_name = quote_ident(&format!("{index_prefix}{}", index.name));
4405                    let cols_sql = index
4406                        .columns
4407                        .iter()
4408                        .map(|c| quote_ident(c))
4409                        .collect::<Vec<_>>()
4410                        .join(", ");
4411                    let index_sql = format!(
4412                        "CREATE {unique}INDEX IF NOT EXISTS {index_name} ON {full} ({cols_sql})"
4413                    );
4414                    self.conn
4415                        .execute(&index_sql, [])
4416                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4417                }
4418            }
4419        }
4420        // FTS exists only on the visible half. It is a local search
4421        // projection, never a synced/base table or a wire-schema member.
4422        for table in &self.schema.tables {
4423            for index in &table.fts_indexes {
4424                self.create_fts_projection(table, index)?;
4425            }
4426        }
4427        Ok(())
4428    }
4429
4430    fn fts_projection_exists(&self, index: &FtsIndexSchema) -> Result<bool, String> {
4431        let count: i64 = self
4432            .conn
4433            .query_row(
4434                "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?1",
4435                rusqlite::params![index.name],
4436                |row| row.get(0),
4437            )
4438            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4439        Ok(count > 0)
4440    }
4441
4442    fn drop_fts_triggers(&self, index: &FtsIndexSchema) -> Result<(), String> {
4443        for suffix in ["ai", "ad", "au"] {
4444            self.conn
4445                .execute(
4446                    &format!(
4447                        "DROP TRIGGER IF EXISTS {}",
4448                        quote_ident(&format!("{}_{suffix}", index.name))
4449                    ),
4450                    [],
4451                )
4452                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4453        }
4454        Ok(())
4455    }
4456
4457    fn create_fts_triggers(
4458        &self,
4459        table: &TableSchema,
4460        index: &FtsIndexSchema,
4461    ) -> Result<(), String> {
4462        let fts = quote_ident(&index.name);
4463        let source = visible_table(&table.name);
4464        let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
4465        let pk = quote_ident(&table.primary_key);
4466        let projection_columns = std::iter::once(source_id.clone())
4467            .chain(index.columns.iter().map(|column| quote_ident(column)))
4468            .collect::<Vec<_>>()
4469            .join(", ");
4470        let new_values = std::iter::once(format!("CAST(new.{pk} AS TEXT)"))
4471            .chain(
4472                index
4473                    .columns
4474                    .iter()
4475                    .map(|column| format!("new.{}", quote_ident(column))),
4476            )
4477            .collect::<Vec<_>>()
4478            .join(", ");
4479        let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
4480        let delete_new = format!("DELETE FROM {fts} WHERE rowid = (SELECT id FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)); DELETE FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)");
4481        let delete_old = format!("DELETE FROM {fts} WHERE rowid = (SELECT id FROM {mapping} WHERE source_id = CAST(old.{pk} AS TEXT)); DELETE FROM {mapping} WHERE source_id = CAST(old.{pk} AS TEXT)");
4482        let insert_new = format!("INSERT INTO {mapping}(source_id) VALUES (CAST(new.{pk} AS TEXT)); INSERT INTO {fts} (rowid, {projection_columns}) VALUES ((SELECT id FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)), {new_values})");
4483        let bi = quote_ident(&format!("{}_bi", index.name));
4484        let ai = quote_ident(&format!("{}_ai", index.name));
4485        let ad = quote_ident(&format!("{}_ad", index.name));
4486        let au = quote_ident(&format!("{}_au", index.name));
4487        let replacement_exists = format!("EXISTS (SELECT 1 FROM {source} WHERE {pk} = new.{pk})");
4488        let sql = format!(
4489            "DROP TRIGGER IF EXISTS {bi};
4490             DROP TRIGGER IF EXISTS {ai};
4491             DROP TRIGGER IF EXISTS {ad};
4492             DROP TRIGGER IF EXISTS {au};
4493             CREATE TRIGGER {bi} BEFORE INSERT ON {source} WHEN {replacement_exists} BEGIN {delete_new}; END;
4494             CREATE TRIGGER {ai} AFTER INSERT ON {source} BEGIN {insert_new}; END;
4495             CREATE TRIGGER {ad} AFTER DELETE ON {source} BEGIN {delete_old}; END;
4496             CREATE TRIGGER {au} AFTER UPDATE ON {source} BEGIN {delete_old}; {delete_new}; {insert_new}; END;",
4497        );
4498        self.conn
4499            .execute_batch(&sql)
4500            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
4501    }
4502
4503    fn rebuild_fts_projection(
4504        &self,
4505        table: &TableSchema,
4506        index: &FtsIndexSchema,
4507    ) -> Result<(), String> {
4508        let fts = quote_ident(&index.name);
4509        let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
4510        let pk = quote_ident(&table.primary_key);
4511        let indexed_columns = index
4512            .columns
4513            .iter()
4514            .map(|column| quote_ident(column))
4515            .collect::<Vec<_>>();
4516        let projection_columns = std::iter::once(source_id.clone())
4517            .chain(indexed_columns.iter().cloned())
4518            .collect::<Vec<_>>()
4519            .join(", ");
4520        let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
4521        let sql = format!(
4522            "DELETE FROM {fts}; DELETE FROM {mapping}; INSERT INTO {fts} ({projection_columns}) SELECT CAST({pk} AS TEXT), {columns} FROM {source}; INSERT INTO {mapping}(id, source_id) SELECT rowid, {source_id} FROM {fts};",
4523            columns = indexed_columns.join(", "),
4524            source = visible_table(&table.name),
4525        );
4526        self.conn
4527            .execute_batch(&sql)
4528            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
4529    }
4530
4531    fn create_fts_projection(
4532        &self,
4533        table: &TableSchema,
4534        index: &FtsIndexSchema,
4535    ) -> Result<(), String> {
4536        self.begin_observation("syncular_fts_schema")?;
4537        let result = (|| {
4538            let existed = self.fts_projection_exists(index)?;
4539            let tokenizer = index.tokenize.replace('\'', "''");
4540            let columns = index
4541                .columns
4542                .iter()
4543                .map(|column| quote_ident(column))
4544                .collect::<Vec<_>>()
4545                .join(", ");
4546            let sql = format!(
4547            "CREATE VIRTUAL TABLE IF NOT EXISTS {fts} USING fts5({source_id} UNINDEXED, {columns}, tokenize='{tokenizer}')",
4548            fts = quote_ident(&index.name),
4549            source_id = quote_ident(FTS_SOURCE_ID_COLUMN),
4550        );
4551            self.conn.execute(&sql, []).map_err(|error| {
4552                format!(
4553                    "cannot create local FTS5 projection {:?}: {error}",
4554                    index.name
4555                )
4556            })?;
4557            let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
4558            let mapped: bool = self
4559                .conn
4560                .query_row(
4561                    "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?1)",
4562                    [format!("_syncular_fts_{}", index.name)],
4563                    |row| row.get(0),
4564                )
4565                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4566            self.conn.execute_batch(&format!("CREATE TABLE IF NOT EXISTS {mapping}(id INTEGER PRIMARY KEY, source_id TEXT NOT NULL UNIQUE)")).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4567            if !mapped && existed {
4568                self.conn
4569                    .execute_batch(&format!(
4570                        "INSERT INTO {mapping}(id, source_id) SELECT rowid, {} FROM {}",
4571                        quote_ident(FTS_SOURCE_ID_COLUMN),
4572                        quote_ident(&index.name)
4573                    ))
4574                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4575            }
4576            self.create_fts_triggers(table, index)?;
4577            // Every visible row has exactly one mapping row, so a count
4578            // mismatch marks a projection an earlier release damaged; the
4579            // open rebuilds it.
4580            let stale: bool = !existed
4581                || self
4582                    .conn
4583                    .query_row(
4584                        &format!(
4585                            "SELECT (SELECT count(*) FROM {mapping}) != (SELECT count(*) FROM {})",
4586                            visible_table(&table.name)
4587                        ),
4588                        [],
4589                        |row| row.get(0),
4590                    )
4591                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4592            if stale {
4593                self.rebuild_fts_projection(table, index)?;
4594            }
4595            Ok(())
4596        })();
4597        if let Err(error) = result {
4598            self.rollback_observation("syncular_fts_schema");
4599            return Err(error);
4600        }
4601        if let Err(error) = self.conn.execute_batch("RELEASE syncular_fts_schema") {
4602            self.rollback_observation("syncular_fts_schema");
4603            return Err(error.to_string());
4604        }
4605        Ok(())
4606    }
4607
4608    // -- persistence write-through --------------------------------------------
4609
4610    fn persist_outbox_insert(&self, commit: &OutboxCommit) -> Result<(), String> {
4611        let ops: Vec<Value> = commit
4612            .ops
4613            .iter()
4614            .map(|op| {
4615                let mut entry = serde_json::json!({
4616                    "op": if op.upsert { "upsert" } else { "delete" },
4617                    "table": op.table,
4618                    "rowId": op.row_id,
4619                    "baseVersion": op.base_version,
4620                    "values": op.values.clone().map(Value::Object),
4621                });
4622                if op.insert {
4623                    entry["insert"] = Value::Bool(true);
4624                }
4625                entry
4626            })
4627            .collect();
4628        self.conn
4629            .execute(
4630                "INSERT OR REPLACE INTO _syncular_outbox (commit_id, ops_json) VALUES (?1, ?2)",
4631                rusqlite::params![commit.client_commit_id, Value::Array(ops).to_string()],
4632            )
4633            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4634        for operation in &commit.ops {
4635            let Some(values) = &operation.values else {
4636                continue;
4637            };
4638            let Some(table) = self.schema.table(&operation.table) else {
4639                continue;
4640            };
4641            for column in table
4642                .columns
4643                .iter()
4644                .filter(|column| column.ty == ColumnType::BlobRef)
4645            {
4646                let Some(Value::String(raw)) = values.get(&column.name) else {
4647                    continue;
4648                };
4649                let Ok(blob_ref) = serde_json::from_str::<Value>(raw) else {
4650                    continue;
4651                };
4652                let Some(blob_id) = blob_ref.get("blobId").and_then(Value::as_str) else {
4653                    continue;
4654                };
4655                self.conn
4656                    .execute(
4657                        "INSERT OR IGNORE INTO _syncular_blob_commit_refs(commit_id, blob_id)
4658                         SELECT ?1, ?2 WHERE EXISTS (SELECT 1 FROM _syncular_blobs WHERE blob_id = ?2)",
4659                        rusqlite::params![commit.client_commit_id, blob_id],
4660                    )
4661                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4662            }
4663        }
4664        Ok(())
4665    }
4666
4667    fn delete_outbox_persisted(&self, client_commit_id: &str) -> Result<(), String> {
4668        self.conn
4669            .execute(
4670                "DELETE FROM _syncular_blob_commit_refs WHERE commit_id = ?1",
4671                rusqlite::params![client_commit_id],
4672            )
4673            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4674        self.conn
4675            .execute(
4676                "DELETE FROM _syncular_outbox WHERE commit_id = ?1",
4677                rusqlite::params![client_commit_id],
4678            )
4679            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4680        self.conn.execute("DELETE FROM _syncular_row_deliveries WHERE NOT EXISTS(SELECT 1 FROM _syncular_outbox,json_each(ops_json) WHERE json_extract(json_each.value,'$.table')=tbl AND json_extract(json_each.value,'$.rowId')=id)", [])
4681            .map(|_| ())
4682            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
4683    }
4684
4685    fn outcome_status_name(status: CommitOutcomeStatus) -> &'static str {
4686        match status {
4687            CommitOutcomeStatus::Applied => "applied",
4688            CommitOutcomeStatus::Cached => "cached",
4689            CommitOutcomeStatus::Conflict => "conflict",
4690            CommitOutcomeStatus::Rejected => "rejected",
4691        }
4692    }
4693
4694    fn outcome_resolution_name(resolution: CommitOutcomeResolution) -> &'static str {
4695        match resolution {
4696            CommitOutcomeResolution::Active => "active",
4697            CommitOutcomeResolution::ResolvedKeepServer => "resolved_keep_server",
4698            CommitOutcomeResolution::Superseded => "superseded",
4699            CommitOutcomeResolution::Dismissed => "dismissed",
4700        }
4701    }
4702
4703    fn parse_outcome_status(value: &str) -> Result<CommitOutcomeStatus, String> {
4704        match value {
4705            "applied" => Ok(CommitOutcomeStatus::Applied),
4706            "cached" => Ok(CommitOutcomeStatus::Cached),
4707            "conflict" => Ok(CommitOutcomeStatus::Conflict),
4708            "rejected" => Ok(CommitOutcomeStatus::Rejected),
4709            _ => Err(format!("invalid persisted commit outcome status {value:?}")),
4710        }
4711    }
4712
4713    fn parse_outcome_resolution(value: &str) -> Result<CommitOutcomeResolution, String> {
4714        match value {
4715            "active" => Ok(CommitOutcomeResolution::Active),
4716            "resolved_keep_server" => Ok(CommitOutcomeResolution::ResolvedKeepServer),
4717            "superseded" => Ok(CommitOutcomeResolution::Superseded),
4718            "dismissed" => Ok(CommitOutcomeResolution::Dismissed),
4719            _ => Err(format!(
4720                "invalid persisted commit outcome resolution {value:?}"
4721            )),
4722        }
4723    }
4724
4725    fn persist_commit_outcome(
4726        &self,
4727        client_commit_id: &str,
4728        status: CommitOutcomeStatus,
4729        results: &[CommitOperationOutcome],
4730        operations: Option<&[OutboxOp]>,
4731    ) -> Result<(), String> {
4732        let results_json = serde_json::to_string(results).map_err(|error| error.to_string())?;
4733        let operations_json = operations
4734            .map(|items| {
4735                serde_json::to_string(&items.iter().map(CommitOperation::from).collect::<Vec<_>>())
4736            })
4737            .transpose()
4738            .map_err(|error| error.to_string())?;
4739        self.conn
4740            .execute(
4741                "INSERT INTO _syncular_commit_outcomes (
4742                   client_commit_id, status, recorded_at_ms, results_json,
4743                   operations_json, resolution
4744                 ) VALUES (?1, ?2, ?3, ?4, ?5, 'active')",
4745                rusqlite::params![
4746                    client_commit_id,
4747                    Self::outcome_status_name(status),
4748                    self.clock_now_ms(),
4749                    results_json,
4750                    operations_json
4751                ],
4752            )
4753            .map(|_| ())
4754            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
4755    }
4756
4757    /// §7.2: the journal holds a stored cell only in its canonical shape. A cell
4758    /// the byte-envelope rule refuses cannot be journaled verbatim.
4759    fn stored_cell_is_representable(cell: &Value) -> bool {
4760        cell.is_null()
4761            || cell.is_string()
4762            || cell.is_boolean()
4763            || cell.is_number()
4764            || is_canonical_bytes_envelope(cell)
4765    }
4766
4767    fn stored_row_is_representable(value: &Value) -> bool {
4768        value
4769            .as_object()
4770            .is_some_and(|row| row.values().all(Self::stored_cell_is_representable))
4771    }
4772
4773    fn stored_operation_is_representable(value: &Value) -> bool {
4774        value.get("table").is_some_and(Value::is_string)
4775            && value.get("rowId").is_some_and(Value::is_string)
4776            && value.get("baseVersion").is_none_or(|v| {
4777                v.as_i64()
4778                    .is_some_and(|n| n.unsigned_abs() <= MAX_JS_SAFE_INTEGER)
4779            })
4780            && match value.get("op").and_then(Value::as_str) {
4781                Some("upsert") => {
4782                    value
4783                        .get("values")
4784                        .is_some_and(Self::stored_row_is_representable)
4785                        && value
4786                            .get("insert")
4787                            .is_none_or(|insert| *insert == Value::Bool(true))
4788                }
4789                Some("delete") => value.get("values").is_none() && value.get("insert").is_none(),
4790                _ => false,
4791            }
4792    }
4793
4794    /// §7.2/§7.4.4: a dropped commit keeps its operation envelope in the journal
4795    /// only when the journal reader accepts the serialized operation, which is the
4796    /// rule `outcome_from_row` applies to this same row later.
4797    fn outbox_op_is_journalable(operation: &OutboxOp) -> bool {
4798        serde_json::to_value(CommitOperation::from(operation))
4799            .is_ok_and(|value| Self::stored_operation_is_representable(&value))
4800    }
4801
4802    fn outcome_from_row(row: StoredCommitOutcomeRow) -> Result<CommitOutcome, String> {
4803        let StoredCommitOutcomeRow {
4804            sequence,
4805            client_commit_id,
4806            status,
4807            recorded_at_ms,
4808            results_json,
4809            operations_json,
4810            resolution,
4811            resolved_at_ms,
4812            replacement_client_commit_id,
4813        } = row;
4814        if sequence.unsigned_abs() > MAX_JS_SAFE_INTEGER
4815            || recorded_at_ms.unsigned_abs() > MAX_JS_SAFE_INTEGER
4816            || resolved_at_ms.is_some_and(|value| value.unsigned_abs() > MAX_JS_SAFE_INTEGER)
4817        {
4818            return Err("sync.local_corrupt: persisted commit outcome is invalid".to_owned());
4819        }
4820        let invalid = || "sync.local_corrupt: persisted commit outcome is invalid".to_owned();
4821        let stored: Vec<Value> = serde_json::from_str(&results_json).map_err(|_| invalid())?;
4822        for result in &stored {
4823            let nested = match result.get("status").and_then(Value::as_str) {
4824                Some("conflict") => result.get("conflict"),
4825                Some("error") => result.get("rejection"),
4826                _ => None,
4827            };
4828            if nested
4829                .and_then(|record| record.get("operation"))
4830                .is_some_and(|op| !Self::stored_operation_is_representable(op))
4831                || (result.get("status").and_then(Value::as_str) == Some("conflict")
4832                    && nested
4833                        .and_then(|record| record.get("serverRow"))
4834                        .is_some_and(|row| !Self::stored_row_is_representable(row)))
4835            {
4836                return Err(invalid());
4837            }
4838        }
4839        let results: Vec<CommitOperationOutcome> =
4840            serde_json::from_value(Value::Array(stored)).map_err(|_| invalid())?;
4841        let has_conflict = results
4842            .iter()
4843            .any(|result| matches!(result, CommitOperationOutcome::Conflict { .. }));
4844        let has_error = results
4845            .iter()
4846            .any(|result| matches!(result, CommitOperationOutcome::Error { .. }));
4847        let status = Self::parse_outcome_status(&status).map_err(|_| invalid())?;
4848        let redacted = results.is_empty() && operations_json.is_none();
4849        if match status {
4850            CommitOutcomeStatus::Applied | CommitOutcomeStatus::Cached => has_conflict || has_error,
4851            CommitOutcomeStatus::Conflict => !(has_conflict || redacted),
4852            CommitOutcomeStatus::Rejected => has_conflict || !(has_error || redacted),
4853        } {
4854            return Err(invalid());
4855        }
4856        let operations: Option<Vec<CommitOperation>> = operations_json
4857            .map(|raw| {
4858                let stored: Vec<Value> = serde_json::from_str(&raw).map_err(|_| invalid())?;
4859                if !stored.iter().all(Self::stored_operation_is_representable) {
4860                    return Err(invalid());
4861                }
4862                serde_json::from_value(Value::Array(stored)).map_err(|_| invalid())
4863            })
4864            .transpose()?;
4865        for result in &results {
4866            match result {
4867                CommitOperationOutcome::Conflict { conflict }
4868                    if conflict.server_version.unsigned_abs() > MAX_JS_SAFE_INTEGER =>
4869                {
4870                    return Err("sync.local_corrupt: persisted commit outcome is invalid".to_owned())
4871                }
4872                CommitOperationOutcome::Error { rejection } => {
4873                    if let Some(details) = &rejection.details {
4874                        let encoded = serde_json::to_string(details).map_err(|_| {
4875                            "sync.local_corrupt: persisted commit outcome is invalid".to_owned()
4876                        })?;
4877                        RejectionDetails::parse(&encoded).map_err(|_| {
4878                            "sync.local_corrupt: persisted commit outcome is invalid".to_owned()
4879                        })?;
4880                    }
4881                }
4882                _ => {}
4883            }
4884        }
4885        Ok(CommitOutcome {
4886            sequence,
4887            client_commit_id,
4888            status,
4889            recorded_at_ms,
4890            results,
4891            operations,
4892            retained_rows: None,
4893            resolution: Self::parse_outcome_resolution(&resolution)?,
4894            resolved_at_ms,
4895            replacement_client_commit_id,
4896        })
4897    }
4898
4899    /// Resolve declared unique keys through SQLite, including affinity and NULL semantics.
4900    fn unique_conflicts(
4901        &self,
4902        table: &TableSchema,
4903        values: &Map<String, Value>,
4904        base: bool,
4905    ) -> Result<Vec<RetainedUniqueConflict>, String> {
4906        let source = if base {
4907            base_table(&table.name)
4908        } else {
4909            visible_table(&table.name)
4910        };
4911        let mut conflicts = Vec::new();
4912        for index in table.indexes.iter().filter(|index| index.unique) {
4913            let mut params = vec![json_param_to_sql(
4914                values.get(&table.primary_key).ok_or("sync.local_corrupt")?,
4915            )?];
4916            for name in &index.columns {
4917                let column = table
4918                    .columns
4919                    .iter()
4920                    .find(|column| column.name == *name)
4921                    .ok_or("sync.local_corrupt")?;
4922                let cell = json_to_column_value(column, values.get(name))?;
4923                params.push(owned_sql_value(RowParam::Cell(&cell))?);
4924            }
4925            let sql = format!(
4926                "SELECT * FROM {source} WHERE {} != ? AND {}",
4927                quote_ident(&table.primary_key),
4928                index
4929                    .columns
4930                    .iter()
4931                    .map(|column| format!("{} = ?", quote_ident(column)))
4932                    .collect::<Vec<_>>()
4933                    .join(" AND ")
4934            );
4935            let mut statement = self
4936                .conn
4937                .prepare_cached(&sql)
4938                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4939            let records = statement
4940                .query_map(rusqlite::params_from_iter(params), |record| {
4941                    let values = table
4942                        .columns
4943                        .iter()
4944                        .enumerate()
4945                        .map(|(index, column)| {
4946                            let value = sql_ref_to_json(column, record.get_ref(index)?);
4947                            json_to_column_value(column, Some(&value)).map_err(|message| {
4948                                rusqlite::Error::ToSqlConversionFailure(message.into())
4949                            })
4950                        })
4951                        .collect::<Result<Row, _>>()?;
4952                    Ok((values, record.get::<_, i64>(table.columns.len())?))
4953                })
4954                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4955            for record in records {
4956                let (row, version) =
4957                    record.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4958                let row_id = render_row_id_json(Some(&column_value_to_json(&row[table.pk_index])))?;
4959                conflicts.push(RetainedUniqueConflict {
4960                    index: index.name.clone(),
4961                    columns: index.columns.clone(),
4962                    row_id,
4963                    server_row: table
4964                        .columns
4965                        .iter()
4966                        .zip(&row)
4967                        .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
4968                        .collect(),
4969                    server_version: version,
4970                });
4971            }
4972        }
4973        Ok(conflicts)
4974    }
4975
4976    fn attach_retained_rows(&self, mut outcome: CommitOutcome) -> Result<CommitOutcome, String> {
4977        if let Some((commit, initial)) = self
4978            .failed_commits
4979            .iter()
4980            .find(|(commit, _)| commit.client_commit_id == outcome.client_commit_id)
4981        {
4982            let mut rows = Vec::new();
4983            for operation in &commit.ops {
4984                // A table the current schema removed has no rows to retain.
4985                let Some(table) = self.schema.table(&operation.table) else {
4986                    continue;
4987                };
4988                let key =
4989                    serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
4990                        .map_err(|_| "sync.local_corrupt".to_owned())?;
4991                let base = self.stored_row(table, &operation.row_id, true)?;
4992                rows.push(RetainedCommitRow {
4993                    table: operation.table.clone(),
4994                    row_id: operation.row_id.clone(),
4995                    unique_conflicts: if operation.upsert {
4996                        self.unique_conflicts(
4997                            table,
4998                            initial
4999                                .get(&key)
5000                                .and_then(Value::as_object)
5001                                .or(operation.values.as_ref())
5002                                .ok_or("sync.local_corrupt")?,
5003                            true,
5004                        )?
5005                    } else {
5006                        Vec::new()
5007                    },
5008                    local_row: if operation.upsert {
5009                        initial
5010                            .get(&key)
5011                            .and_then(Value::as_object)
5012                            .cloned()
5013                            .or_else(|| operation.values.clone())
5014                    } else {
5015                        None
5016                    },
5017                    server_row: base.as_ref().map(|(values, _)| {
5018                        table
5019                            .columns
5020                            .iter()
5021                            .zip(values)
5022                            .map(|(column, value)| {
5023                                (column.name.clone(), column_value_to_json(value))
5024                            })
5025                            .collect()
5026                    }),
5027                    server_version: base.map(|(_, version)| version),
5028                });
5029            }
5030            outcome.retained_rows = Some(rows);
5031        }
5032        Ok(outcome)
5033    }
5034
5035    pub fn commit_outcome(&self, client_commit_id: &str) -> Result<Option<CommitOutcome>, String> {
5036        let row = self
5037            .conn
5038            .query_row(
5039                "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
5040                        resolution, resolved_at_ms, replacement_client_commit_id
5041                   FROM _syncular_commit_outcomes WHERE client_commit_id = ?1",
5042                rusqlite::params![client_commit_id],
5043                |row| {
5044                    Ok(StoredCommitOutcomeRow {
5045                        sequence: row.get(0)?,
5046                        client_commit_id: row.get(1)?,
5047                        status: row.get(2)?,
5048                        recorded_at_ms: row.get(3)?,
5049                        results_json: row.get(4)?,
5050                        operations_json: row.get(5)?,
5051                        resolution: row.get(6)?,
5052                        resolved_at_ms: row.get(7)?,
5053                        replacement_client_commit_id: row.get(8)?,
5054                    })
5055                },
5056            )
5057            .optional()
5058            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5059        row.map(|row| self.attach_retained_rows(Self::outcome_from_row(row)?))
5060            .transpose()
5061    }
5062
5063    /// Internal host step. The host must run its idle drive before finishing.
5064    #[doc(hidden)]
5065    pub fn prepare_current_base(
5066        &self,
5067        client_commit_id: &str,
5068        sequence: i64,
5069    ) -> Result<CurrentBasePreparation, CurrentBaseProof> {
5070        let outcome = self
5071            .commit_outcome(client_commit_id)
5072            .map_err(|_| CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed))?;
5073        let valid = outcome.as_ref().is_some_and(|outcome| {
5074            outcome.sequence == sequence
5075                && outcome.resolution == CommitOutcomeResolution::Active
5076                && matches!(
5077                    outcome.status,
5078                    CommitOutcomeStatus::Conflict | CommitOutcomeStatus::Rejected
5079                )
5080                && outcome
5081                    .operations
5082                    .as_ref()
5083                    .zip(outcome.retained_rows.as_ref())
5084                    .is_some_and(|(operations, rows)| {
5085                        !rows.is_empty() && operations.len() == rows.len()
5086                    })
5087        });
5088        if !valid {
5089            return Err(CurrentBaseProof::failure(
5090                CurrentBaseProofReason::OutcomeUnavailable,
5091            ));
5092        }
5093        if !self.transport_enabled {
5094            return Err(CurrentBaseProof::failure(CurrentBaseProofReason::Offline));
5095        }
5096        Ok(CurrentBasePreparation {
5097            client_commit_id: client_commit_id.to_owned(),
5098            sequence,
5099            units: load_units(&self.conn)
5100                .map_err(|_| CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed))?,
5101            transport_epoch: self.transport_epoch,
5102            registration_epoch: self.registration_epoch,
5103            security_epoch: self.security_epoch,
5104        })
5105    }
5106
5107    /// Internal host step after the completed post-rejection idle drive.
5108    #[doc(hidden)]
5109    pub fn finish_current_base(
5110        &mut self,
5111        prepared: CurrentBasePreparation,
5112        drive: SyncOutcome,
5113    ) -> CurrentBaseProof {
5114        if !self.transport_enabled || self.transport_epoch != prepared.transport_epoch {
5115            return CurrentBaseProof::failure(CurrentBaseProofReason::Offline);
5116        }
5117        if self.registration_epoch != prepared.registration_epoch {
5118            return CurrentBaseProof::failure(CurrentBaseProofReason::MissingCoverage);
5119        }
5120        if self.security_preflight || self.security_epoch != prepared.security_epoch {
5121            return CurrentBaseProof::failure(CurrentBaseProofReason::SyncFailed);
5122        }
5123        let report = match drive {
5124            SyncOutcome::Ok(report) => report,
5125            SyncOutcome::BudgetExhausted(_) => {
5126                return CurrentBaseProof::failure(CurrentBaseProofReason::BudgetExhausted)
5127            }
5128            _ => return CurrentBaseProof::failure(CurrentBaseProofReason::SyncFailed),
5129        };
5130        if report.schema_floor.is_some() {
5131            return CurrentBaseProof::failure(CurrentBaseProofReason::SchemaFloor);
5132        }
5133        if self
5134            .conn
5135            .execute_batch("SAVEPOINT syncular_base_proof")
5136            .is_err()
5137        {
5138            return CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed);
5139        }
5140        let proof = (|| {
5141            let snapshot = snapshot_read_connection(
5142                &self.conn,
5143                &SnapshotReadRequest {
5144                    units: prepared.units.iter().map(|unit| unit.id.clone()).collect(),
5145                    commit_ids: vec![prepared.client_commit_id.clone()],
5146                    ..Default::default()
5147                },
5148            )
5149            .map_err(|_| CurrentBaseProofReason::SnapshotFailed)?;
5150            let current = self
5151                .commit_outcome(&prepared.client_commit_id)
5152                .map_err(|_| CurrentBaseProofReason::SnapshotFailed)?;
5153            let unavailable = if report.revoked.is_empty() {
5154                CurrentBaseProofReason::OutcomeUnavailable
5155            } else {
5156                CurrentBaseProofReason::Revoked
5157            };
5158            let Some(outcome) = current else {
5159                return Err(unavailable);
5160            };
5161            if !matches!(
5162                snapshot.deliveries.first(),
5163                Some(CommitDelivery::Known { .. })
5164            ) || outcome.sequence != prepared.sequence
5165                || outcome.resolution != CommitOutcomeResolution::Active
5166                || !matches!(
5167                    outcome.status,
5168                    CommitOutcomeStatus::Conflict | CommitOutcomeStatus::Rejected
5169                )
5170            {
5171                return Err(unavailable);
5172            }
5173            let (Some(operations), Some(rows)) = (outcome.operations, outcome.retained_rows) else {
5174                return Err(unavailable);
5175            };
5176            if rows.is_empty() || operations.len() != rows.len() {
5177                return Err(unavailable);
5178            }
5179            let after =
5180                load_units(&self.conn).map_err(|_| CurrentBaseProofReason::SnapshotFailed)?;
5181            let complete: HashSet<&str> = snapshot
5182                .units
5183                .iter()
5184                .filter_map(|unit| match unit {
5185                    UnitCatchup::Known {
5186                        id,
5187                        bootstrap_complete: true,
5188                        ..
5189                    } => Some(id.as_str()),
5190                    _ => None,
5191                })
5192                .collect();
5193            // A unit id names one scope map (§4.1), so the same id before and
5194            // after the drive is the same registration.
5195            let candidates: Vec<&UnitRecord> = after
5196                .iter()
5197                .filter(|unit| {
5198                    complete.contains(unit.id.as_str())
5199                        && prepared.units.iter().any(|before| before.id == unit.id)
5200                })
5201                .collect();
5202            let covered = |table_name: &str, row: &Map<String, Value>| {
5203                let Some(table) = self.schema.table(table_name) else {
5204                    return false;
5205                };
5206                candidates.iter().any(|unit| {
5207                    covers(table, &unit.scopes)
5208                        && pair_scopes(&unit.scopes, table_name, unit.echo.as_deref()).is_some_and(
5209                            |scopes| {
5210                                scopes.iter().all(|(variable, allowed)| {
5211                                    table
5212                                        .scope_column(variable)
5213                                        .and_then(|column| match row.get(column) {
5214                                            Some(Value::String(value)) => Some(value.clone()),
5215                                            Some(Value::Bool(value)) => Some(value.to_string()),
5216                                            Some(Value::Number(value)) => {
5217                                                value.as_f64().map(|number| {
5218                                                    ryu_js::Buffer::new()
5219                                                        .format_finite(number)
5220                                                        .to_owned()
5221                                                })
5222                                            }
5223                                            _ => None,
5224                                        })
5225                                        .is_some_and(|value| {
5226                                            !value.is_empty() && allowed.contains(&value)
5227                                        })
5228                                })
5229                            },
5230                        )
5231                })
5232            };
5233            for row in &rows {
5234                let Some(base) = row.server_row.as_ref().or(row.local_row.as_ref()) else {
5235                    return Err(if report.revoked.is_empty() {
5236                        CurrentBaseProofReason::MissingCoverage
5237                    } else {
5238                        CurrentBaseProofReason::Revoked
5239                    });
5240                };
5241                if !covered(&row.table, base)
5242                    || row
5243                        .unique_conflicts
5244                        .iter()
5245                        .any(|conflict| !covered(&row.table, &conflict.server_row))
5246                {
5247                    return Err(if report.revoked.is_empty() {
5248                        CurrentBaseProofReason::MissingCoverage
5249                    } else {
5250                        CurrentBaseProofReason::Revoked
5251                    });
5252                }
5253            }
5254            Ok(rows)
5255        })();
5256        if self
5257            .conn
5258            .execute_batch("RELEASE syncular_base_proof")
5259            .is_err()
5260        {
5261            return CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed);
5262        }
5263        match proof {
5264            Ok(rows) => CurrentBaseProof {
5265                proven: true,
5266                rows: Some(rows),
5267                reason: None,
5268            },
5269            Err(reason) => CurrentBaseProof::failure(reason),
5270        }
5271    }
5272
5273    pub fn prove_current_base(
5274        &mut self,
5275        transport: &mut dyn Transport,
5276        client_commit_id: &str,
5277        sequence: i64,
5278        max_rounds: Option<u32>,
5279    ) -> CurrentBaseProof {
5280        let prepared = match self.prepare_current_base(client_commit_id, sequence) {
5281            Ok(prepared) => prepared,
5282            Err(proof) => return proof,
5283        };
5284        let drive = self.sync_until_idle(transport, max_rounds);
5285        self.finish_current_base(prepared, drive)
5286    }
5287
5288    pub fn commit_outcomes(&self, query: CommitOutcomeQuery) -> Result<Vec<CommitOutcome>, String> {
5289        if query.limit == Some(0) {
5290            return Err("sync.invalid_request: commit outcome limit must be positive".to_owned());
5291        }
5292        let mut sql = String::from(
5293            "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
5294                    resolution, resolved_at_ms, replacement_client_commit_id
5295               FROM _syncular_commit_outcomes",
5296        );
5297        if query.active_only {
5298            sql.push_str(" WHERE resolution = 'active' AND status IN ('conflict', 'rejected')");
5299        }
5300        sql.push_str(" ORDER BY seq DESC");
5301        if let Some(limit) = query.limit {
5302            sql.push_str(&format!(" LIMIT {limit}"));
5303        }
5304        let mut stmt = self
5305            .conn
5306            .prepare(&sql)
5307            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5308        let rows = stmt
5309            .query_map([], |row| {
5310                Ok(StoredCommitOutcomeRow {
5311                    sequence: row.get(0)?,
5312                    client_commit_id: row.get(1)?,
5313                    status: row.get(2)?,
5314                    recorded_at_ms: row.get(3)?,
5315                    results_json: row.get(4)?,
5316                    operations_json: row.get(5)?,
5317                    resolution: row.get(6)?,
5318                    resolved_at_ms: row.get(7)?,
5319                    replacement_client_commit_id: row.get(8)?,
5320                })
5321            })
5322            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5323        let mut outcomes = Vec::new();
5324        for row in rows {
5325            outcomes.push(self.attach_retained_rows(Self::outcome_from_row(
5326                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
5327            )?)?);
5328        }
5329        Ok(outcomes)
5330    }
5331
5332    fn prune_commit_outcomes(&self) -> Result<(), String> {
5333        #[cfg(test)]
5334        self.outcome_prune_count
5335            .set(self.outcome_prune_count.get() + 1);
5336        let max_entries = self.limits.outcome_retention_max_entries.unwrap_or(1_000);
5337        let count = self
5338            .conn
5339            .query_row(
5340                "SELECT COUNT(*) FROM _syncular_commit_outcomes",
5341                [],
5342                |row| row.get::<_, i64>(0),
5343            )
5344            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
5345            as usize;
5346        let excess = count.saturating_sub(max_entries);
5347        if excess == 0 {
5348            return Ok(());
5349        }
5350        let mut stmt = self
5351            .conn
5352            .prepare(
5353                "SELECT seq FROM _syncular_commit_outcomes
5354                  WHERE status IN ('applied', 'cached') OR resolution != 'active'
5355                  ORDER BY seq ASC LIMIT ?1",
5356            )
5357            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5358        let rows = stmt
5359            .query_map(rusqlite::params![excess as i64], |row| row.get::<_, i64>(0))
5360            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5361        let sequences = rows
5362            .collect::<Result<Vec<_>, _>>()
5363            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5364        drop(stmt);
5365        for sequence in sequences {
5366            self.conn
5367                .execute(
5368                    "DELETE FROM _syncular_commit_outcomes WHERE seq = ?1",
5369                    rusqlite::params![sequence],
5370                )
5371                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5372        }
5373        Ok(())
5374    }
5375
5376    // -- units (§4.1) ------------------------------------------------------------
5377
5378    /// Every registered unit in registration order (§4.1).
5379    pub fn units(&self) -> Result<Vec<UnitRecord>, String> {
5380        self.stored(load_units(&self.conn))
5381    }
5382
5383    pub fn unit(&self, id: &str) -> Result<Option<UnitRecord>, String> {
5384        self.stored(get_unit(&self.conn, id))
5385    }
5386
5387    /// Register one unit (§4.1) and return its id.
5388    pub fn subscribe(&mut self, scopes: UnitScopes) -> Result<String, String> {
5389        Ok(self.subscribe_many(vec![scopes])?.remove(0))
5390    }
5391
5392    /// Register several units in one local transaction and return their ids:
5393    /// one commit for all of them (on OPFS a commit costs milliseconds, so a
5394    /// cold start with hundreds of units pays it once). An error rolls back
5395    /// every registration of the call. Registering an active unit again
5396    /// changes nothing; a revoked unit re-arms (§3.3). A call that registers or
5397    /// re-arms a unit raises sync-needed with an interactive intent.
5398    pub fn subscribe_many(&mut self, units: Vec<UnitScopes>) -> Result<Vec<String>, String> {
5399        self.begin_observation("syncular_subscribe")?;
5400        let mut changed = false;
5401        let registered = (|| {
5402            let mut ids = Vec::with_capacity(units.len());
5403            for scopes in units {
5404                let (id, held) = self.hold_unit(scopes, true)?;
5405                changed |= held;
5406                ids.push(id);
5407            }
5408            self.conn
5409                .execute_batch("RELEASE syncular_subscribe")
5410                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5411            Ok(ids)
5412        })();
5413        let ids = match registered {
5414            Ok(ids) => ids,
5415            Err(error) => {
5416                self.rollback_observation("syncular_subscribe");
5417                return Err(error);
5418            }
5419        };
5420        if changed {
5421            self.registration_epoch = self.registration_epoch.wrapping_add(1);
5422            self.set_sync_needed(true, true);
5423        }
5424        Ok(ids)
5425    }
5426
5427    /// Release the explicit reference to a unit. The unit leaves the
5428    /// registration list when no window base holds it (§4.1); its rows stay.
5429    pub fn unsubscribe(&mut self, id: &str) -> Result<(), String> {
5430        let Some(unit) = self.stored(get_unit(&self.conn, id))? else {
5431            return Ok(());
5432        };
5433        if !unit.explicit {
5434            return Ok(());
5435        }
5436        if !self.stored(window_refs(&self.conn, id))?.is_empty() {
5437            return self.stored(save_unit(
5438                &self.conn,
5439                &UnitRecord {
5440                    explicit: false,
5441                    ..unit
5442                },
5443            ));
5444        }
5445        self.conn
5446            .execute("DELETE FROM _syncular_units WHERE id = ?1", [id])
5447            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5448        self.registration_epoch = self.registration_epoch.wrapping_add(1);
5449        if unit.status == UnitStatus::Active {
5450            self.set_sync_needed(true, true);
5451        }
5452        Ok(())
5453    }
5454
5455    /// Take one reference on a unit (§4.1): validate the scope map, register a
5456    /// new unit catching up with cursor `-1`, re-arm a revoked one (§3.3).
5457    /// Returns the id and whether the registration list changed.
5458    fn hold_unit(&mut self, scopes: UnitScopes, explicit: bool) -> Result<(String, bool), String> {
5459        if scopes.is_empty()
5460            || scopes.values().any(|value| value == "*")
5461            || !self
5462                .schema
5463                .tables
5464                .iter()
5465                .any(|table| covers(table, &scopes))
5466        {
5467            return Err("sync.invalid_subscription: a unit names one non-wildcard value per variable and covers a table".to_owned());
5468        }
5469        let id = ssp2::registration::unit_id(&scope_pairs(&scopes));
5470        let existing = self.stored(get_unit(&self.conn, &id))?;
5471        if let Some(existing) = existing
5472            .as_ref()
5473            .filter(|unit| unit.status != UnitStatus::Revoked)
5474        {
5475            if explicit && !existing.explicit {
5476                self.stored(save_unit(
5477                    &self.conn,
5478                    &UnitRecord {
5479                        explicit,
5480                        ..existing.clone()
5481                    },
5482                ))?;
5483            }
5484            return Ok((id, false));
5485        }
5486        self.delete_pending_evict(&id)?;
5487        self.stored(save_unit(
5488            &self.conn,
5489            &UnitRecord {
5490                id: id.clone(),
5491                scopes,
5492                explicit: explicit || existing.is_some_and(|unit| unit.explicit),
5493                status: UnitStatus::Active,
5494                reason_code: None,
5495                cursor: Some(-1),
5496                bootstrap_state: None,
5497                seeded: Vec::new(),
5498                echo: None,
5499            },
5500        ))?;
5501        Ok((id, true))
5502    }
5503
5504    // -- windows (§4.8) -----------------------------------------------------------
5505
5506    /// §4.8: set the windowed-in values of a base, one unit per value. Added
5507    /// values take a reference on their unit (a new unit bootstraps on the
5508    /// next sync); removed values release theirs, and a unit no reference
5509    /// holds leaves the registration list fused with its first eviction chunk
5510    /// (E1–E4). Idempotent; re-entry cancels a deferred eviction.
5511    pub fn set_window(
5512        &mut self,
5513        base: &WindowBase,
5514        units: &[String],
5515    ) -> Result<CommandEffects, String> {
5516        let base_key = window_base_key(base);
5517        let live = self.load_window_units(&base_key)?;
5518        let mut changed = false;
5519        for value in units {
5520            if live.iter().any(|(held, _)| held == value) {
5521                continue;
5522            }
5523            let rollback = self.capture();
5524            self.begin_observation("syncular_window")?;
5525            let widened = (|| {
5526                let (id, registered) = self.hold_unit(window_unit_scopes(base, value), false)?;
5527                self.sync_needed |= registered;
5528                self.conn
5529                    .execute(
5530                        "INSERT OR REPLACE INTO _syncular_windows(base, unit, sub_id) VALUES (?1, ?2, ?3)",
5531                        rusqlite::params![base_key, value, id],
5532                    )
5533                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5534                let mut batch = ChangeAccumulator::default();
5535                batch.window(&base_key, value);
5536                batch.status = true;
5537                self.finish_observation("syncular_window", batch)
5538            })();
5539            if let Err(error) = widened {
5540                self.rollback_observation("syncular_window");
5541                self.restore(rollback);
5542                return Err(error);
5543            }
5544            changed = true;
5545            self.registration_epoch = self.registration_epoch.wrapping_add(1);
5546        }
5547        for (value, id) in live {
5548            if units.contains(&value) {
5549                continue;
5550            }
5551            self.rebuild_overlay_if_dirty()?;
5552            let rollback = self.capture();
5553            self.begin_observation("syncular_window")?;
5554            let remaining = self.release_window_unit(&base_key, &value, &id);
5555            let remaining = match remaining {
5556                Ok(remaining) => remaining,
5557                Err(error) => {
5558                    self.rollback_observation("syncular_window");
5559                    self.restore(rollback);
5560                    return Err(error);
5561                }
5562            };
5563            changed = true;
5564            self.registration_epoch = self.registration_epoch.wrapping_add(1);
5565            if remaining {
5566                self.drain_pending_evictions()?;
5567            }
5568        }
5569        Ok(if changed && self.transport_enabled {
5570            CommandEffects::interactive()
5571        } else {
5572            CommandEffects::none()
5573        })
5574    }
5575
5576    /// §4.8 completeness oracle: the windowed-in values of a base and those
5577    /// whose unit is not complete. A value is complete once its unit is active
5578    /// and no longer bootstrapping; registration alone is not completeness.
5579    pub fn window_state(&self, base: &WindowBase) -> Result<WindowState, String> {
5580        persisted_window_state(&self.conn, base).map_err(|failure| failure.message)
5581    }
5582
5583    /// Registry rows for a base, ordered by value: (value, unit id).
5584    fn load_window_units(&self, base_key: &str) -> Result<Vec<(String, String)>, String> {
5585        let mut statement = self
5586            .conn
5587            .prepare_cached(
5588                "SELECT unit, sub_id FROM _syncular_windows WHERE base = ?1 ORDER BY unit ASC",
5589            )
5590            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5591        let rows = statement
5592            .query_map([base_key], |row| Ok((row.get(0)?, row.get(1)?)))
5593            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5594        rows.collect::<Result<_, _>>()
5595            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5596    }
5597
5598    /// Every window registry row: (base key, value).
5599    fn load_window_registry(&self) -> Result<Vec<(String, String)>, String> {
5600        let mut statement = self
5601            .conn
5602            .prepare_cached("SELECT base, unit FROM _syncular_windows ORDER BY base, unit")
5603            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5604        let rows = statement
5605            .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))
5606            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5607        rows.collect::<Result<_, _>>()
5608            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5609    }
5610
5611    /// §4.8: release a window value's reference inside the caller's
5612    /// observation. When no reference holds the unit, unregister it fused with
5613    /// its first eviction chunk (E3): delete the rows of every covered table
5614    /// matching the persisted echo, except rows pinned by the outbox (E1) or
5615    /// owned by another unit (§3.3), and park the per-table scopes of what
5616    /// remains in the pending-eviction registry. Fails closed without a local
5617    /// scope-column mapping. Returns whether unpinned rows remain.
5618    fn release_window_unit(
5619        &mut self,
5620        base_key: &str,
5621        value: &str,
5622        id: &str,
5623    ) -> Result<bool, String> {
5624        self.conn
5625            .execute(
5626                "DELETE FROM _syncular_windows WHERE base = ?1 AND unit = ?2",
5627                rusqlite::params![base_key, value],
5628            )
5629            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5630        let mut batch = ChangeAccumulator::default();
5631        batch.window(base_key, value);
5632        batch.status = true;
5633        let unit = self.stored(get_unit(&self.conn, id))?;
5634        let mut remaining = false;
5635        if let Some(unit) = unit.filter(|unit| !unit.explicit) {
5636            if self.stored(window_refs(&self.conn, id))?.is_empty() {
5637                let units = self.stored(load_units(&self.conn))?;
5638                self.conn
5639                    .execute("DELETE FROM _syncular_units WHERE id = ?1", [id])
5640                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5641                self.sync_needed = true;
5642                for table in self.schema.tables.clone() {
5643                    let Some(effective) = unit
5644                        .echo
5645                        .as_deref()
5646                        .filter(|_| covers(&table, &unit.scopes))
5647                        .and_then(|echo| pair_scopes(&unit.scopes, &table.name, Some(echo)))
5648                    else {
5649                        continue;
5650                    };
5651                    let pinned = self.pinned_row_ids(&table.name);
5652                    let owners = unit_owners(&units, &table, id);
5653                    let (more, deferred) =
5654                        self.evict_scope_rows(&table.name, &effective, &owners, &pinned)?;
5655                    remaining |= more;
5656                    if deferred {
5657                        self.save_pending_evict(id, &table.name, &effective)?;
5658                    }
5659                    self.record_scope_map(&mut batch, &table.name, &effective);
5660                }
5661            }
5662        }
5663        self.finish_observation("syncular_window", batch)?;
5664        Ok(remaining)
5665    }
5666
5667    /// §4.8 E1: delete base rows owned by the effective scopes
5668    /// ([`local_scope_predicate`]) EXCEPT pinned primary keys. Returns (more
5669    /// unpinned work, any scoped rows remain). A missing scope-column mapping
5670    /// fails without deleting rows.
5671    fn evict_scope_rows(
5672        &self,
5673        table_name: &str,
5674        effective: &[(String, Vec<String>)],
5675        owners: &[Vec<(String, Vec<String>)>],
5676        pinned: &std::collections::HashSet<String>,
5677    ) -> Result<(bool, bool), String> {
5678        let table = self.schema.table(table_name).ok_or("sync.unknown_table")?;
5679        let Some((scope, mut params)) = local_scope_predicate(table, effective, owners)
5680            .map_err(|()| "sync.scope_revoked".to_owned())?
5681        else {
5682            return Ok((false, false));
5683        };
5684        let scope_params = params.len();
5685        let pk = quote_ident(&table.primary_key);
5686        // Pinned ids are rowId text; the text form matches every key type.
5687        let predicate = if pinned.is_empty() {
5688            scope.clone()
5689        } else {
5690            params.extend(pinned.iter().cloned().map(SqlValue::Text));
5691            let holes = vec!["?"; pinned.len()].join(", ");
5692            format!("{scope} AND CAST({pk} AS TEXT) NOT IN ({holes})")
5693        };
5694        let base = base_table(table_name);
5695        let visible = visible_table(table_name);
5696        let mut ids: Vec<SqlValue> = Vec::new();
5697        for source in [&base, &visible] {
5698            let mut stmt = self
5699                .conn
5700                .prepare_cached(&format!(
5701                    "SELECT {pk} FROM {source} WHERE {predicate} LIMIT 1024"
5702                ))
5703                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5704            ids = stmt
5705                .query_map(rusqlite::params_from_iter(&params), |row| row.get(0))
5706                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
5707                .collect::<Result<_, _>>()
5708                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5709            if !ids.is_empty() {
5710                break;
5711            }
5712        }
5713        if !ids.is_empty() {
5714            for id in &ids {
5715                self.conn
5716                    .execute(
5717                        "DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2",
5718                        rusqlite::params![table_name, id],
5719                    )
5720                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5721            }
5722            let holes = vec!["?"; ids.len()].join(", ");
5723            for source in [&base, &visible] {
5724                self.conn
5725                    .execute(
5726                        &format!("DELETE FROM {source} WHERE {pk} IN ({holes})"),
5727                        rusqlite::params_from_iter(&ids),
5728                    )
5729                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5730            }
5731        }
5732        let mut remaining = false;
5733        let mut deferred = false;
5734        for source in [&base, &visible] {
5735            remaining |= self
5736                .conn
5737                .query_row(
5738                    &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {predicate})"),
5739                    rusqlite::params_from_iter(&params),
5740                    |row| row.get::<_, bool>(0),
5741                )
5742                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5743            deferred |= self
5744                .conn
5745                .query_row(
5746                    &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {scope})"),
5747                    rusqlite::params_from_iter(&params[..scope_params]),
5748                    |row| row.get::<_, bool>(0),
5749                )
5750                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5751        }
5752        Ok((remaining, deferred))
5753    }
5754
5755    /// §4.8 E1: continue deferred evictions after the outbox drains or between
5756    /// chunks, retaining outbox pins and their durable marker. A unit that
5757    /// re-entered the window has no pending entry left.
5758    fn drain_pending_evictions(&mut self) -> Result<(), String> {
5759        let pending = self.load_pending_evictions()?;
5760        if pending.is_empty() {
5761            return Ok(());
5762        }
5763        self.rebuild_overlay_if_dirty()?;
5764        for (unit_id, table_name, effective) in pending {
5765            let Some(table) = self.schema.table(&table_name).cloned() else {
5766                self.delete_pending_evict_table(&unit_id, &table_name)?;
5767                continue;
5768            };
5769            loop {
5770                std::thread::yield_now();
5771                self.begin_observation("syncular_evict")?;
5772                let chunk = (|| {
5773                    let pinned = self.pinned_row_ids(&table_name);
5774                    let owners =
5775                        unit_owners(&self.stored(load_units(&self.conn))?, &table, &unit_id);
5776                    let (remaining, deferred) =
5777                        self.evict_scope_rows(&table_name, &effective, &owners, &pinned)?;
5778                    if !deferred {
5779                        self.delete_pending_evict_table(&unit_id, &table_name)?;
5780                    }
5781                    let mut batch = ChangeAccumulator::default();
5782                    self.record_scope_map(&mut batch, &table_name, &effective);
5783                    self.finish_observation("syncular_evict", batch)?;
5784                    Ok(remaining)
5785                })();
5786                match chunk {
5787                    Ok(true) => {}
5788                    Ok(false) => break,
5789                    Err(error) => {
5790                        self.rollback_observation("syncular_evict");
5791                        return Err(error);
5792                    }
5793                }
5794            }
5795        }
5796        Ok(())
5797    }
5798
5799    /// §4.8 E1: primary keys of `table` referenced by a pending outbox
5800    /// commit — rows that MUST NOT be evicted until the commit drains.
5801    fn pinned_row_ids(&self, table: &str) -> std::collections::HashSet<String> {
5802        let mut pinned = std::collections::HashSet::new();
5803        for commit in self
5804            .outbox
5805            .iter()
5806            .map(|commit| &**commit)
5807            .chain(self.failed_commits.iter().map(|(commit, _)| commit))
5808        {
5809            for op in &commit.ops {
5810                if op.table == table {
5811                    pinned.insert(op.row_id.clone());
5812                }
5813            }
5814        }
5815        pinned
5816    }
5817
5818    fn save_pending_evict(
5819        &self,
5820        unit_id: &str,
5821        table: &str,
5822        effective: &[(String, Vec<String>)],
5823    ) -> Result<(), String> {
5824        self.conn.execute(
5825            "INSERT OR REPLACE INTO _syncular_pending_evictions(unit_id, tbl, effective_scopes) VALUES (?1, ?2, ?3)",
5826            rusqlite::params![unit_id, table, scope_map_to_json(effective).to_string()],
5827        ).map(|_| ()).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5828    }
5829
5830    /// Re-entry cancels every pending eviction of the unit.
5831    fn delete_pending_evict(&self, unit_id: &str) -> Result<(), String> {
5832        self.conn
5833            .execute(
5834                "DELETE FROM _syncular_pending_evictions WHERE unit_id = ?1",
5835                [unit_id],
5836            )
5837            .map(|_| ())
5838            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5839    }
5840
5841    fn delete_pending_evict_table(&self, unit_id: &str, table: &str) -> Result<(), String> {
5842        self.conn
5843            .execute(
5844                "DELETE FROM _syncular_pending_evictions WHERE unit_id = ?1 AND tbl = ?2",
5845                [unit_id, table],
5846            )
5847            .map(|_| ())
5848            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5849    }
5850
5851    fn load_pending_evictions(&self) -> Result<Vec<PendingEvict>, String> {
5852        let mut stmt = self
5853            .conn
5854            .prepare("SELECT unit_id, tbl, effective_scopes FROM _syncular_pending_evictions")
5855            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5856        let rows = stmt
5857            .query_map([], |row| {
5858                Ok((
5859                    row.get::<_, String>(0)?,
5860                    row.get::<_, String>(1)?,
5861                    row.get::<_, String>(2)?,
5862                ))
5863            })
5864            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5865        rows.map(|row| {
5866            let (unit_id, table, json) =
5867                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5868            let value: Value = serde_json::from_str(&json)
5869                .map_err(|_| "sync.local_corrupt: invalid pending eviction scopes".to_owned())?;
5870            let effective = json_to_scope_map(&value)
5871                .map_err(|_| "sync.local_corrupt: invalid pending eviction scopes".to_owned())?;
5872            Ok((unit_id, table, effective))
5873        })
5874        .collect()
5875    }
5876
5877    /// Select failed-intent retention; reload durable retained aggregates.
5878    pub fn set_retain_failed_commits(&mut self, enabled: bool) -> Result<(), String> {
5879        self.retain_failed_commits = enabled;
5880        for (commit, _) in self.failed_commits.iter() {
5881            for operation in &commit.ops {
5882                self.overlay_dirty.table(&operation.table);
5883            }
5884        }
5885        self.failed_commits = self.load_failed_commits()?.into();
5886        for (commit, _) in self.failed_commits.iter() {
5887            for operation in &commit.ops {
5888                self.overlay_dirty.table(&operation.table);
5889            }
5890        }
5891        Ok(())
5892    }
5893
5894    fn load_failed_commits(&self) -> Result<Vec<FailedCommit>, String> {
5895        let mut statement = self.conn.prepare("SELECT client_commit_id, operations_json, initial_json FROM _syncular_failed_commits ORDER BY seq")
5896            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5897        let rows = statement
5898            .query_map([], |row| {
5899                Ok((
5900                    row.get::<_, String>(0)?,
5901                    row.get::<_, String>(1)?,
5902                    row.get::<_, String>(2)?,
5903                ))
5904            })
5905            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5906        rows.map(|row| {
5907            let (id, operations, initial) =
5908                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5909            let operations: Vec<CommitOperation> =
5910                serde_json::from_str(&operations).map_err(|_| "sync.local_corrupt".to_owned())?;
5911            let initial =
5912                serde_json::from_str(&initial).map_err(|_| "sync.local_corrupt".to_owned())?;
5913            Ok((
5914                OutboxCommit {
5915                    client_commit_id: id,
5916                    ops: operations
5917                        .into_iter()
5918                        .map(|op| OutboxOp {
5919                            upsert: op.op == "upsert",
5920                            table: op.table,
5921                            row_id: op.row_id,
5922                            base_version: op.base_version,
5923                            values: op.values,
5924                            insert: op.insert,
5925                        })
5926                        .collect(),
5927                },
5928                initial,
5929            ))
5930        })
5931        .collect()
5932    }
5933
5934    fn retain_failed_commit(&mut self, id: &str, operations: &[OutboxOp]) -> Result<(), String> {
5935        let mut initial = Map::new();
5936        for operation in operations {
5937            // A table the current schema removed has no rows to retain.
5938            let Some(table) = self.schema.table(&operation.table) else {
5939                continue;
5940            };
5941            let initial_row = match self.stored_row(table, &operation.row_id, true)? {
5942                Some(row) => Some(row),
5943                None => self.visible_row(table, &operation.row_id)?,
5944            };
5945            if let Some((values, _)) = initial_row {
5946                let mut intended: Map<String, Value> = table
5947                    .columns
5948                    .iter()
5949                    .zip(values.iter())
5950                    .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
5951                    .collect();
5952                if let Some(patch) = &operation.values {
5953                    intended.extend(patch.clone());
5954                }
5955                initial.insert(
5956                    serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
5957                        .map_err(|_| "sync.local_corrupt".to_owned())?,
5958                    Value::Object(intended),
5959                );
5960            }
5961        }
5962        self.conn.execute("INSERT INTO _syncular_failed_commits(client_commit_id, operations_json, initial_json) VALUES (?1, ?2, ?3)", rusqlite::params![id,
5963            serde_json::to_string(&operations.iter().map(CommitOperation::from).collect::<Vec<_>>()).map_err(|_| "sync.local_corrupt".to_owned())?,
5964            serde_json::to_string(&initial).map_err(|_| "sync.local_corrupt".to_owned())?])
5965            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5966        self.failed_commits.push((
5967            OutboxCommit {
5968                client_commit_id: id.to_owned(),
5969                ops: operations.to_vec(),
5970            },
5971            initial,
5972        ));
5973        Ok(())
5974    }
5975
5976    fn drop_failed_commits(
5977        &mut self,
5978        matches: impl Fn(&str, &Map<String, Value>) -> bool,
5979    ) -> Result<Vec<OutboxCommit>, String> {
5980        let mut doomed = self.drop_acknowledged_rows(&matches)?;
5981        for (commit, initial) in self.failed_commits.iter() {
5982            for operation in &commit.ops {
5983                let key =
5984                    serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
5985                        .map_err(|_| "sync.local_corrupt".to_owned())?;
5986                if initial
5987                    .get(&key)
5988                    .and_then(Value::as_object)
5989                    .is_some_and(|values| matches(&operation.table, values))
5990                {
5991                    doomed.push(commit.clone());
5992                    break;
5993                }
5994            }
5995        }
5996        for commit in &doomed {
5997            self.conn
5998                .execute(
5999                    "DELETE FROM _syncular_failed_commits WHERE client_commit_id = ?1",
6000                    rusqlite::params![commit.client_commit_id],
6001                )
6002                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6003        }
6004        for commit in &doomed {
6005            self.conn.execute("UPDATE _syncular_commit_outcomes SET operations_json = NULL, results_json = '[]' WHERE client_commit_id = ?1", rusqlite::params![commit.client_commit_id]).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6006        }
6007        self.conflicts.retain(|record| {
6008            !doomed
6009                .iter()
6010                .any(|commit| commit.client_commit_id == record.client_commit_id)
6011        });
6012        self.rejections.retain(|record| {
6013            !doomed
6014                .iter()
6015                .any(|commit| commit.client_commit_id == record.client_commit_id)
6016        });
6017        self.failed_commits.retain(|(commit, _)| {
6018            !doomed
6019                .iter()
6020                .any(|dropped| dropped.client_commit_id == commit.client_commit_id)
6021        });
6022        for commit in &doomed {
6023            for operation in &commit.ops {
6024                self.overlay_dirty.table(&operation.table);
6025            }
6026        }
6027        Ok(doomed)
6028    }
6029
6030    /// Record one commit of local mutations. The returned commit ID is the
6031    /// §2.3 idempotency key. Failures are structured so callers read
6032    /// `code`/`message`/`details` instead of parsing a string prefix.
6033    pub fn mutate(&mut self, mutations: Vec<Mutation>) -> Result<String, ClientError> {
6034        self.mutate_with_commit_id(mutations, None)
6035    }
6036
6037    /// [`Self::mutate`] under a caller-supplied `clientCommitId` (§7.1), so an
6038    /// app that loses the reply knows its commit. The ID is non-empty (§6.1);
6039    /// one this replica still holds fails with `client.commit_id_reused`.
6040    pub fn mutate_with_commit_id(
6041        &mut self,
6042        mutations: Vec<Mutation>,
6043        client_commit_id: Option<String>,
6044    ) -> Result<String, ClientError> {
6045        let prior = self.storage_failure.borrow_mut().take();
6046        if let Some(id) = &client_commit_id {
6047            let held = if id.is_empty() {
6048                Err("sync.invalid_request: clientCommitId is empty".to_owned())
6049            } else {
6050                self.conn
6051                    .query_row(
6052                        "SELECT EXISTS(SELECT 1 FROM _syncular_outbox WHERE commit_id=?1 UNION ALL SELECT 1 FROM _syncular_commit_outcomes WHERE client_commit_id=?1)",
6053                        [id],
6054                        |row| row.get(0),
6055                    )
6056                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
6057            };
6058            match held {
6059                Ok(false) => {}
6060                Ok(true) => {
6061                    return self.finish_authoring(
6062                        prior,
6063                        Err(ClientError {
6064                            code: "client.commit_id_reused".to_owned(),
6065                            message: "clientCommitId is held".to_owned(),
6066                            details: Some(json!({ "clientCommitId": id })),
6067                            retryable: false,
6068                        }),
6069                    )
6070                }
6071                Err(message) => return self.finish_authoring(prior, Err(message.into())),
6072            }
6073        }
6074        let outcome = (|| -> Result<String, ClientError> {
6075            if mutations.is_empty() {
6076                return Err(
6077                    "sync.invalid_request: a commit must contain at least one operation (§6.1)"
6078                        .to_owned()
6079                        .into(),
6080                );
6081            }
6082            let mut ops = Vec::with_capacity(mutations.len());
6083            // §7.1: a patch reads the row as the commit's earlier operations
6084            // leave it: the values an earlier upsert wrote, or None after an
6085            // earlier delete. A patch changes neither presence nor a scope
6086            // column, so it records nothing.
6087            let mut staged: HashMap<(String, String), Option<Map<String, Value>>> = HashMap::new();
6088            for mutation in mutations {
6089                match mutation {
6090                    Mutation::Patch {
6091                        table,
6092                        values,
6093                        base_version,
6094                    } => {
6095                        let schema_table = self
6096                            .schema
6097                            .table(&table)
6098                            .ok_or_else(|| "sync.unknown_table".to_owned())?;
6099                        let values = normalize_values_casing(schema_table, values)?;
6100                        let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
6101                        let base = staged.get(&(table.clone(), row_id.clone()));
6102                        ops.push(self.prepare_patch(
6103                            &table,
6104                            &row_id,
6105                            values,
6106                            base_version,
6107                            base,
6108                        )?);
6109                    }
6110                    Mutation::Upsert {
6111                        table,
6112                        values,
6113                        base_version,
6114                    } => {
6115                        let schema_table = self.schema.table(&table).ok_or_else(|| {
6116                            format!("sync.unknown_table: unknown table {table:?}")
6117                        })?;
6118                        // §5: value keys are accepted in snake_case AND the
6119                        // generated row types' camelCase; normalize to SQL truth
6120                        // before the pk lookup / codec see them.
6121                        let values = normalize_values_casing(schema_table, values)?;
6122                        let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
6123                        // §6.7 `mutate` marks every column present: fill missing
6124                        // nullable columns with NULL before the payload is built.
6125                        let values = full_row_values(schema_table, values)?;
6126                        self.refuse_sealed_row(schema_table, &row_id)?;
6127                        // §5.11: validate the payload encodes with the current
6128                        // codec. A key-selection or unknown-key failure is a
6129                        // durable rejection at the push seam (§10.3), never an
6130                        // author-time error. Full-row `mutate` presents every
6131                        // column, so no stored fallback is needed.
6132                        self.validate_author_encode(schema_table, &row_id, &values)?;
6133                        // §7.1 insert intent: the row is absent from the
6134                        // visible replica at call time, before this batch
6135                        // applies anything.
6136                        let insert = !self.visible_row_exists(schema_table, &row_id)?;
6137                        staged.insert((table.clone(), row_id.clone()), Some(values.clone()));
6138                        ops.push(OutboxOp {
6139                            upsert: true,
6140                            table,
6141                            row_id,
6142                            base_version,
6143                            values: Some(values),
6144                            insert,
6145                        });
6146                    }
6147                    Mutation::Delete {
6148                        table,
6149                        row_id,
6150                        base_version,
6151                    } => {
6152                        let schema_table = self.schema.table(&table).ok_or_else(|| {
6153                            format!("sync.unknown_table: unknown table {table:?}")
6154                        })?;
6155                        self.refuse_sealed_row(schema_table, &row_id)?;
6156                        // §2.4: the server rejects a rowId holding U+0000.
6157                        if row_id.contains('\0') {
6158                            return Err("sync.invalid_request: rowId holds U+0000 (§2.4)"
6159                                .to_owned()
6160                                .into());
6161                        }
6162                        staged.insert((table.clone(), row_id.clone()), None);
6163                        ops.push(OutboxOp {
6164                            upsert: false,
6165                            table,
6166                            row_id,
6167                            base_version,
6168                            values: None,
6169                            insert: false,
6170                        });
6171                    }
6172                }
6173            }
6174            Ok(self.record_outbox_commit(ops, client_commit_id)?)
6175        })();
6176        self.finish_authoring(prior, outcome)
6177    }
6178
6179    /// Scope the retained `storage_failure` diagnostic to this call so an
6180    /// earlier operation's failure cannot classify a new one, and surface the
6181    /// call's own classified failure with its details. A fresh classified
6182    /// failure replaces the retained diagnostic; otherwise the earlier one is
6183    /// kept for the next sync.
6184    fn finish_authoring(
6185        &self,
6186        prior: Option<QueryReadFailure>,
6187        outcome: Result<String, ClientError>,
6188    ) -> Result<String, ClientError> {
6189        let fresh = self.storage_failure.borrow_mut().take();
6190        *self.storage_failure.borrow_mut() = fresh.clone().or(prior);
6191        outcome.map_err(|error| fresh.map(ClientError::from).unwrap_or(error))
6192    }
6193
6194    fn record_outbox_commit(
6195        &mut self,
6196        ops: Vec<OutboxOp>,
6197        client_commit_id: Option<String>,
6198    ) -> Result<String, String> {
6199        let commit = OutboxCommit {
6200            client_commit_id: client_commit_id.unwrap_or_else(|| uuid::Uuid::new_v4().to_string()),
6201            ops,
6202        };
6203        self.begin_observation("syncular_mutation")?;
6204        let mut batch = ChangeAccumulator::default();
6205        for op in &commit.ops {
6206            let mut precise = self.record_row_scopes(&mut batch, &op.table, &op.row_id, false);
6207            if let Some(values) = &op.values {
6208                if let Some(table) = self.schema.table(&op.table) {
6209                    for scope in &table.scope_variables {
6210                        if let Some(Value::String(value)) = values.get(&scope.column) {
6211                            batch.scope(&op.table, format!("{}:{value}", scope.prefix));
6212                            precise = true;
6213                        }
6214                    }
6215                }
6216            }
6217            if !precise {
6218                batch.table(&op.table);
6219            }
6220        }
6221        if let Err(error) = self.persist_outbox_insert(&commit) {
6222            self.rollback_observation("syncular_mutation");
6223            return Err(error);
6224        }
6225        // A clean overlay contains the FIFO fold of every previous commit, so
6226        // the new commit applies over it. §7.1: an operation that does not
6227        // apply (a secondary unique collision) fails the whole commit; the
6228        // savepoint rollback removes the outbox entry and every visible write.
6229        // The append undoes itself: a rollback capture would copy the outbox.
6230        let was_dirty = self.overlay_dirty.snapshot();
6231        let id = commit.client_commit_id.clone();
6232        let appended = self.rebuild_overlay_if_dirty().and_then(|()| {
6233            commit
6234                .ops
6235                .iter()
6236                .try_for_each(|op| self.apply_outbox_op(op))?;
6237            self.outbox.push(Arc::new(commit));
6238            batch.status = true;
6239            self.finish_observation("syncular_mutation", batch)
6240                .inspect_err(|_| drop(self.outbox.pop()))
6241        });
6242        if let Err(error) = appended {
6243            self.rollback_observation("syncular_mutation");
6244            self.overlay_dirty.restore(was_dirty);
6245            return Err(error);
6246        }
6247        Ok(id)
6248    }
6249
6250    /// Record one §6.1 sparse upsert: the primary key plus the supplied
6251    /// non-scope columns are present; every other column is absent and stays
6252    /// untouched on the server and in the local overlay. An absent local base
6253    /// refuses the whole batch with sync.row_missing before enqueueing (§7.1).
6254    /// Failures are structured so callers read `code`/`message`/`details`.
6255    pub fn patch(
6256        &mut self,
6257        table: &str,
6258        row_id: &str,
6259        partial: Map<String, Value>,
6260        base_version: Option<i64>,
6261    ) -> Result<String, ClientError> {
6262        let prior = self.storage_failure.borrow_mut().take();
6263        let outcome = self.perform_patch(table, row_id, partial, base_version);
6264        self.finish_authoring(prior, outcome)
6265    }
6266
6267    /// The unscoped patch body. Public authoring entry points scope
6268    /// `storage_failure` around the whole operation, including any pre-read.
6269    fn perform_patch(
6270        &mut self,
6271        table: &str,
6272        row_id: &str,
6273        partial: Map<String, Value>,
6274        base_version: Option<i64>,
6275    ) -> Result<String, ClientError> {
6276        let operation = self.prepare_patch(table, row_id, partial, base_version, None)?;
6277        Ok(self.record_outbox_commit(vec![operation], None)?)
6278    }
6279
6280    /// `staged` is the row as the batch's earlier operations leave it
6281    /// (`Some(None)` after a delete); `None` reads the visible replica.
6282    fn prepare_patch(
6283        &self,
6284        table: &str,
6285        row_id: &str,
6286        partial: Map<String, Value>,
6287        base_version: Option<i64>,
6288        staged: Option<&Option<Map<String, Value>>>,
6289    ) -> Result<OutboxOp, ClientError> {
6290        let schema_table = self
6291            .schema
6292            .table(table)
6293            .ok_or_else(|| format!("sync.unknown_table: unknown table {table:?}"))?;
6294        let partial = normalize_values_casing(schema_table, partial)?;
6295        if let Some(pk) = partial.get(&schema_table.primary_key) {
6296            if render_row_id_json(Some(pk)).ok().as_deref() != Some(row_id) {
6297                return Err(invalid_authoring(
6298                    "patch cannot change the primary key",
6299                    json!({ "table": table }),
6300                ));
6301            }
6302        }
6303        let exists = match staged {
6304            Some(row) => row.is_some(),
6305            None => self.visible_row_exists(schema_table, row_id)?,
6306        };
6307        if !exists {
6308            return Err("sync.row_missing: a sparse patch requires a local row"
6309                .to_owned()
6310                .into());
6311        }
6312        self.refuse_sealed_row(schema_table, row_id)?;
6313        // §3.4 rule 5 / §6.2: scope columns are immutable on update. The
6314        // server accepts a present scope column whose value equals the
6315        // stored row and applies it as a no-op, so a decoded envelope
6316        // round-trips. The local row is the only value the client can prove
6317        // equality against; when the row is absent locally the client cannot
6318        // prove equality against the server's stored row and fails closed.
6319        // A primary key that is also a scope column needs no stored-row
6320        // read: the primary key in a sparse payload is by construction the
6321        // row id being patched (§6.1), so its value is proven equal already.
6322        for scope in &schema_table.scope_variables {
6323            if !partial.contains_key(&scope.column) || scope.column == schema_table.primary_key {
6324                continue;
6325            }
6326            let Some(column) = schema_table
6327                .columns
6328                .iter()
6329                .find(|column| column.name == scope.column)
6330            else {
6331                continue;
6332            };
6333            let stored = match staged {
6334                Some(row) => row.as_ref().and_then(|row| {
6335                    json_to_column_value(column, row.get(&scope.column))
6336                        .ok()
6337                        .map(|value| column_value_to_json(&value))
6338                }),
6339                None => self
6340                    .conn
6341                    .query_row(
6342                        &format!(
6343                            "SELECT {} FROM {} WHERE {}",
6344                            quote_ident(&scope.column),
6345                            visible_table(&schema_table.name),
6346                            row_id_predicate(schema_table)
6347                        ),
6348                        rusqlite::params![row_id],
6349                        |row| Ok(sql_ref_to_json(column, row.get_ref(0)?)),
6350                    )
6351                    .optional()
6352                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
6353            };
6354            // Decode the supplied value through the same column-type path the
6355            // rest of the patch encoding uses, so a value that only equals
6356            // the stored one in another number representation still compares
6357            // equal.
6358            let incoming = json_to_column_value(column, partial.get(&scope.column))
6359                .ok()
6360                .map(|value| column_value_to_json(&value));
6361            // A stored blob never equals a patch value, matching the server's
6362            // scope-column comparison (§3.4): bytes are unstorable as scope
6363            // values.
6364            let equal = stored.as_ref().is_some_and(|stored| {
6365                stored.get("$bytes").is_none() && incoming.as_ref() == Some(stored)
6366            });
6367            if !equal {
6368                return Err(invalid_authoring(
6369                    "patch cannot write a scope column (§3.4)",
6370                    json!({ "table": table, "column": scope.column }),
6371                ));
6372            }
6373        }
6374        let mut values = partial;
6375        // Presence-set semantics: a proven-equal scope column is dropped, so
6376        // the patch leaves the stored value untouched (§6.2).
6377        for scope in &schema_table.scope_variables {
6378            values.remove(&scope.column);
6379        }
6380        let pk_value = match schema_table.columns[schema_table.pk_index].ty {
6381            // §2.4: the rowId of an integer key is its canonical decimal form.
6382            ColumnType::Integer => match row_id.parse::<i64>() {
6383                Ok(v)
6384                    if v.to_string() == row_id
6385                        && (-ssp2::primitives::I64_SAFE_MAX..=ssp2::primitives::I64_SAFE_MAX)
6386                            .contains(&v) =>
6387                {
6388                    Value::from(v)
6389                }
6390                _ => {
6391                    return Err(format!(
6392                    "sync.invalid_request: table {table:?}: invalid integer primary key {row_id:?}"
6393                )
6394                    .into())
6395                }
6396            },
6397            ColumnType::Boolean => match row_id {
6398                "true" => Value::Bool(true),
6399                "false" => Value::Bool(false),
6400                _ => {
6401                    return Err(format!(
6402                    "sync.invalid_request: table {table:?}: invalid boolean primary key {row_id:?}"
6403                )
6404                    .into())
6405                }
6406            },
6407            _ => Value::from(row_id.to_owned()),
6408        };
6409        values.insert(schema_table.primary_key.clone(), pk_value);
6410        // §6.1: a patch cannot null a non-nullable column.
6411        if let Some(column) = schema_table
6412            .columns
6413            .iter()
6414            .find(|column| !column.nullable && values.get(&column.name) == Some(&Value::Null))
6415        {
6416            return Err(invalid_authoring(
6417                "a non-nullable column is null (§6.1)",
6418                json!({ "table": table, "column": column.name }),
6419            ));
6420        }
6421        // §5.11: structural validation only; an absent key-id selector is
6422        // resolved at the push seam from the stored local row, and a
6423        // key-selection or unknown-key failure becomes a durable
6424        // `client.encrypt_failed` rejection there (§10.3), never an
6425        // author-time error.
6426        self.validate_author_encode(schema_table, row_id, &values)?;
6427        Ok(OutboxOp {
6428            upsert: true,
6429            table: table.to_owned(),
6430            row_id: row_id.to_owned(),
6431            base_version,
6432            values: Some(values),
6433            insert: false,
6434        })
6435    }
6436
6437    /// §5.11 key-id fallback: the stored local row's selector value as a
6438    /// positional row (only the selector slot filled). Returns None when no
6439    /// selector is configured, the row is locally absent, or the stored
6440    /// value is not a usable key id. A failed storage read is an error.
6441    fn stored_key_fallback(
6442        &self,
6443        table: &TableSchema,
6444        row_id: &str,
6445    ) -> Result<Option<Row>, String> {
6446        let Some(selector) = self.encryption.key_id_columns.get(&table.name) else {
6447            return Ok(None);
6448        };
6449        let Some(index) = table
6450            .columns
6451            .iter()
6452            .position(|column| &column.name == selector)
6453        else {
6454            return Ok(None);
6455        };
6456        let sql = format!(
6457            "SELECT {} FROM {} WHERE {}",
6458            quote_ident(selector),
6459            visible_table(&table.name),
6460            row_id_predicate(table)
6461        );
6462        let key = self
6463            .conn
6464            .query_row(&sql, rusqlite::params![row_id], |row| {
6465                row.get::<_, Option<String>>(0)
6466            })
6467            .optional()
6468            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
6469            .flatten()
6470            .filter(|key| !key.is_empty());
6471        let Some(key) = key else {
6472            return Ok(None);
6473        };
6474        let mut fallback: Row = vec![None; table.columns.len()];
6475        fallback[index] = Some(ColumnValue::String(key));
6476        Ok(Some(fallback))
6477    }
6478
6479    /// §5.11/§10.3 author-time encode check: a structural failure (a
6480    /// non-nullable NULL, a value the codec rejects) is an author-time
6481    /// error; a key-selection or unknown-key failure is deferred to the push
6482    /// seam, where it becomes a durable `client.encrypt_failed` rejection.
6483    fn validate_author_encode(
6484        &self,
6485        table: &TableSchema,
6486        row_id: &str,
6487        values: &Map<String, Value>,
6488    ) -> Result<(), String> {
6489        match encode_sparse_row_json(table, row_id, values, &self.encryption, None) {
6490            Err(error) if error.starts_with(ENCRYPT_FAILED_CODE) => Ok(()),
6491            result => result.map(|_| ()),
6492        }
6493    }
6494
6495    /// §5.11/§10.3: drop every pending commit whose push encode fails with
6496    /// `client.encrypt_failed` (an unresolvable key id or an unknown key) and
6497    /// raise one durable rejection per commit. `sync()` never aborts for an
6498    /// encode failure. Mirrors §7.4.4's `drop_incompatible_outbox`, including
6499    /// undoing the dropped commit's purely-optimistic rows.
6500    /// §7.1/§7.4.4: drop pending commits whose values the current codec refuses
6501    /// (an envelope or a type an earlier version accepted). `with_encryption`
6502    /// additionally drops commits whose §5.11 encryption cannot resolve a key;
6503    /// the startup boundary leaves those to the send seam, where hosts configure
6504    /// keys after open.
6505    /// Recovery runs as one observation. An enclosing observation (the guarded
6506    /// schema reset) passes its `batch`, so the rejection, outcome, and table
6507    /// domains join that change event instead of publishing a second one.
6508    fn drop_unencodable_outbox(
6509        &mut self,
6510        with_encryption: bool,
6511        batch: Option<&mut ChangeAccumulator>,
6512    ) -> Result<bool, String> {
6513        let mut failures = Vec::new();
6514        for commit in self.outbox.iter() {
6515            for op in &commit.ops {
6516                let Some(table) = self.schema.table(&op.table).filter(|_| op.upsert) else {
6517                    continue;
6518                };
6519                // An insert lacking a non-nullable column is left to
6520                // `drop_incompatible_outbox` (§7.4.4).
6521                let Some(values) = op.presented_values(table) else {
6522                    continue;
6523                };
6524                let fallback = self.stored_key_fallback(table, &op.row_id)?;
6525                let failure = match encode_sparse_row_json(
6526                    table,
6527                    &op.row_id,
6528                    &values,
6529                    &self.encryption,
6530                    fallback.as_ref(),
6531                ) {
6532                    Err(error) if error.starts_with(ENCRYPT_FAILED_CODE) => {
6533                        with_encryption.then_some((error, ENCRYPT_FAILED_CODE))
6534                    }
6535                    // §7.1/§7.4.4: a value the current codec refuses was
6536                    // persisted by a version that accepted it; it leaves
6537                    // through the incompatible-outbox path, so the FIFO
6538                    // queue never wedges.
6539                    Err(error) if error.starts_with(INVALID_REQUEST_CODE) => Some((
6540                        LEGACY_VALUE_REFUSAL_MESSAGE.to_owned(),
6541                        OUTBOX_INCOMPATIBLE_CODE,
6542                    )),
6543                    _ => None,
6544                };
6545                if let Some((error, code)) = failure {
6546                    failures.push((commit.clone(), error, code));
6547                    break;
6548                }
6549            }
6550        }
6551        if failures.is_empty() {
6552            return Ok(false);
6553        }
6554        // §7.4.4: one observation covers the durable outcome writes, the outbox
6555        // removal, the in-memory drop, and the overlay rebuild, so a storage fault
6556        // leaves the replica and the client unchanged, a successful recovery
6557        // publishes one batch with the rejection, outcome, and table domains, and
6558        // an ordinary open with nothing to recover publishes nothing.
6559        let rollback = self.capture();
6560        let owns_observation = batch.is_none();
6561        let owns_transaction = self.conn.is_autocommit();
6562        let mut domains = ChangeAccumulator::default();
6563        if owns_observation {
6564            self.begin_observation("syncular_value_recovery")?;
6565        }
6566        let dropped = (|| -> Result<(), String> {
6567            for (commit, message, code) in failures {
6568                if self.retain_failed_commits && code == ENCRYPT_FAILED_CODE {
6569                    self.retain_failed_commit(&commit.client_commit_id, &commit.ops)?;
6570                }
6571                let representable = commit.ops.iter().all(Self::outbox_op_is_journalable);
6572                let rejection = RejectionRecord {
6573                    client_commit_id: commit.client_commit_id.clone(),
6574                    op_index: 0,
6575                    code: code.to_owned(),
6576                    message,
6577                    retryable: false,
6578                    details: (code == OUTBOX_INCOMPATIBLE_CODE).then(|| RejectionDetails {
6579                        field_paths: None,
6580                        reason: Some(INVALID_STORED_VALUES_REASON.to_owned()),
6581                        required_action: None,
6582                        references: None,
6583                    }),
6584                    operation: representable
6585                        .then(|| commit.ops.first())
6586                        .flatten()
6587                        .map(CommitOperation::from),
6588                };
6589                self.persist_commit_outcome(
6590                    &commit.client_commit_id,
6591                    CommitOutcomeStatus::Rejected,
6592                    &[CommitOperationOutcome::Error {
6593                        rejection: rejection.clone(),
6594                    }],
6595                    representable.then_some(commit.ops.as_slice()),
6596                )?;
6597                self.delete_outbox_persisted(&commit.client_commit_id)?;
6598                self.outbox
6599                    .retain(|candidate| candidate.client_commit_id != commit.client_commit_id);
6600                self.rejections.push(rejection);
6601                for operation in &commit.ops {
6602                    domains.table(operation.table.as_str());
6603                }
6604            }
6605            domains.status = true;
6606            domains.rejections = true;
6607            domains.outcomes = true;
6608            self.prune_commit_outcomes()?;
6609            // The visible projection is re-derived from the surviving base + outbox.
6610            self.overlay_dirty.set(true);
6611            self.rebuild_overlay()?;
6612            Ok(())
6613        })();
6614        let result = match batch {
6615            Some(batch) => {
6616                if dropped.is_ok() {
6617                    for table in domains.tables.keys() {
6618                        batch.table(table.as_str());
6619                    }
6620                    batch.status |= domains.status;
6621                    batch.rejections |= domains.rejections;
6622                    batch.outcomes |= domains.outcomes;
6623                }
6624                dropped
6625            }
6626            None => {
6627                dropped.and_then(|()| self.finish_observation("syncular_value_recovery", domains))
6628            }
6629        };
6630        if let Err(error) = result {
6631            if owns_observation {
6632                if owns_transaction {
6633                    if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
6634                        let mut failure = self.storage_failure.borrow_mut();
6635                        let failure =
6636                            failure.get_or_insert_with(|| QueryReadFailure::from(error.clone()));
6637                        failure.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
6638                    }
6639                } else {
6640                    self.rollback_observation("syncular_value_recovery");
6641                }
6642            }
6643            self.restore(rollback);
6644            return Err(error);
6645        }
6646        Ok(true)
6647    }
6648
6649    pub fn pending_commit_ids(&self) -> Vec<String> {
6650        self.outbox
6651            .iter()
6652            .map(|c| c.client_commit_id.clone())
6653            .collect()
6654    }
6655
6656    /// §6.1: the sparse push payload bytes of every pending upsert operation,
6657    /// FIFO — exactly what the next push round would put on the wire. Used by
6658    /// conformance B.21(h) to pin cross-core byte identity.
6659    pub fn pending_payloads(&self) -> Result<Vec<Vec<u8>>, String> {
6660        let mut payloads = Vec::new();
6661        for commit in self.outbox.iter() {
6662            for op in &commit.ops {
6663                let Some(table) = self.schema.table(&op.table).filter(|_| op.upsert) else {
6664                    continue;
6665                };
6666                let Some(values) = op.presented_values(table) else {
6667                    continue;
6668                };
6669                let fallback = self.stored_key_fallback(table, &op.row_id)?;
6670                if let Ok(payload) = encode_sparse_row_json(
6671                    table,
6672                    &op.row_id,
6673                    &values,
6674                    &self.encryption,
6675                    fallback.as_ref(),
6676                ) {
6677                    payloads.push(payload);
6678                }
6679            }
6680        }
6681        Ok(payloads)
6682    }
6683
6684    pub fn conflicts(&self) -> &[ConflictRecord] {
6685        &self.conflicts
6686    }
6687
6688    pub fn rejections(&self) -> &[RejectionRecord] {
6689        &self.rejections
6690    }
6691
6692    pub fn resolve_commit_outcome(
6693        &mut self,
6694        input: ResolveCommitOutcomeInput,
6695    ) -> Result<CommitOutcome, String> {
6696        let current = self
6697            .commit_outcome(&input.client_commit_id)?
6698            .ok_or_else(|| {
6699                format!(
6700                    "sync.outcome_not_found: no durable outcome exists for {:?}",
6701                    input.client_commit_id
6702                )
6703            })?;
6704        if current.resolution != CommitOutcomeResolution::Active {
6705            return Ok(current);
6706        }
6707        if input.resolution == CommitOutcomeResolution::Active {
6708            return Err("sync.invalid_request: resolution must leave active state".to_owned());
6709        }
6710        match input.resolution {
6711            CommitOutcomeResolution::Superseded => {
6712                let replacement = input
6713                    .replacement_client_commit_id
6714                    .as_deref()
6715                    .filter(|value| !value.is_empty() && *value != input.client_commit_id)
6716                    .ok_or_else(|| {
6717                        "sync.invalid_request: superseded outcomes require a distinct replacementClientCommitId"
6718                            .to_owned()
6719                    })?;
6720                let _ = replacement;
6721            }
6722            _ if input.replacement_client_commit_id.is_some() => {
6723                return Err(
6724                    "sync.invalid_request: replacementClientCommitId is valid only for superseded outcomes"
6725                        .to_owned(),
6726                );
6727            }
6728            _ => {}
6729        }
6730        let allowed = match current.status {
6731            CommitOutcomeStatus::Conflict => matches!(
6732                input.resolution,
6733                CommitOutcomeResolution::ResolvedKeepServer | CommitOutcomeResolution::Superseded
6734            ),
6735            CommitOutcomeStatus::Rejected => {
6736                matches!(
6737                    input.resolution,
6738                    CommitOutcomeResolution::Superseded
6739                        | CommitOutcomeResolution::ResolvedKeepServer
6740                )
6741            }
6742            CommitOutcomeStatus::Applied | CommitOutcomeStatus::Cached => {
6743                input.resolution == CommitOutcomeResolution::Dismissed
6744            }
6745        };
6746        if !allowed {
6747            return Err(format!(
6748                "sync.invalid_request: resolution {:?} is invalid for {:?} outcome",
6749                input.resolution, current.status
6750            ));
6751        }
6752
6753        let rollback = self.capture();
6754        self.begin_observation("syncular_outcome_resolution")?;
6755        let result = (|| {
6756            self.conn
6757                .execute(
6758                    "UPDATE _syncular_commit_outcomes
6759                        SET resolution = ?1, resolved_at_ms = ?2,
6760                            replacement_client_commit_id = ?3
6761                      WHERE client_commit_id = ?4 AND resolution = 'active'",
6762                    rusqlite::params![
6763                        Self::outcome_resolution_name(input.resolution),
6764                        self.clock_now_ms(),
6765                        input.replacement_client_commit_id,
6766                        input.client_commit_id
6767                    ],
6768                )
6769                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6770            let resolved = self
6771                .commit_outcome(&current.client_commit_id)?
6772                .ok_or_else(|| "sync.outcome_not_found: outcome disappeared".to_owned())?;
6773            self.conn
6774                .execute(
6775                    "DELETE FROM _syncular_failed_commits WHERE client_commit_id = ?1",
6776                    rusqlite::params![input.client_commit_id],
6777                )
6778                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6779            for (commit, _) in self.failed_commits.iter() {
6780                if commit.client_commit_id == input.client_commit_id {
6781                    for operation in &commit.ops {
6782                        self.overlay_dirty.table(&operation.table);
6783                    }
6784                }
6785            }
6786            self.failed_commits
6787                .retain(|(commit, _)| commit.client_commit_id != input.client_commit_id);
6788            self.prune_commit_outcomes()?;
6789            Ok(resolved)
6790        })();
6791        let resolved = match result {
6792            Ok(outcome) => outcome,
6793            Err(error) => {
6794                self.rollback_observation("syncular_outcome_resolution");
6795                self.restore(rollback);
6796                return Err(error);
6797            }
6798        };
6799        self.conflicts
6800            .retain(|record| record.client_commit_id != current.client_commit_id);
6801        self.rejections
6802            .retain(|record| record.client_commit_id != current.client_commit_id);
6803        let mut batch = ChangeAccumulator {
6804            conflicts: current.status == CommitOutcomeStatus::Conflict,
6805            rejections: current.status == CommitOutcomeStatus::Rejected,
6806            outcomes: true,
6807            ..ChangeAccumulator::default()
6808        };
6809        if let Some(operations) = &current.operations {
6810            for operation in operations {
6811                batch.table(&operation.table);
6812            }
6813        }
6814        if let Err(error) = self
6815            .rebuild_overlay_if_dirty()
6816            .and_then(|()| self.finish_observation("syncular_outcome_resolution", batch))
6817        {
6818            self.rollback_observation("syncular_outcome_resolution");
6819            self.restore(rollback);
6820            return Err(error);
6821        }
6822        Ok(resolved)
6823    }
6824
6825    pub fn schema_floor(&self) -> Option<&SchemaFloor> {
6826        self.schema_floor.as_ref()
6827    }
6828
6829    /// §7.3.5: the client's opaque auth-lease state, if any.
6830    pub fn lease_state(&self) -> Option<&LeaseState> {
6831        self.lease_state.as_ref()
6832    }
6833
6834    /// §7.3.5: record a request-level lease error (stop-and-surface). Only
6835    /// the two lease codes set it; other errors leave leaseState untouched.
6836    fn record_lease_error(&mut self, code: &str) -> Result<(), String> {
6837        if code != "sync.auth_lease_required" && code != "sync.auth_lease_revoked" {
6838            return Ok(());
6839        }
6840        let mut next = self.lease_state.clone().unwrap_or_default();
6841        next.error_code = Some(code.to_owned());
6842        self.set_lease_state(Some(next))
6843    }
6844
6845    fn set_lease_state(&mut self, next: Option<LeaseState>) -> Result<(), String> {
6846        if self.lease_state == next {
6847            return Ok(());
6848        }
6849        self.begin_observation("syncular_lease")?;
6850        let rollback = self.capture();
6851        self.lease_state = next;
6852        let persisted = match &self.lease_state {
6853            Some(lease) => serde_json::to_string(lease)
6854                .map_err(|error| error.to_string())
6855                .and_then(|json| self.set_meta(LEASE_STATE_KEY, &json)),
6856            None => self.delete_meta(LEASE_STATE_KEY),
6857        };
6858        let batch = ChangeAccumulator {
6859            status: true,
6860            ..ChangeAccumulator::default()
6861        };
6862        if let Err(error) =
6863            persisted.and_then(|()| self.finish_observation("syncular_lease", batch))
6864        {
6865            self.rollback_observation("syncular_lease");
6866            self.restore(rollback);
6867            return Err(error);
6868        }
6869        Ok(())
6870    }
6871
6872    fn set_schema_floor(&mut self, next: Option<SchemaFloor>) -> Result<(), String> {
6873        if self.schema_floor == next {
6874            return Ok(());
6875        }
6876        self.begin_observation("syncular_schema_floor")?;
6877        let rollback = self.capture();
6878        self.stopped = next.is_some();
6879        self.schema_floor = next;
6880        let persisted = match &self.schema_floor {
6881            Some(floor) => serde_json::to_string(floor)
6882                .map_err(|error| error.to_string())
6883                .and_then(|json| self.set_meta(SCHEMA_FLOOR_KEY, &json)),
6884            None => self.delete_meta(SCHEMA_FLOOR_KEY),
6885        };
6886        let batch = ChangeAccumulator {
6887            status: true,
6888            ..ChangeAccumulator::default()
6889        };
6890        if let Err(error) =
6891            persisted.and_then(|()| self.finish_observation("syncular_schema_floor", batch))
6892        {
6893            self.rollback_observation("syncular_schema_floor");
6894            self.restore(rollback);
6895            return Err(error);
6896        }
6897        Ok(())
6898    }
6899
6900    fn set_upgrading(&mut self, value: bool) {
6901        if self.upgrading == value {
6902            return;
6903        }
6904        if self.begin_observation("syncular_upgrading").is_err() {
6905            return;
6906        }
6907        self.upgrading = value;
6908        let batch = ChangeAccumulator {
6909            status: true,
6910            ..ChangeAccumulator::default()
6911        };
6912        if self
6913            .finish_observation("syncular_upgrading", batch)
6914            .is_err()
6915        {
6916            self.rollback_observation("syncular_upgrading");
6917        }
6918    }
6919
6920    pub fn sync_needed(&self) -> bool {
6921        self.sync_needed
6922    }
6923
6924    pub fn read_rows(&self, table: &str) -> Result<Vec<RowState>, String> {
6925        let schema_table = self
6926            .schema
6927            .table(table)
6928            .ok_or_else(|| format!("unknown table {table:?}"))?;
6929        let sql = format!(
6930            "SELECT * FROM {} ORDER BY {} ASC",
6931            visible_table(table),
6932            quote_ident(&schema_table.primary_key)
6933        );
6934        let mut stmt = self
6935            .conn
6936            .prepare(&sql)
6937            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6938        let mut rows = stmt
6939            .query([])
6940            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6941        let mut out = Vec::new();
6942        while let Some(row) = rows
6943            .next()
6944            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
6945        {
6946            let mut values = Map::new();
6947            for (i, column) in schema_table.columns.iter().enumerate() {
6948                let value = row
6949                    .get_ref(i)
6950                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6951                values.insert(column.name.clone(), sql_ref_to_json(column, value));
6952            }
6953            let version: i64 = row
6954                .get(schema_table.columns.len())
6955                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6956            let row_id = match values.get(&schema_table.primary_key) {
6957                Some(Value::String(s)) => s.clone(),
6958                Some(Value::Number(n)) => n.to_string(),
6959                Some(Value::Bool(b)) => b.to_string(),
6960                other => format!("{}", other.cloned().unwrap_or(Value::Null)),
6961            };
6962            let sealed_key = if schema_table.has_encrypted_columns() {
6963                row.get(schema_table.columns.len() + 1)
6964                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
6965            } else {
6966                None
6967            };
6968            out.push(RowState {
6969                row_id,
6970                version,
6971                values,
6972                sealed_key,
6973            });
6974        }
6975        Ok(out)
6976    }
6977
6978    // -- §5.10.5 native CRDT (the `crdt-yjs` feature) --------------------------
6979    //
6980    // The Rust face of the §5.10.4 client model: a local crdt edit loads the
6981    // current stored (server-merged ⊕ pending-overlay) column bytes, applies
6982    // the op with `yrs`, re-encodes the whole doc state, and records a
6983    // baseVersion-less sparse upsert carrying only the primary key and the
6984    // crdt column (§5.10.3, §6.6). No local merge — merging is server-side;
6985    // the overlay re-materializes the edit immediately (optimistic apply,
6986    // §7.1) and the server-merged bytes arrive on the next pull, idempotently.
6987    // Byte-compatible with `@syncular/crdt-yjs`.
6988
6989    /// The current stored value of a `crdt` column for one row — the visible
6990    /// (optimistic) bytes, or `None` when the row is absent or the column is
6991    /// NULL (the empty document, §5.10.1). Errors if the column is not a
6992    /// `crdt` column (guards the app against a typo'd column name).
6993    #[cfg(feature = "crdt-yjs")]
6994    fn crdt_column_bytes(
6995        &self,
6996        table: &str,
6997        row_id: &str,
6998        column: &str,
6999    ) -> Result<Option<Vec<u8>>, String> {
7000        let schema_table = self
7001            .schema
7002            .table(table)
7003            .ok_or_else(|| format!("unknown table {table:?}"))?;
7004        let col = schema_table
7005            .columns
7006            .iter()
7007            .find(|c| c.name == column)
7008            .ok_or_else(|| format!("table {table:?} has no column {column:?}"))?;
7009        if col.ty != ColumnType::Crdt {
7010            return Err(format!("column {column:?} is not a crdt column (§5.10.1)"));
7011        }
7012        let sql = format!(
7013            "SELECT {} FROM {} WHERE {}",
7014            quote_ident(column),
7015            visible_table(table),
7016            row_id_predicate(schema_table)
7017        );
7018        let bytes: Option<Vec<u8>> = self
7019            .conn
7020            .prepare_cached(&sql)
7021            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
7022            .query_row(rusqlite::params![row_id], |row| {
7023                row.get::<_, Option<Vec<u8>>>(0)
7024            })
7025            .map_err(|e| match e {
7026                rusqlite::Error::QueryReturnedNoRows => "no such row".to_owned(),
7027                other => Self::sqlite_failure(&self.storage_failure, other),
7028            })?;
7029        Ok(bytes)
7030    }
7031
7032    /// §5.10.4 materialize: the collaborative text of a `crdt` column, decoded
7033    /// from the stored bytes with `yrs` — `YjsColumn.text(name).toString()`.
7034    /// An absent row / NULL column is the empty document (empty string).
7035    #[cfg(feature = "crdt-yjs")]
7036    pub fn crdt_text(
7037        &self,
7038        table: &str,
7039        row_id: &str,
7040        column: &str,
7041        name: &str,
7042    ) -> Result<String, String> {
7043        let bytes = self
7044            .crdt_column_bytes(table, row_id, column)?
7045            .unwrap_or_default();
7046        crate::crdt::text(&bytes, name)
7047    }
7048
7049    /// §5.10.4 push-an-update: apply a text insert to a `crdt` column and
7050    /// record the resulting crdt-only sparse operation. Returns the enqueued
7051    /// `clientCommitId`.
7052    #[cfg(feature = "crdt-yjs")]
7053    pub fn crdt_insert_text(
7054        &mut self,
7055        table: &str,
7056        row_id: &str,
7057        column: &str,
7058        name: &str,
7059        index: u32,
7060        value: &str,
7061    ) -> Result<String, ClientError> {
7062        let prior = self.storage_failure.borrow_mut().take();
7063        let outcome = (|| -> Result<String, ClientError> {
7064            let current = self
7065                .crdt_column_bytes(table, row_id, column)?
7066                .unwrap_or_default();
7067            let update = crate::crdt::insert_text(&current, name, index, value)?;
7068            self.perform_crdt_push_update(table, row_id, column, &update)
7069        })();
7070        self.finish_authoring(prior, outcome)
7071    }
7072
7073    /// §5.10.4 push-an-update: apply a text delete to a `crdt` column and
7074    /// record the resulting crdt-only sparse operation. Returns the enqueued
7075    /// `clientCommitId`.
7076    #[cfg(feature = "crdt-yjs")]
7077    pub fn crdt_delete_text(
7078        &mut self,
7079        table: &str,
7080        row_id: &str,
7081        column: &str,
7082        name: &str,
7083        index: u32,
7084        len: u32,
7085    ) -> Result<String, ClientError> {
7086        let prior = self.storage_failure.borrow_mut().take();
7087        let outcome = (|| -> Result<String, ClientError> {
7088            let current = self
7089                .crdt_column_bytes(table, row_id, column)?
7090                .unwrap_or_default();
7091            let update = crate::crdt::delete_text(&current, name, index, len)?;
7092            self.perform_crdt_push_update(table, row_id, column, &update)
7093        })();
7094        self.finish_authoring(prior, outcome)
7095    }
7096
7097    /// §5.10.4 generic escape hatch: apply an arbitrary Yjs update onto a
7098    /// `crdt` column's current state and record the resulting crdt-only sparse
7099    /// operation. The app authored the update with its own `yrs` model.
7100    /// Returns the enqueued `clientCommitId`.
7101    #[cfg(feature = "crdt-yjs")]
7102    pub fn crdt_apply_update(
7103        &mut self,
7104        table: &str,
7105        row_id: &str,
7106        column: &str,
7107        update: &[u8],
7108    ) -> Result<String, ClientError> {
7109        let prior = self.storage_failure.borrow_mut().take();
7110        let outcome = (|| -> Result<String, ClientError> {
7111            let current = self
7112                .crdt_column_bytes(table, row_id, column)?
7113                .unwrap_or_default();
7114            let next = crate::crdt::apply_update(&current, update)?;
7115            self.perform_crdt_push_update(table, row_id, column, &next)
7116        })();
7117        self.finish_authoring(prior, outcome)
7118    }
7119
7120    /// Shared tail of the crdt edit methods: record a §6.1 sparse crdt-only
7121    /// upsert — the primary key plus the crdt column, no `baseVersion` — and
7122    /// enqueue it. The server merges the crdt column and leaves every other
7123    /// column untouched (§5.10.3), so a baseVersion-less crdt edit cannot
7124    /// clobber a concurrent edit to another column. A locally absent row
7125    /// records the same partial operation; the server answers per §6.2.
7126    #[cfg(feature = "crdt-yjs")]
7127    fn perform_crdt_push_update(
7128        &mut self,
7129        table: &str,
7130        row_id: &str,
7131        column: &str,
7132        crdt_bytes: &[u8],
7133    ) -> Result<String, ClientError> {
7134        let mut bytes_obj = Map::new();
7135        bytes_obj.insert("$bytes".to_owned(), Value::from(bytes_to_hex(crdt_bytes)));
7136        let mut partial = Map::new();
7137        partial.insert(column.to_owned(), Value::Object(bytes_obj));
7138        self.perform_patch(table, row_id, partial, None)
7139    }
7140
7141    /// Run an arbitrary read-only SQL query against the local database and
7142    /// return each row as a `column-name → JSON value` map. This is the seam
7143    /// the React `useSyncQuery` live-query API needs (it takes app-authored
7144    /// SQL over the visible tables/views, not a fixed table read like
7145    /// [`read_rows`]).
7146    ///
7147    /// Bound `params` are the driver value forms: JSON strings/numbers/bools/
7148    /// null bind directly; a `{"$bytes": hex}` object binds as a BLOB — the
7149    /// same envelope the command surface uses everywhere else. Output BLOB
7150    /// columns come back as `{"$bytes": hex}` to round-trip cleanly.
7151    ///
7152    /// The result column typing is dynamic (SQLite's stored affinity), because
7153    /// arbitrary SQL can alias, join, and compute — there is no schema column
7154    /// to consult per output cell, unlike [`read_rows`].
7155    pub fn query(&self, sql: &str, params: &[QueryValue]) -> Result<Vec<QueryRow>, String> {
7156        query_connection(&self.conn, sql, params).map_err(|failure| failure.to_string())
7157    }
7158
7159    /// Repository benchmark and conformance access, present only with `bench-internals`.
7160    #[cfg(feature = "bench-internals")]
7161    #[doc(hidden)]
7162    pub fn benchmark_connection(&mut self) -> &mut Connection {
7163        &mut self.conn
7164    }
7165
7166    /// Private benchmark intervals; absent from ordinary builds and diagnostics.
7167    #[cfg(feature = "bench-internals")]
7168    #[doc(hidden)]
7169    pub fn benchmark_phases(
7170        &self,
7171        enabled: Option<bool>,
7172        reset: bool,
7173    ) -> Result<Option<Value>, String> {
7174        self.benchmark_phases.configure(enabled, reset)?;
7175        Ok(self.benchmark_phases.snapshot())
7176    }
7177
7178    /// One serialized, read-only authority observation, including durable coverage.
7179    pub fn authority_snapshot(&self) -> Result<Value, String> {
7180        validate_authority_reads(&self.schema, &self.limits.authority_reads)?;
7181        if self.limits.authority_reads.is_empty() {
7182            return Err("client.authority_read_forbidden: no authority reads were declared at client creation".into());
7183        }
7184        self.conn
7185            .execute_batch("SAVEPOINT authority_snapshot")
7186            .map_err(|e| e.to_string())?;
7187        let result = (|| {
7188            let revision = self
7189                .get_meta(LOCAL_REVISION_KEY)?
7190                .ok_or("sync.local_corrupt: authority revision is missing")?;
7191            let parsed_revision = revision
7192                .parse::<u64>()
7193                .map_err(|_| "sync.local_corrupt: authority revision is invalid")?;
7194            if parsed_revision.to_string() != revision {
7195                return Err("sync.local_corrupt: authority revision is invalid".into());
7196            }
7197            let mut tables = Vec::new();
7198            for read in &self.limits.authority_reads {
7199                let table = self
7200                    .schema
7201                    .table(&read.table)
7202                    .ok_or("client.authority_read_forbidden: authority table is not declared")?;
7203                let scalar = |column: &str, value: &Value| -> Result<String, String> {
7204                    let ty = table
7205                        .columns
7206                        .iter()
7207                        .find(|c| c.name == column)
7208                        .expect("validated authority column")
7209                        .ty;
7210                    if ty == ColumnType::Boolean {
7211                        if let Some(value) = value.as_bool() {
7212                            return Ok(value.to_string());
7213                        }
7214                        return match value.as_i64() {
7215                            Some(0) => Ok("false".into()),
7216                            Some(1) => Ok("true".into()),
7217                            _ => Err("sync.local_corrupt: authority boolean is invalid".into()),
7218                        };
7219                    }
7220                    match value {
7221                        Value::String(s) => Ok(s.clone()),
7222                        Value::Number(n) => Ok(n.to_string()),
7223                        Value::Object(o) if ty == ColumnType::Integer && o.len() == 1 => o
7224                            .get("$bigint")
7225                            .and_then(Value::as_str)
7226                            .map(str::to_owned)
7227                            .ok_or("sync.local_corrupt: authority integer is invalid".into()),
7228                        _ => {
7229                            Err("sync.local_corrupt: authority scalar is missing or invalid".into())
7230                        }
7231                    }
7232                };
7233                let matches = |row: &QueryRow| -> Result<bool, String> {
7234                    for (variable, values) in &read.scopes {
7235                        let column = table
7236                            .scope_column(variable)
7237                            .expect("validated authority scope");
7238                        let value = row
7239                            .get(column)
7240                            .ok_or("sync.local_corrupt: authority scope column is missing")?;
7241                        if value.is_null() || !values.contains(&scalar(column, value)?) {
7242                            return Ok(false);
7243                        }
7244                    }
7245                    Ok(true)
7246                };
7247                let selected = read
7248                    .columns
7249                    .iter()
7250                    .map(|c| quote_ident(c))
7251                    .collect::<Vec<_>>()
7252                    .join(",");
7253                let mut intent = BTreeSet::new();
7254                let mut relevant = BTreeSet::new();
7255                for row in query_connection(&self.conn,
7256                    "SELECT id,values_json FROM (SELECT json_extract(value,'$.table') AS tbl,json_extract(value,'$.rowId') AS id,json_extract(value,'$.values') AS values_json FROM _syncular_outbox,json_each(ops_json) UNION ALL SELECT json_extract(value,'$.table'),json_extract(value,'$.rowId'),json_extract(value,'$.values') FROM _syncular_failed_commits,json_each(operations_json) UNION ALL SELECT tbl,id,intent_json FROM _syncular_acked_rows) WHERE tbl=?",
7257                    &[Value::String(read.table.clone())]).map_err(|e| e.to_string())? {
7258                    let id = row.get("id").and_then(Value::as_str).ok_or("sync.local_corrupt: authority intent id is invalid")?.to_owned();
7259                    intent.insert(id.clone());
7260                    if let Some(raw) = row.get("values_json").and_then(Value::as_str) {
7261                        let values: QueryRow = serde_json::from_str(raw).map_err(|_| "sync.local_corrupt: authority intent values are invalid")?;
7262                        if read.scopes.keys().all(|v| values.contains_key(table.scope_column(v).expect("validated scope"))) && matches(&values)? {relevant.insert(id);}
7263                    }
7264                }
7265                let mut rows = BTreeMap::new();
7266                for full in [base_table(&read.table), visible_table(&read.table)] {
7267                    for mut values in query_connection(
7268                        &self.conn,
7269                        &format!(
7270                            "SELECT {selected},_syncular_version AS authority_version FROM {full}"
7271                        ),
7272                        &[],
7273                    )
7274                    .map_err(|e| e.to_string())?
7275                    {
7276                        if !matches(&values)? {
7277                            continue;
7278                        }
7279                        let id = scalar(
7280                            &table.primary_key,
7281                            values
7282                                .get(&table.primary_key)
7283                                .expect("selected primary key"),
7284                        )?;
7285                        relevant.insert(id.clone());
7286                        if full == base_table(&read.table) {
7287                            let version = values
7288                                .remove("authority_version")
7289                                .expect("selected version");
7290                            let version = version
7291                                .as_i64()
7292                                .ok_or("sync.local_corrupt: authority server version is invalid")?;
7293                            if version >= 0 {
7294                                rows.insert(id.clone(), json!({"values": values, "version": version, "hasLocalIntent": intent.contains(&id)}));
7295                            }
7296                        }
7297                    }
7298                }
7299                intent.retain(|id| relevant.contains(id));
7300                let rows = rows.into_values().collect::<Vec<_>>();
7301                // Every unit covering the table, as its (unit, table) pair (§4.4).
7302                let replica = self.stored(replica_cursor(&self.conn))?;
7303                let persisted = self
7304                    .stored(load_units(&self.conn))?
7305                    .into_iter()
7306                    .filter(|unit| covers(table, &unit.scopes))
7307                    .map(|unit| {
7308                        let effective = unit.echo.as_deref().and_then(|echo| {
7309                            pair_scopes(&unit.scopes, &read.table, Some(echo))
7310                        });
7311                        let requested = read
7312                            .scopes
7313                            .keys()
7314                            .map(|v| (v.clone(), json!(unit.scopes.get(v).into_iter().collect::<Vec<_>>())))
7315                            .collect::<Map<_, _>>();
7316                        let complete = effective
7317                            .as_ref()
7318                            .is_some_and(|scopes| scopes.len() == read.scopes.len())
7319                            && unit.status == UnitStatus::Active
7320                            && !unit.bootstrapping();
7321                        let exposed = effective.map(|scopes| {
7322                            read.scopes
7323                                .keys()
7324                                .map(|v| {
7325                                    let held = scopes.iter().find(|(variable, _)| variable == v);
7326                                    (v.clone(), json!(held.map_or(&Vec::new(), |(_, values)| values)))
7327                                })
7328                                .collect::<Map<_, _>>()
7329                        });
7330                        json!({"requestedScopes": requested, "status": unit.status, "cursor": unit.cursor.unwrap_or(replica), "effectiveScopes": exposed, "complete": complete})
7331                    })
7332                    .collect::<Vec<_>>();
7333                fn covered(selectors: &[(&String, &Vec<String>)], candidates: &[&Value]) -> bool {
7334                    match selectors.split_first() {
7335                        None => !candidates.is_empty(),
7336                        Some(((variable, values), rest)) => values.iter().all(|value| {
7337                            covered(
7338                                rest,
7339                                &candidates
7340                                    .iter()
7341                                    .copied()
7342                                    .filter(|p| {
7343                                        p["effectiveScopes"][variable.as_str()]
7344                                            .as_array()
7345                                            .is_some_and(|held| {
7346                                                held.contains(&Value::String(value.clone()))
7347                                                    || held.contains(&Value::String("*".into()))
7348                                            })
7349                                    })
7350                                    .collect::<Vec<_>>(),
7351                            )
7352                        }),
7353                    }
7354                }
7355                let coverage = if covered(
7356                    &read.scopes.iter().collect::<Vec<_>>(),
7357                    &persisted
7358                        .iter()
7359                        .filter(|p| p["complete"] == true)
7360                        .collect::<Vec<_>>(),
7361                ) {
7362                    "complete"
7363                } else if persisted.iter().any(|p| {
7364                    p["status"] == "active"
7365                        && read.scopes.iter().all(|(v, values)| {
7366                            values.iter().any(|value| {
7367                                p["requestedScopes"][v].as_array().is_some_and(|held| {
7368                                    held.contains(&Value::String(value.clone()))
7369                                })
7370                            })
7371                        })
7372                }) {
7373                    "pending"
7374                } else {
7375                    "missing"
7376                };
7377                tables.push(json!({"table": read.table, "rows": rows, "localIntentRowIds": intent, "scopes": read.scopes, "coverage": coverage, "persisted": persisted}));
7378            }
7379            Ok(
7380                json!({"revision": revision, "complete": tables.iter().all(|t| t["coverage"] == "complete"), "tables": tables}),
7381            )
7382        })();
7383        match result {
7384            Ok(snapshot) => {
7385                self.conn
7386                    .execute_batch("RELEASE authority_snapshot")
7387                    .map_err(|e| e.to_string())?;
7388                Ok(snapshot)
7389            }
7390            Err(error) => {
7391                self.conn
7392                    .execute_batch("ROLLBACK TO authority_snapshot; RELEASE authority_snapshot")
7393                    .map_err(|e| format!("{error}; rollback failed: {e}"))?;
7394                Err(error)
7395            }
7396        }
7397    }
7398
7399    /// Rows, coverage, and local revision from one SQLite read snapshot.
7400    /// §7.5 atomic snapshot read. An `owner` records a failure in
7401    /// `diagnostics_snapshot().query_failures` until its next successful read.
7402    pub fn query_snapshot(
7403        &mut self,
7404        sql: &str,
7405        params: &[QueryValue],
7406        coverage: &[WindowCoverage],
7407        owner: Option<QueryOwner<'_>>,
7408    ) -> Result<QuerySnapshot, String> {
7409        let result = snapshot_connection(&self.conn, sql, params, coverage);
7410        if let Some(owner) = owner {
7411            self.record_query_read(owner, result.as_ref().err());
7412        }
7413        result.map_err(|failure| failure.to_string())
7414    }
7415
7416    /// §7.6: record the outcome of one owned snapshot read, including reads a
7417    /// host ran on a separate connection. A success removes the owner's entry;
7418    /// an identical failure leaves it unchanged; a changed failure moves last.
7419    pub fn record_query_read(&mut self, owner: QueryOwner<'_>, failure: Option<&QueryReadFailure>) {
7420        let position = self
7421            .query_failures
7422            .iter()
7423            .position(|entry| entry.id == owner.id);
7424        let Some(failure) = failure else {
7425            if let Some(index) = position {
7426                self.query_failures.remove(index);
7427            }
7428            return;
7429        };
7430        let tables: Vec<String> = BTreeSet::from_iter(owner.tables.iter().copied())
7431            .into_iter()
7432            .map(str::to_owned)
7433            .collect();
7434        let code = failure.code.unwrap_or("client.query_failed").to_owned();
7435        if position.is_some_and(|index| {
7436            let entry = &self.query_failures[index];
7437            entry.tables == tables && entry.code == code && entry.sqlite_code == failure.sqlite_code
7438        }) {
7439            return;
7440        }
7441        if let Some(index) = position {
7442            self.query_failures.remove(index);
7443        }
7444        self.query_failures.push(DiagnosticQueryFailure {
7445            id: owner.id.to_owned(),
7446            tables,
7447            code,
7448            sqlite_code: failure.sqlite_code,
7449            at_ms: self.clock_now_ms(),
7450        });
7451        if self.query_failures.len() > MAX_DIAGNOSTIC_QUERY_FAILURES {
7452            self.query_failures.remove(0);
7453        }
7454    }
7455
7456    // -- request building ---------------------------------------------------------
7457
7458    fn build_request(&self, url_capable: bool) -> Result<(Message, RequestMeta), Box<SyncOutcome>> {
7459        #[cfg(feature = "bench-internals")]
7460        let _phase = self.benchmark_phases.start(Phase::RequestPrepare);
7461        let log_epoch = self.get_meta(LOG_EPOCH_KEY).map_err(|message| {
7462            Box::new(SyncOutcome::Failed {
7463                error_code: "storage.failed".into(),
7464                message,
7465                details: None,
7466                recommended_action: None,
7467            })
7468        })?;
7469        let max_commits = self.limits.max_push_commits_per_request;
7470        let max_ops = self
7471            .limits
7472            .max_push_operations_per_request
7473            .unwrap_or(PUSH_OPS_PER_REQUEST);
7474        let byte_cap = self.limits.max_push_request_bytes;
7475        // §4.2/§5.4: bit 3 is advertised iff the transport can fetch a
7476        // bare URL — capability negotiation, decided per transport.
7477        let accept = self.limits.accept.unwrap_or(if url_capable {
7478            DEFAULT_ACCEPT | ACCEPT_SIGNED_URLS
7479        } else {
7480            DEFAULT_ACCEPT
7481        });
7482        let storage_failed = |message: String| {
7483            Box::new(SyncOutcome::Failed {
7484                error_code: "storage.failed".into(),
7485                message,
7486                details: None,
7487                recommended_action: None,
7488            })
7489        };
7490        let header = Frame::ReqHeader {
7491            client_id: self.client_id.clone(),
7492            schema_version: self.schema.version,
7493            log_epoch: log_epoch.clone(),
7494            outbox_id: self.get_meta(OUTBOX_ID_KEY).map_err(storage_failed)?,
7495        };
7496        // §1.5: the lead frames are REQ_HEADER plus the optional ROUND_FAILURE,
7497        // which stays ahead of every push, pull, unit and progress frame; the
7498        // byte-capacity probes below measure this exact sequence.
7499        let mut lead = vec![header];
7500        lead.extend(
7501            self.round_failure
7502                .iter()
7503                .map(|code| Frame::RoundFailure { code: code.clone() }),
7504        );
7505        let registered: Vec<UnitRecord> = self
7506            .stored(load_units(&self.conn))
7507            .map_err(storage_failed)?
7508            .into_iter()
7509            .filter(|unit| unit.status == UnitStatus::Active)
7510            .collect();
7511        let registration_digest = ssp2::registration::registration_digest(
7512            registered
7513                .iter()
7514                .map(|unit| scope_pairs(&unit.scopes))
7515                .collect::<Vec<_>>()
7516                .iter()
7517                .map(Vec::as_slice),
7518        );
7519        let echo_digest = self
7520            .get_meta(ECHO_DIGEST_KEY)
7521            .map_err(storage_failed)?
7522            .filter(|hex| !hex.is_empty())
7523            .map(|hex| {
7524                crate::values::hex_to_bytes(&hex)
7525                    .ok()
7526                    .and_then(|bytes| <[u8; 32]>::try_from(bytes).ok())
7527                    .ok_or_else(|| {
7528                        "sync.local_corrupt: persisted echo digest is invalid".to_owned()
7529                    })
7530            })
7531            .transpose()
7532            .map_err(storage_failed)?;
7533        let pull = Frame::PullHeader {
7534            limit_commits: self.limits.limit_commits.unwrap_or(0),
7535            limit_snapshot_rows: self.limits.limit_snapshot_rows.unwrap_or(0),
7536            max_snapshot_pages: self.limits.max_snapshot_pages.unwrap_or(0),
7537            accept,
7538            cursor: self
7539                .stored(replica_cursor(&self.conn))
7540                .map_err(storage_failed)?,
7541            registration_digest,
7542            echo_digest,
7543        };
7544        let registration_digest = bytes_to_hex(&registration_digest);
7545        // §4.1: the list travels when it changed since the server stored it,
7546        // after `unregistered` and after a reset (both clear the stored digest).
7547        let travels = self
7548            .get_meta(REGISTERED_DIGEST_KEY)
7549            .map_err(storage_failed)?
7550            .as_deref()
7551            != Some(registration_digest.as_str());
7552        let mut sub_frames: Vec<Frame> = Vec::new();
7553        if travels {
7554            sub_frames.extend(registered.iter().map(|unit| Frame::Unit {
7555                scopes: scope_pairs(&unit.scopes),
7556            }));
7557        }
7558        // §4.3: one PROGRESS per catching-up unit, in registration order.
7559        sub_frames.extend(registered.iter().filter_map(|unit| {
7560            Some(Frame::Progress {
7561                id: unit.id.clone(),
7562                cursor: unit.cursor?,
7563                bootstrap_state: unit.bootstrap_state.clone().map(RawJson),
7564            })
7565        }));
7566        // The fixed frames (order-independent size) and the header+pull probe
7567        // both encode canonically, so the byte budget is the exact wire size.
7568        let fixed_bytes = match byte_cap {
7569            Some(_) => {
7570                let mut frames = lead.clone();
7571                frames.push(pull.clone());
7572                frames.extend(sub_frames.iter().cloned());
7573                encode_message(&Message {
7574                    wire_version: WIRE_VERSION,
7575                    msg_kind: MsgKind::Request,
7576                    frames,
7577                })
7578                .len()
7579            }
7580            None => 0,
7581        };
7582        if let Some(cap) = byte_cap {
7583            if fixed_bytes > cap {
7584                return Err(Self::push_capacity_outcome("bytes", cap, fixed_bytes, None));
7585            }
7586        }
7587        let base_bytes = match byte_cap {
7588            Some(_) => {
7589                let mut frames = lead.clone();
7590                frames.push(pull.clone());
7591                encode_message(&Message {
7592                    wire_version: WIRE_VERSION,
7593                    msg_kind: MsgKind::Request,
7594                    frames,
7595                })
7596                .len()
7597            }
7598            None => 0,
7599        };
7600        let total_outbox = if log_epoch.is_some() {
7601            self.outbox.len()
7602        } else {
7603            0
7604        };
7605        let mut push_frames: Vec<Frame> = Vec::new();
7606        let mut pushed_ids = Vec::new();
7607        let mut ops_in_request = 0usize;
7608        let mut bytes_in_request = 0usize;
7609        let mut deferred_commits = 0usize;
7610        #[cfg(feature = "bench-internals")]
7611        let outbox_phase = self.benchmark_phases.start(Phase::OutboxEncode);
7612        for (index, commit) in self.outbox.iter().take(total_outbox).enumerate() {
7613            // §6.1/§7.1: the commit-count and operation caps are hard. A
7614            // first commit over the operation cap is a typed capacity error
7615            // and stays queued; a later one defers with the complete suffix.
7616            if max_commits.is_some_and(|cap| push_frames.len() >= cap) {
7617                deferred_commits = total_outbox - index;
7618                break;
7619            }
7620            if ops_in_request + commit.ops.len() > max_ops {
7621                if push_frames.is_empty() {
7622                    return Err(Self::push_capacity_outcome(
7623                        "operations",
7624                        max_ops,
7625                        commit.ops.len(),
7626                        Some(commit.client_commit_id.as_str()),
7627                    ));
7628                }
7629                deferred_commits = total_outbox - index;
7630                break;
7631            }
7632            // §7.1 chained base: the commit waits for the change of the commit
7633            // it chains on, which supplies its rebased version.
7634            if self.outbox[..index]
7635                .iter()
7636                .flat_map(|earlier| &earlier.ops)
7637                .any(|earlier| commit.ops.iter().any(|op| chains_on(earlier, op)))
7638            {
7639                deferred_commits = total_outbox - index;
7640                break;
7641            }
7642            let operations = commit
7643                .ops
7644                .iter()
7645                .map(|op| {
7646                    let payload = match self
7647                        .schema
7648                        .table(&op.table)
7649                        .and_then(|table| Some((table, op.presented_values(table)?)))
7650                    {
7651                        Some((table, values)) => {
7652                            // §0: outbox entries encode at send time with the
7653                            // current codec (validated at mutate()). §6.1: the
7654                            // values map's keys are the sparse presence set; an
7655                            // insert-intent operation presents every column (§7.1).
7656                            // §5.11: encrypted columns are encrypted here, with
7657                            // an absent key-id selector read from the stored row.
7658                            // `sync_inner` runs `drop_unencodable_outbox` first,
7659                            // so an encrypt failure cannot reach this encode; the
7660                            // author seam validated every other failure.
7661                            let fallback =
7662                                self.stored_key_fallback(table, &op.row_id)
7663                                    .map_err(|message| {
7664                                        Box::new(SyncOutcome::Failed {
7665                                            error_code: "storage.failed".into(),
7666                                            message,
7667                                            details: None,
7668                                            recommended_action: None,
7669                                        })
7670                                    })?;
7671                            encode_sparse_row_json(
7672                                table,
7673                                &op.row_id,
7674                                &values,
7675                                &self.encryption,
7676                                fallback.as_ref(),
7677                            )
7678                            .ok()
7679                        }
7680                        None => None,
7681                    };
7682                    Ok(ssp2::model::Operation {
7683                        table: op.table.clone(),
7684                        row_id: op.row_id.clone(),
7685                        op: if op.upsert { Op::Upsert } else { Op::Delete },
7686                        base_version: op.base_version,
7687                        payload,
7688                    })
7689                })
7690                .collect::<Result<Vec<_>, Box<SyncOutcome>>>()?;
7691            // §6.3.2, §7.1: the commit's outbox sequence.
7692            let client_seq = self
7693                .conn
7694                .prepare_cached("SELECT seq FROM _syncular_outbox WHERE commit_id = ?1")
7695                .and_then(|mut statement| {
7696                    statement.query_row([&commit.client_commit_id], |row| row.get(0))
7697                })
7698                .map_err(|error| {
7699                    storage_failed(Self::sqlite_failure(&self.storage_failure, error))
7700                })?;
7701            let frame = Frame::PushCommit {
7702                client_commit_id: commit.client_commit_id.clone(),
7703                client_seq,
7704                operations,
7705            };
7706            let frame_bytes = match byte_cap {
7707                Some(_) => {
7708                    let mut frames = lead.clone();
7709                    frames.push(frame.clone());
7710                    frames.push(pull.clone());
7711                    encode_message(&Message {
7712                        wire_version: WIRE_VERSION,
7713                        msg_kind: MsgKind::Request,
7714                        frames,
7715                    })
7716                    .len()
7717                        - base_bytes
7718                }
7719                None => 0,
7720            };
7721            // The head is always encoded so the caller can report the full
7722            // projected request size; a later commit defers once it no longer
7723            // fits.
7724            if let Some(cap) = byte_cap {
7725                if !push_frames.is_empty() && fixed_bytes + bytes_in_request + frame_bytes > cap {
7726                    deferred_commits = total_outbox - index;
7727                    break;
7728                }
7729            }
7730            push_frames.push(frame);
7731            pushed_ids.push(commit.client_commit_id.clone());
7732            ops_in_request += commit.ops.len();
7733            bytes_in_request += frame_bytes;
7734        }
7735        #[cfg(feature = "bench-internals")]
7736        drop(outbox_phase);
7737        if let Some(cap) = byte_cap {
7738            if fixed_bytes + bytes_in_request > cap {
7739                // The head alone does not fit this budget; the commit stays
7740                // queued and no transport round is attempted.
7741                let blocked = if push_frames.is_empty() {
7742                    None
7743                } else {
7744                    self.outbox.first().map(|c| c.client_commit_id.as_str())
7745                };
7746                return Err(Self::push_capacity_outcome(
7747                    "bytes",
7748                    cap,
7749                    fixed_bytes + bytes_in_request,
7750                    blocked,
7751                ));
7752            }
7753        }
7754        let mut frames = lead;
7755        frames.extend(push_frames);
7756        frames.push(pull);
7757        frames.extend(sub_frames);
7758        let message = Message {
7759            wire_version: WIRE_VERSION,
7760            msg_kind: MsgKind::Request,
7761            frames,
7762        };
7763        Ok((
7764            message,
7765            RequestMeta {
7766                pushed_ids,
7767                registration_digest,
7768                accept,
7769                deferred_commits,
7770                applied: BTreeMap::new(),
7771                deferred: BTreeSet::new(),
7772            },
7773        ))
7774    }
7775
7776    /// §7.1: a configured push capacity limit cannot carry this request.
7777    fn push_capacity_outcome(
7778        kind: &str,
7779        limit: usize,
7780        size: usize,
7781        client_commit_id: Option<&str>,
7782    ) -> Box<SyncOutcome> {
7783        let mut details = Map::new();
7784        details.insert("kind".to_owned(), Value::from(kind));
7785        details.insert("limit".to_owned(), Value::from(limit as u64));
7786        details.insert("size".to_owned(), Value::from(size as u64));
7787        if let Some(id) = client_commit_id {
7788            details.insert("clientCommitId".to_owned(), Value::from(id));
7789        }
7790        Box::new(SyncOutcome::Failed {
7791            error_code: "client.push_request_too_large".into(),
7792            message: "the push request exceeds a configured capacity limit".into(),
7793            details: Some(Value::Object(details)),
7794            recommended_action: None,
7795        })
7796    }
7797
7798    // -- sync -------------------------------------------------------------------
7799
7800    /// Cloneable observation handle; no client lock is needed to read it.
7801    pub fn progress(&self) -> ProgressObserver {
7802        self.progress.clone()
7803    }
7804
7805    pub fn sync(&mut self, transport: &mut dyn Transport) -> SyncOutcome {
7806        self.sync_bounded(transport, None)
7807    }
7808
7809    /// One round; `idle_bound` is the idle drive's round bound (§7.8).
7810    fn sync_bounded(
7811        &mut self,
7812        transport: &mut dyn Transport,
7813        idle_bound: Option<(u64, u32)>,
7814    ) -> SyncOutcome {
7815        let mut prepared = match self.prepare_sync_round(transport.supports_url_fetch()) {
7816            Ok(prepared) => prepared,
7817            Err(outcome) => return *outcome,
7818        };
7819        prepared.idle_bound = idle_bound;
7820        // The round's network deadline rides `PreparedSyncRound` and is scoped
7821        // to each exchange, so uploads, continuations, the main request, and
7822        // every segment fetch share one monotonic anchor without leaving
7823        // deadline state on a reused transport.
7824        let mut next = prepared;
7825        loop {
7826            match self.apply_sync_round(next.exchange(transport)) {
7827                crate::AppliedSyncRound::Continue(prepared) => next = prepared,
7828                crate::AppliedSyncRound::Complete {
7829                    outcome, controls, ..
7830                } => {
7831                    if self.transport_enabled {
7832                        for text in controls {
7833                            let _ = transport.realtime_send(&text);
7834                        }
7835                    } else {
7836                        self.disconnect_realtime(transport);
7837                    }
7838                    return outcome;
7839                }
7840            }
7841        }
7842    }
7843
7844    fn finish_sync_round(
7845        &mut self,
7846        started_at_ms: i64,
7847        attempt: u64,
7848        mut outcome: SyncOutcome,
7849    ) -> SyncOutcome {
7850        if let SyncOutcome::Failed {
7851            error_code,
7852            message,
7853            details,
7854            ..
7855        } = &mut outcome
7856        {
7857            if let Some((failure, code)) = self
7858                .storage_failure
7859                .borrow()
7860                .as_ref()
7861                .and_then(|failure| failure.code.map(|code| (failure, code)))
7862            {
7863                *error_code = code.to_owned();
7864                *message = match code {
7865                    "client.storage_busy" => "local SQLite storage is busy",
7866                    "client.storage_full" => "local SQLite storage is full",
7867                    "client.storage_io" => "local SQLite storage I/O failed",
7868                    "client.storage_corrupt" => "local SQLite storage is corrupt",
7869                    _ => unreachable!("unrecognized classified storage failure"),
7870                }
7871                .to_owned();
7872                *details = failure.details();
7873            }
7874        }
7875        let details = match &outcome {
7876            SyncOutcome::Failed { details, .. } => details.as_ref(),
7877            _ => None,
7878        };
7879        let cause_kind = details
7880            .and_then(|details| details.get("causeKind"))
7881            .and_then(Value::as_str)
7882            .and_then(|cause| match cause {
7883                "network" => Some(crate::TransportCauseKind::Network),
7884                "connect" => Some(crate::TransportCauseKind::Connect),
7885                "tls" => Some(crate::TransportCauseKind::Tls),
7886                "io" => Some(crate::TransportCauseKind::Io),
7887                "request" => Some(crate::TransportCauseKind::Request),
7888                "protocol" => Some(crate::TransportCauseKind::Protocol),
7889                "timeout" => Some(crate::TransportCauseKind::Timeout),
7890                "round_deadline" => Some(crate::TransportCauseKind::RoundDeadline),
7891                "aborted" => Some(crate::TransportCauseKind::Aborted),
7892                "body" => Some(crate::TransportCauseKind::Body),
7893                "status" => Some(crate::TransportCauseKind::Status),
7894                "redirect" => Some(crate::TransportCauseKind::Redirect),
7895                "unknown" => Some(crate::TransportCauseKind::Unknown),
7896                _ => None,
7897            });
7898        let http_status = if matches!(
7899            cause_kind,
7900            Some(
7901                crate::TransportCauseKind::Status
7902                    | crate::TransportCauseKind::Body
7903                    | crate::TransportCauseKind::Redirect
7904            )
7905        ) {
7906            details
7907                .and_then(|details| details.get("httpStatus"))
7908                .and_then(Value::as_f64)
7909                .filter(|status| status.fract() == 0.0 && (100.0..=599.0).contains(status))
7910                .map(|status| status as u16)
7911        } else {
7912            None
7913        };
7914        let retry_delay_ms = self.round_retry_delay_ms;
7915        self.progress.update(|p| match &outcome {
7916            SyncOutcome::Ok(report) | SyncOutcome::BudgetExhausted(report)
7917                if report.failed.is_empty() =>
7918            {
7919                p.state = ProgressState::Complete
7920            }
7921            SyncOutcome::Ok(_) | SyncOutcome::BudgetExhausted(_) => {
7922                p.state = ProgressState::Failed;
7923                p.error_code = Some("sync.scope_revoked".into());
7924            }
7925            SyncOutcome::Failed { error_code, .. } => {
7926                p.state = ProgressState::Failed;
7927                p.error_code = Some(Self::diagnostic_code(error_code));
7928                p.cause_kind = cause_kind;
7929                p.http_status = http_status;
7930                p.retry_delay_ms = retry_delay_ms;
7931            }
7932            SyncOutcome::RealtimeUnavailable { .. } => {
7933                p.state = ProgressState::Failed;
7934                p.error_code = Some(REALTIME_UNAVAILABLE_CODE.to_owned());
7935                p.retry_delay_ms = retry_delay_ms;
7936            }
7937        });
7938        let completed_at_ms = self.clock_now_ms();
7939        self.last_round = Some(match &outcome {
7940            SyncOutcome::Ok(report) | SyncOutcome::BudgetExhausted(report) => DiagnosticLastRound {
7941                epoch: self.epoch.clone(),
7942                attempt,
7943                status: "succeeded".to_owned(),
7944                started_at_ms,
7945                completed_at_ms,
7946                duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
7947                counters: Some(DiagnosticRoundCounters {
7948                    pushed: report.pushed,
7949                    applied: report.applied.len(),
7950                    rejected: report.rejected.len(),
7951                    retryable: report.retryable.len(),
7952                    conflicts: report.conflicts,
7953                    commits_applied: report.commits_applied,
7954                    segment_rows_applied: report.segment_rows_applied,
7955                    bootstrapping: report.bootstrapping.len(),
7956                    resets: report.resets.len(),
7957                    revoked: report.revoked.len(),
7958                    failed: report.failed.len(),
7959                    deferred_commits: report.deferred_commits,
7960                }),
7961                error_code: None,
7962                cause_kind: None,
7963                http_status: None,
7964            },
7965            SyncOutcome::Failed { error_code, .. } => DiagnosticLastRound {
7966                epoch: self.epoch.clone(),
7967                attempt,
7968                status: "failed".to_owned(),
7969                started_at_ms,
7970                completed_at_ms,
7971                duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
7972                counters: None,
7973                error_code: Some(Self::diagnostic_code(error_code)),
7974                cause_kind,
7975                http_status,
7976            },
7977            SyncOutcome::RealtimeUnavailable { .. } => DiagnosticLastRound {
7978                epoch: self.epoch.clone(),
7979                attempt,
7980                status: "failed".to_owned(),
7981                started_at_ms,
7982                completed_at_ms,
7983                duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
7984                counters: None,
7985                error_code: Some(REALTIME_UNAVAILABLE_CODE.to_owned()),
7986                cause_kind: None,
7987                http_status: None,
7988            },
7989        });
7990        outcome
7991    }
7992
7993    /// Capture one request on the client owner. Mutations made after this
7994    /// call remain in the outbox for a later round; no network I/O occurs here.
7995    pub fn prepare_sync_round(
7996        &mut self,
7997        url_capable: bool,
7998    ) -> Result<crate::PreparedSyncRound, Box<SyncOutcome>> {
7999        if !self.transport_enabled {
8000            return Err(Box::new(SyncOutcome::Failed {
8001                error_code: "sync.offline".into(),
8002                message: "transport gate is closed".into(),
8003                details: None,
8004                recommended_action: None,
8005            }));
8006        }
8007        if self.active_round.is_some() {
8008            return Err(Box::new(SyncOutcome::Failed {
8009                error_code: "client.round_in_progress".into(),
8010                message: "a sync round is already in flight".into(),
8011                details: None,
8012                recommended_action: None,
8013            }));
8014        }
8015        self.storage_failure.borrow_mut().take();
8016        let attempt = self.progress.start(&self.epoch);
8017        self.round_retry_delay_ms = None;
8018        let started_at_ms = self.clock_now_ms();
8019        let prepared = (|| {
8020            if self.stopped {
8021                return Err(Box::new(SyncOutcome::Ok(SyncReport {
8022                    schema_floor: self.schema_floor.clone(),
8023                    ..SyncReport::default()
8024                })));
8025            }
8026            if self.security_preflight {
8027                return Err(Box::new(SyncOutcome::Failed {
8028                    error_code: SECURITY_PREFLIGHT_REQUIRED_CODE.into(),
8029                    message: "security preflight must complete before syncing".into(),
8030                    details: None,
8031                    recommended_action: None,
8032                }));
8033            }
8034            self.set_sync_needed(false, false);
8035            self.drain_pending_evictions()
8036                .and_then(|_| self.drop_incompatible_outbox())
8037                .and_then(|_| self.drop_unencodable_outbox(true, None))
8038                .map_err(|message| {
8039                    Box::new(SyncOutcome::Failed {
8040                        error_code: "storage.failed".into(),
8041                        message,
8042                        details: None,
8043                        recommended_action: None,
8044                    })
8045                })?;
8046            // §5.11: reveal rows sealed under a key the keyring gained since.
8047            self.reveal_sealed_rows().map_err(|error| {
8048                Box::new(SyncOutcome::Failed {
8049                    error_code: error.code,
8050                    message: error.message,
8051                    details: error.details,
8052                    recommended_action: None,
8053                })
8054            })?;
8055            let has_log_epoch = self.get_meta(LOG_EPOCH_KEY).map_err(|message| {
8056                Box::new(SyncOutcome::Failed {
8057                    error_code: "storage.failed".into(),
8058                    message,
8059                    details: None,
8060                    recommended_action: None,
8061                })
8062            })?;
8063            let uploads = if has_log_epoch.is_some() && self.schema_has_blobs() {
8064                self.pending_blob_uploads().map_err(|e| {
8065                    Box::new(SyncOutcome::Failed {
8066                        error_code: e.code,
8067                        message: e.message,
8068                        details: e.details,
8069                        recommended_action: None,
8070                    })
8071                })?
8072            } else {
8073                Vec::new()
8074            };
8075            if self.realtime_state != RealtimeState::Connected
8076                && self.realtime_policy == RealtimePolicy::Required
8077            {
8078                let retry_delay_ms = self.schedule_background_retry();
8079                self.realtime_retry_delay_ms = Some(retry_delay_ms);
8080                return Err(Box::new(SyncOutcome::RealtimeUnavailable {
8081                    state: self.realtime_state,
8082                    reason_code: self.realtime_reason_code.clone(),
8083                    retry_delay_ms,
8084                }));
8085            }
8086            let (message, mut meta) = self.build_request(url_capable)?;
8087            let units = self.stored(load_units(&self.conn)).map_err(|message| {
8088                Box::new(SyncOutcome::Failed {
8089                    error_code: "storage.failed".into(),
8090                    message,
8091                    details: None,
8092                    recommended_action: None,
8093                })
8094            })?;
8095            for unit in &units {
8096                let applied =
8097                    self.stored(applied_rows(&self.conn, &unit.id))
8098                        .map_err(|message| {
8099                            Box::new(SyncOutcome::Failed {
8100                                error_code: "storage.failed".into(),
8101                                message,
8102                                details: None,
8103                                recommended_action: None,
8104                            })
8105                        })?;
8106                if !applied.is_empty() {
8107                    meta.applied.insert(unit.id.clone(), applied);
8108                }
8109            }
8110            let id = uuid::Uuid::new_v4();
8111            self.active_round = Some((id, started_at_ms, attempt));
8112            let phase = if uploads.is_empty() {
8113                crate::round::RoundPhase::Sync
8114            } else {
8115                crate::round::RoundPhase::UploadGrant
8116            };
8117            Ok(crate::PreparedSyncRound {
8118                id,
8119                started_at_ms,
8120                attempt,
8121                message,
8122                meta,
8123                uploads: uploads.into(),
8124                phase,
8125                encoded: None,
8126                continuation: None,
8127                completed: None,
8128                realtime: self.realtime_state == RealtimeState::Connected,
8129                // §5.5: the owning unit's scope map, each value a one-element list.
8130                scopes: units
8131                    .into_iter()
8132                    .map(|unit| {
8133                        (
8134                            unit.id,
8135                            canonical_scope_json(&unit_scope_lists(&unit.scopes)),
8136                        )
8137                    })
8138                    .collect(),
8139                #[cfg(feature = "bench-internals")]
8140                benchmark_phases: self.benchmark_phases.clone(),
8141                progress: self.progress.clone(),
8142                fixed_now: self
8143                    .now_ms
8144                    .as_ref()
8145                    .map(|clock| clock.load(std::sync::atomic::Ordering::Relaxed)),
8146                round_deadline_at: None,
8147                idle_bound: None,
8148            })
8149        })();
8150        prepared
8151            .map_err(|outcome| Box::new(self.finish_sync_round(started_at_ms, attempt, *outcome)))
8152    }
8153
8154    /// Discard the reply of an old authorization or registration context. Its
8155    /// submitted commits stay durable and can be retried with the same ids.
8156    /// §7.6: an abandoned in-flight round ends its attempt as failed.
8157    pub fn cancel_sync_round(&mut self) {
8158        if let Some((_, started_at_ms, attempt)) = self.active_round.take() {
8159            // A storage failure recorded since the round started is not its cause.
8160            self.storage_failure.borrow_mut().take();
8161            self.finish_sync_round(
8162                started_at_ms,
8163                attempt,
8164                SyncOutcome::Failed {
8165                    error_code: "client.round_cancelled".into(),
8166                    message: "sync round belongs to a released client context".into(),
8167                    details: None,
8168                    recommended_action: None,
8169                },
8170            );
8171        }
8172    }
8173
8174    /// Apply captured network results on the owner, preserving all normal
8175    /// version, schema, content-address and cursor checks. No network I/O.
8176    pub fn apply_sync_round(
8177        &mut self,
8178        completed: crate::CompletedSyncRound,
8179    ) -> crate::AppliedSyncRound {
8180        let mut prepared = completed.prepared;
8181        if self.active_round.map(|(id, ..)| id) != Some(prepared.id) || self.security_preflight {
8182            return crate::AppliedSyncRound::Complete {
8183                outcome: SyncOutcome::Failed {
8184                    error_code: "client.round_cancelled".into(),
8185                    message: "sync round belongs to a released client context".into(),
8186                    details: None,
8187                    recommended_action: None,
8188                },
8189                controls: Vec::new(),
8190                more: false,
8191                bootstrap_advanced: false,
8192            };
8193        }
8194        self.storage_failure.borrow_mut().take();
8195        let (response, mut downloads, transport_failed) = match completed.exchange {
8196            crate::round::ExchangeResult::Upload { id, result } => {
8197                let result = match result {
8198                    // §5.9.7: a non-retryable server refusal (`blob.too_large`)
8199                    // is the body's final outcome. The commits naming it take
8200                    // the incompatible-commit path with the server's code and
8201                    // leave this round's request; the round continues.
8202                    Err(error)
8203                        if error
8204                            .details
8205                            .as_ref()
8206                            .and_then(|details| details.get("retryable"))
8207                            == Some(&Value::Bool(false)) =>
8208                    {
8209                        self.conn
8210                            .prepare(
8211                                "SELECT commit_id FROM _syncular_blob_commit_refs WHERE blob_id = ?",
8212                            )
8213                            .and_then(|mut statement| {
8214                                statement
8215                                    .query_map(rusqlite::params![id], |row| row.get::<_, String>(0))?
8216                                    .collect::<Result<HashSet<_>, _>>()
8217                            })
8218                            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
8219                            .and_then(|refused| {
8220                                let commits = self
8221                                    .outbox
8222                                    .iter()
8223                                    .filter(|commit| refused.contains(&commit.client_commit_id))
8224                                    .cloned()
8225                                    .collect::<Vec<_>>();
8226                                if !commits.is_empty() {
8227                                    self.drop_outbox_commits(&commits, &error.code, &error.message)?;
8228                                }
8229                                prepared.message.frames.retain(|frame| {
8230                                    !matches!(frame, Frame::PushCommit { client_commit_id, .. }
8231                                        if refused.contains(client_commit_id))
8232                                });
8233                                prepared.meta.pushed_ids.retain(|id| !refused.contains(id));
8234                                prepared.continuation = Some(crate::round::RoundPhase::UploadGrant);
8235                                Ok(())
8236                            })
8237                            .map_err(|message| TransportError::new("client.failed", message))
8238                    }
8239                    result => result,
8240                };
8241                let result = result.and_then(|()| {
8242                    self.conn
8243                        .execute(
8244                            "DELETE FROM _syncular_blob_uploads WHERE blob_id = ?",
8245                            rusqlite::params![id],
8246                        )
8247                        .map(|_| ())
8248                        .map_err(|error| {
8249                            TransportError::new(
8250                                "client.failed",
8251                                Self::sqlite_failure(&self.storage_failure, error),
8252                            )
8253                        })
8254                });
8255                match result {
8256                    Ok(()) => return crate::AppliedSyncRound::Continue(prepared),
8257                    Err(TransportError {
8258                        code,
8259                        message,
8260                        details,
8261                    }) => {
8262                        self.active_round = None;
8263                        if Self::unreachable_transport(&details) {
8264                            // The round's work is still owed; the host's
8265                            // connectivity resume runs it.
8266                            self.set_sync_needed(true, false);
8267                        } else if Self::retryable_failure_code(&code) {
8268                            self.schedule_background_retry();
8269                        }
8270                        let recommended_action = recommended_action_of(&details);
8271                        let outcome = self.finish_sync_round(
8272                            prepared.started_at_ms,
8273                            prepared.attempt,
8274                            SyncOutcome::Failed {
8275                                error_code: code,
8276                                message,
8277                                details,
8278                                recommended_action,
8279                            },
8280                        );
8281                        return crate::AppliedSyncRound::Complete {
8282                            outcome,
8283                            controls: Vec::new(),
8284                            more: false,
8285                            bootstrap_advanced: false,
8286                        };
8287                    }
8288                }
8289            }
8290            crate::round::ExchangeResult::Reply {
8291                response,
8292                downloads,
8293                transport_failed,
8294            } => (response, downloads, transport_failed),
8295        };
8296        self.active_round = None;
8297        let before = self.resume_tokens();
8298        let outcome = (|| {
8299            let response = match response {
8300                Ok(response) => {
8301                    // §1.5: a complete, decodable response ends the report, an
8302                    // in-band ERROR included (the server already recorded it).
8303                    self.round_failure = None;
8304                    response
8305                }
8306                Err(TransportError {
8307                    code,
8308                    message,
8309                    details,
8310                }) => {
8311                    // §1.5: the server cannot see a round whose transport
8312                    // failed or whose response never decoded, so its code rides
8313                    // the following requests. A code the frame grammar rejects
8314                    // is not reportable, and the previous code stays; the
8315                    // client-local drive timeout never goes on the wire (§7.8).
8316                    if code != "sync.idle_timeout" && ssp2::model::valid_round_failure_code(&code) {
8317                        self.round_failure = Some(code.clone());
8318                    }
8319                    if transport_failed {
8320                        if prepared.realtime {
8321                            self.set_realtime_state(RealtimeState::Lost, Some(&code));
8322                        }
8323                        if let Err(message) = self.record_lease_error(&code) {
8324                            return SyncOutcome::Failed {
8325                                error_code: "storage.failed".to_owned(),
8326                                message,
8327                                details: None,
8328                                recommended_action: None,
8329                            };
8330                        }
8331                    }
8332                    if Self::retryable_failure_code(&code) && !Self::unreachable_transport(&details)
8333                    {
8334                        let delay = self.schedule_background_retry();
8335                        if prepared.realtime && transport_failed {
8336                            self.realtime_retry_delay_ms = Some(delay);
8337                        }
8338                    }
8339                    let recommended_action = recommended_action_of(&details);
8340                    return SyncOutcome::Failed {
8341                        error_code: code,
8342                        message,
8343                        details,
8344                        recommended_action,
8345                    };
8346                }
8347            };
8348            let mut outcome = self.process_response(&mut downloads, response, &prepared.meta);
8349            if let SyncOutcome::Ok(report) = &mut outcome {
8350                report.deferred_commits = prepared.meta.deferred_commits;
8351            }
8352            match &outcome {
8353                SyncOutcome::Ok(_) => self.reset_background_retry(),
8354                SyncOutcome::Failed {
8355                    error_code,
8356                    details,
8357                    ..
8358                } if Self::retryable_failure_code(error_code)
8359                    && !Self::unreachable_transport(details) =>
8360                {
8361                    self.schedule_background_retry();
8362                }
8363                _ => {}
8364            }
8365            if prepared.meta.deferred_commits > 0 {
8366                self.set_sync_needed(true, true);
8367            }
8368            self.previous_version_lifetime_check();
8369            outcome
8370        })();
8371        match &outcome {
8372            // §7.1: a unit still catching up needs another round, also when the
8373            // drive that started it ends early.
8374            SyncOutcome::Ok(report) if !report.bootstrapping.is_empty() => {
8375                self.set_sync_needed(true, false)
8376            }
8377            // The round's work is still owed; the host's connectivity resume
8378            // runs it.
8379            SyncOutcome::Failed { details, .. } if Self::unreachable_transport(details) => {
8380                self.set_sync_needed(true, false)
8381            }
8382            _ => {}
8383        }
8384        let more = self.transport_enabled
8385            && matches!(&outcome, SyncOutcome::Ok(report)
8386                if report.schema_floor.is_none()
8387                    && (!report.bootstrapping.is_empty()
8388                        || report.commits_applied > 0
8389                        || report.segment_rows_applied > 0
8390                        || !report.resets.is_empty()
8391                        || report.deferred_commits > 0
8392                        || self.sync_needed));
8393        let bootstrap_advanced = matches!(&outcome, SyncOutcome::Ok(report) if report.segment_rows_applied > 0 && !report.bootstrapping.is_empty())
8394            && before != self.resume_tokens();
8395        crate::AppliedSyncRound::Complete {
8396            outcome: self.finish_sync_round(prepared.started_at_ms, prepared.attempt, outcome),
8397            controls: if self.transport_enabled {
8398                downloads.controls
8399            } else {
8400                Vec::new()
8401            },
8402            more,
8403            bootstrap_advanced,
8404        }
8405    }
8406
8407    /// Every unit's resume token and applied-segment record: a bootstrap
8408    /// that advanced changes one (§4.7), also when it applied images of a
8409    /// section it deferred (§5.3, §5.6).
8410    #[allow(clippy::type_complexity)]
8411    fn resume_tokens(&self) -> Option<Vec<(String, Option<String>, Option<String>)>> {
8412        self.conn
8413            .prepare_cached(
8414                "SELECT id, bootstrap_state, applied FROM _syncular_units ORDER BY rowid",
8415            )
8416            .and_then(|mut stmt| {
8417                stmt.query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))?
8418                    .collect()
8419            })
8420            .ok()
8421    }
8422
8423    /// §7.8: one idle drive's default total deadline, the TS client's value.
8424    pub const SYNC_IDLE_TIMEOUT_MS: u64 = 60_000;
8425
8426    pub fn sync_until_idle(
8427        &mut self,
8428        transport: &mut dyn Transport,
8429        max_rounds: Option<u32>,
8430    ) -> SyncOutcome {
8431        self.sync_until_idle_with_deadline(transport, max_rounds, Some(Self::SYNC_IDLE_TIMEOUT_MS))
8432    }
8433
8434    /// As [`SyncClient::sync_until_idle`], with an explicit total deadline in
8435    /// milliseconds (`None` disables it). The deadline is checked against the
8436    /// client clock between rounds: a drive past it after a completed round
8437    /// ends as `BudgetExhausted`. Each round is bounded by `timeout_ms` on its
8438    /// own, or the transport's tighter whole-round deadline (§1.8); a round
8439    /// that reaches the drive bound fails with `sync.idle_timeout`.
8440    pub fn sync_until_idle_with_deadline(
8441        &mut self,
8442        transport: &mut dyn Transport,
8443        max_rounds: Option<u32>,
8444        timeout_ms: Option<u64>,
8445    ) -> SyncOutcome {
8446        // §7.8: an explicit budget is a strict positive round count. Zero is a
8447        // caller error, not an exhaustion.
8448        if max_rounds == Some(0) {
8449            return SyncOutcome::Failed {
8450                details: None,
8451                error_code: "sync.invalid_request".into(),
8452                message: "maxRounds must be a positive integer".into(),
8453                recommended_action: None,
8454            };
8455        }
8456        if timeout_ms.is_some_and(|ms| ms == 0 || i64::try_from(ms).is_err()) {
8457            return SyncOutcome::Failed {
8458                details: None,
8459                error_code: "sync.invalid_request".into(),
8460                message: "timeoutMs must be a positive integer".into(),
8461                recommended_action: None,
8462            };
8463        }
8464        let started_at_ms = self.clock_now_ms();
8465        let rounds = max_rounds.unwrap_or(20);
8466        let mut aggregate = SyncReport::default();
8467        let mut spent = 0;
8468        let mut completed = false;
8469        while spent < rounds {
8470            // §7.8: a deadline that elapsed across completed rounds ends the
8471            // drive like an exhausted round budget; the next drive resumes.
8472            if completed
8473                && timeout_ms.is_some_and(|ms| {
8474                    self.clock_now_ms() >= started_at_ms.saturating_add(ms as i64)
8475                })
8476            {
8477                return SyncOutcome::BudgetExhausted(aggregate);
8478            }
8479            let idle_bound = timeout_ms.map(|ms| (ms, spent));
8480            spent += 1;
8481            let before = self.resume_tokens();
8482            match self.sync_bounded(transport, idle_bound) {
8483                SyncOutcome::Failed {
8484                    error_code,
8485                    message,
8486                    details,
8487                    recommended_action,
8488                } => {
8489                    return SyncOutcome::Failed {
8490                        details,
8491                        error_code,
8492                        message,
8493                        recommended_action,
8494                    };
8495                }
8496                outcome @ SyncOutcome::RealtimeUnavailable { .. } => return outcome,
8497                SyncOutcome::BudgetExhausted(_) => {
8498                    unreachable!("sync() never exhausts a round budget")
8499                }
8500                SyncOutcome::Ok(report) => {
8501                    completed = true;
8502                    if max_rounds.is_none()
8503                        && report.segment_rows_applied > 0
8504                        && !report.bootstrapping.is_empty()
8505                        && before != self.resume_tokens()
8506                    {
8507                        spent = 0;
8508                    }
8509                    // TS parity: a disabled transport or a schema floor ends
8510                    // the loop. The latest round still merges first.
8511                    let stop = !self.transport_enabled() || report.schema_floor.is_some();
8512                    aggregate.merge(&report);
8513                    if stop {
8514                        return SyncOutcome::Ok(aggregate);
8515                    }
8516                    // §4.5: pull again whenever the response contained
8517                    // commits or segments; resets re-bootstrap; a pending
8518                    // resume token continues paging (§4.7); deferred outbox
8519                    // commits and a raised sync-needed signal push on the next
8520                    // round.
8521                    let more = !report.bootstrapping.is_empty()
8522                        || report.commits_applied > 0
8523                        || report.segment_rows_applied > 0
8524                        || !report.resets.is_empty()
8525                        || report.deferred_commits > 0
8526                        || self.sync_needed;
8527                    if !more {
8528                        return SyncOutcome::Ok(aggregate);
8529                    }
8530                }
8531            }
8532        }
8533        // §7.8: partial success. The aggregate report is retained and the run
8534        // is explicitly not idle.
8535        SyncOutcome::BudgetExhausted(aggregate)
8536    }
8537
8538    fn process_response(
8539        &mut self,
8540        transport: &mut dyn Transport,
8541        response: Message,
8542        meta: &RequestMeta,
8543    ) -> SyncOutcome {
8544        #[cfg(feature = "bench-internals")]
8545        let _phase = self.benchmark_phases.start(Phase::ResponseApply);
8546        let mut report = SyncReport {
8547            pushed: meta.pushed_ids.len() as u32,
8548            ..SyncReport::default()
8549        };
8550        let mut rejection_details_by_commit: HashMap<String, BTreeMap<i32, RejectionDetails>> =
8551            HashMap::new();
8552        let pushed_ids = meta
8553            .pushed_ids
8554            .iter()
8555            .map(String::as_str)
8556            .collect::<HashSet<_>>();
8557        let mut last_final_push_result_id: Option<String> = None;
8558        for frame in &response.frames {
8559            match frame {
8560                Frame::PushResultDetails {
8561                    client_commit_id,
8562                    entries,
8563                } => {
8564                    let details = rejection_details_by_commit
8565                        .entry(client_commit_id.clone())
8566                        .or_default();
8567                    for entry in entries {
8568                        let parsed = match RejectionDetails::parse(&entry.details.0) {
8569                            Ok(value) => value,
8570                            Err(message) => {
8571                                return SyncOutcome::Failed {
8572                                    details: None,
8573                                    error_code: "sync.invalid_request".to_owned(),
8574                                    message,
8575                                    recommended_action: None,
8576                                };
8577                            }
8578                        };
8579                        details.insert(entry.op_index, parsed);
8580                    }
8581                }
8582                Frame::PushResult {
8583                    client_commit_id,
8584                    status,
8585                    results,
8586                    ..
8587                } if pushed_ids.contains(client_commit_id.as_str())
8588                    && Self::push_result_is_final(*status, results) =>
8589                {
8590                    last_final_push_result_id = Some(client_commit_id.clone());
8591                }
8592                _ => {}
8593            }
8594        }
8595        let mut frames = response.frames.into_iter().peekable();
8596        match frames.next() {
8597            Some(Frame::RespHeader {
8598                required_schema_version,
8599                latest_schema_version,
8600                log_epoch,
8601                reset_required,
8602            }) => {
8603                if let Some(required) = required_schema_version {
8604                    // §1.6 schema-floor response: nothing else is processed;
8605                    // stop syncing and surface the upgrade requirement.
8606                    let floor = SchemaFloor {
8607                        required_schema_version: Some(required),
8608                        latest_schema_version,
8609                    };
8610                    if let Err(message) = self.set_schema_floor(Some(floor.clone())) {
8611                        return SyncOutcome::Failed {
8612                            details: None,
8613                            error_code: "storage.failed".to_owned(),
8614                            message,
8615                            recommended_action: None,
8616                        };
8617                    }
8618                    report.schema_floor = Some(floor);
8619                    return SyncOutcome::Ok(report);
8620                }
8621                if reset_required {
8622                    if frames.next().is_some() {
8623                        return SyncOutcome::Failed {
8624                            details: None,
8625                            error_code: "client.invalid_host_response".to_owned(),
8626                            message: "log-epoch reset response contains body frames".to_owned(),
8627                            recommended_action: None,
8628                        };
8629                    }
8630                    match self.run_log_epoch_reset(&log_epoch) {
8631                        Ok(resets) => {
8632                            report.resets = resets;
8633                            return SyncOutcome::Ok(report);
8634                        }
8635                        Err(message) => {
8636                            return SyncOutcome::Failed {
8637                                details: None,
8638                                error_code: "sync.local_corrupt".to_owned(),
8639                                message,
8640                                recommended_action: None,
8641                            };
8642                        }
8643                    }
8644                }
8645                let stored_epoch = match self.get_meta(LOG_EPOCH_KEY) {
8646                    Ok(stored) => stored,
8647                    Err(message) => {
8648                        return SyncOutcome::Failed {
8649                            details: None,
8650                            error_code: "storage.failed".to_owned(),
8651                            message,
8652                            recommended_action: None,
8653                        };
8654                    }
8655                };
8656                if stored_epoch.as_deref() != Some(log_epoch.as_str()) {
8657                    return SyncOutcome::Failed {
8658                        details: None,
8659                        error_code: "client.invalid_host_response".to_owned(),
8660                        message: "server changed logEpoch without requiring a reset".to_owned(),
8661                        recommended_action: None,
8662                    };
8663                }
8664            }
8665            _ => {
8666                return SyncOutcome::Failed {
8667                    details: None,
8668                    error_code: "sync.invalid_request".to_owned(),
8669                    message: "response does not start with RESP_HEADER".to_owned(),
8670                    recommended_action: None,
8671                };
8672            }
8673        }
8674
8675        if let Err(message) = self.begin_coalesced_apply() {
8676            return SyncOutcome::Failed {
8677                details: None,
8678                error_code: "storage.failed".into(),
8679                message,
8680                recommended_action: None,
8681            };
8682        }
8683        let mut failure: Option<(String, String, Option<String>, Option<Value>)> = None;
8684        // §4.5 client application: the units whose persisted echo a live
8685        // change must match, fixed at LIVE_START after every SUB_END persisted.
8686        let mut live: Option<(LiveStatus, String, Vec<UnitRecord>)> = None;
8687        let mut live_ended = false;
8688        while let Some(frame) = frames.next() {
8689            match frame {
8690                frame @ Frame::PushResult { .. } => {
8691                    let successful = matches!(
8692                        &frame,
8693                        Frame::PushResult {
8694                            status: PushStatus::Applied | PushStatus::Cached,
8695                            ..
8696                        }
8697                    );
8698                    let mut results = vec![frame];
8699                    if successful {
8700                        while matches!(
8701                            frames.peek(),
8702                            Some(Frame::PushResult {
8703                                status: PushStatus::Applied | PushStatus::Cached,
8704                                ..
8705                            })
8706                        ) {
8707                            if let Some(next) = frames.next() {
8708                                results.push(next);
8709                            }
8710                        }
8711                    }
8712                    if self
8713                        .handle_push_results(
8714                            &results,
8715                            &pushed_ids,
8716                            &rejection_details_by_commit,
8717                            &mut report,
8718                            last_final_push_result_id.as_deref(),
8719                        )
8720                        .is_err()
8721                    {
8722                        failure = Some((
8723                            "client.outcome_persistence_failed".to_owned(),
8724                            "local commit outcome could not be persisted".to_owned(),
8725                            None,
8726                            None,
8727                        ));
8728                        break;
8729                    }
8730                }
8731                Frame::PushResultDetails { .. } => {}
8732                Frame::SubStart {
8733                    id,
8734                    status,
8735                    reason_code,
8736                    bootstrap,
8737                    fenced,
8738                } => {
8739                    // §1.4 rule 5: an ERROR closes the section; its body still
8740                    // applied, its SUB_END values are never persisted.
8741                    let mut body = Vec::new();
8742                    let mut sub_end: Option<(i64, Option<String>)> = None;
8743                    let mut error: Option<Frame> = None;
8744                    for inner in frames.by_ref() {
8745                        match inner {
8746                            Frame::SubEnd {
8747                                next_cursor,
8748                                bootstrap_state,
8749                            } => {
8750                                sub_end = Some((next_cursor, bootstrap_state.map(|r| r.0)));
8751                                break;
8752                            }
8753                            inner @ Frame::Error { .. } => {
8754                                error = Some(inner);
8755                                break;
8756                            }
8757                            Frame::Unknown { .. } => {}
8758                            other => body.push(other),
8759                        }
8760                    }
8761                    let ended = sub_end.is_some();
8762                    let section = SectionStart {
8763                        id,
8764                        status,
8765                        reason_code,
8766                        bootstrap,
8767                        fenced,
8768                    };
8769                    match self.process_section(
8770                        transport,
8771                        &section,
8772                        body,
8773                        sub_end,
8774                        meta,
8775                        &mut report,
8776                    ) {
8777                        Err(SectionError::Abort(code, message)) => {
8778                            failure = Some((code, message, None, None));
8779                            break;
8780                        }
8781                        Err(SectionError::Transport(error)) => {
8782                            failure = Some((error.code, error.message, None, error.details));
8783                            break;
8784                        }
8785                        _ => {}
8786                    }
8787                    if ended {
8788                        // The counters stay: `complete` and `failed` report them.
8789                        self.progress.update(|p| {
8790                            p.unit_id = None;
8791                            p.table = None;
8792                            p.segment_id = None;
8793                        });
8794                    }
8795                    match error {
8796                        Some(Frame::Error {
8797                            code,
8798                            message,
8799                            recommended_action,
8800                            details,
8801                            ..
8802                        }) => {
8803                            failure = Some(Self::error_frame_failure(
8804                                code,
8805                                message,
8806                                recommended_action,
8807                                details,
8808                            ));
8809                            break;
8810                        }
8811                        Some(_) => unreachable!("only an ERROR closes a section early"),
8812                        None if !ended => {
8813                            failure = Some((
8814                                "sync.invalid_request".to_owned(),
8815                                "unit section without SUB_END".to_owned(),
8816                                None,
8817                                None,
8818                            ));
8819                            break;
8820                        }
8821                        None => {}
8822                    }
8823                }
8824                Frame::LiveStart {
8825                    status,
8826                    reason_code,
8827                } => {
8828                    let receivers = match self.stored(load_units(&self.conn)) {
8829                        Ok(units) => units
8830                            .into_iter()
8831                            .filter(|unit| {
8832                                unit.status == UnitStatus::Active
8833                                    && unit.echo.is_some()
8834                                    && !unit.bootstrapping()
8835                            })
8836                            .collect(),
8837                        Err(message) => {
8838                            failure = Some(("storage.failed".to_owned(), message, None, None));
8839                            break;
8840                        }
8841                    };
8842                    live = Some((status, reason_code, receivers));
8843                }
8844                Frame::Commit {
8845                    commit_seq,
8846                    tables,
8847                    changes,
8848                    ..
8849                } => {
8850                    let Some((LiveStatus::Active, _, receivers)) = &live else {
8851                        continue;
8852                    };
8853                    let receivers = receivers.clone();
8854                    match self.apply_live_commit(
8855                        &receivers,
8856                        &tables,
8857                        changes,
8858                        commit_seq,
8859                        AppliedServerCommitSource::Pull,
8860                    ) {
8861                        Ok(applied) => report.commits_applied += u32::from(applied),
8862                        Err(error) => {
8863                            failure = Some((error.code, error.message, None, error.details));
8864                            break;
8865                        }
8866                    }
8867                }
8868                Frame::LiveEnd {
8869                    through_seq,
8870                    echo_digest,
8871                } => {
8872                    let Some((status, reason_code, _)) = live.take() else {
8873                        continue;
8874                    };
8875                    live_ended = true;
8876                    live = Some((status, String::new(), Vec::new()));
8877                    if let Err(message) = self.finish_live_section(
8878                        status,
8879                        &reason_code,
8880                        through_seq,
8881                        echo_digest,
8882                        &meta.registration_digest,
8883                        &mut report,
8884                    ) {
8885                        failure = Some(("storage.failed".to_owned(), message, None, None));
8886                        break;
8887                    }
8888                }
8889                Frame::Lease {
8890                    lease_id,
8891                    expires_at_ms,
8892                } => {
8893                    // §7.3.5: persist the opaque lease; a fresh lease clears
8894                    // any prior lease error (the outage/revocation is over).
8895                    if let Err(message) = self.set_lease_state(Some(LeaseState {
8896                        lease_id: Some(lease_id),
8897                        expires_at_ms: Some(expires_at_ms),
8898                        error_code: None,
8899                    })) {
8900                        failure = Some(("storage.failed".to_owned(), message, None, None));
8901                        break;
8902                    }
8903                }
8904                Frame::Error {
8905                    code,
8906                    message,
8907                    recommended_action,
8908                    details,
8909                    ..
8910                } => {
8911                    // §1.6: the whole request failed; already-completed
8912                    // units keep their applied data and cursors.
8913                    failure = Some(Self::error_frame_failure(
8914                        code,
8915                        message,
8916                        recommended_action,
8917                        details,
8918                    ));
8919                    break;
8920                }
8921                Frame::Unknown { .. } => {}
8922                _ => {
8923                    failure = Some((
8924                        "sync.invalid_request".to_owned(),
8925                        "unexpected frame in response".to_owned(),
8926                        None,
8927                        None,
8928                    ));
8929                    break;
8930                }
8931            }
8932        }
8933
8934        let applied = match failure {
8935            Some(failure) => Err(failure),
8936            // Local apply boundaries already reconciled their affected overlay.
8937            // Refuse round success if any remaining dirty work fails to
8938            // reconcile. §4.8 E1: the push half may have drained commits that
8939            // pinned rows of a shrunk window unit, so retry deferred evictions.
8940            None => self
8941                .rebuild_overlay_if_dirty()
8942                .and_then(|()| self.drain_pending_evictions())
8943                .map_err(|message| ("storage.failed".to_owned(), message, None, None)),
8944        };
8945        // §1.4 rule 4: frames completed before a failure stay committed; a
8946        // failed COMMIT of the coalesced transaction fails the round.
8947        if let Err(message) = self.finish_coalesced_apply() {
8948            return SyncOutcome::Failed {
8949                details: None,
8950                error_code: "storage.failed".into(),
8951                message,
8952                recommended_action: None,
8953            };
8954        }
8955        if let Err((error_code, message, recommended_action, details)) = applied {
8956            return SyncOutcome::Failed {
8957                details,
8958                error_code,
8959                message,
8960                recommended_action,
8961            };
8962        }
8963        let catching_up = match self.stored(load_units(&self.conn)) {
8964            Ok(units) => units
8965                .into_iter()
8966                .filter(|unit| unit.status == UnitStatus::Active && unit.cursor.is_some())
8967                .map(|unit| unit.id)
8968                .collect::<Vec<_>>(),
8969            Err(message) => {
8970                return SyncOutcome::Failed {
8971                    details: None,
8972                    error_code: "storage.failed".into(),
8973                    message,
8974                    recommended_action: None,
8975                }
8976            }
8977        };
8978        // §7.1: a response without LIVE_END, one answered `unregistered` or
8979        // `reset`, or a unit still catching up leaves sync-needed raised;
8980        // after `unregistered` the next round starts at once.
8981        let status = live.map(|(status, _, _)| status);
8982        if !live_ended || status != Some(LiveStatus::Active) || !catching_up.is_empty() {
8983            self.set_sync_needed(true, status == Some(LiveStatus::Unregistered));
8984        }
8985        self.ack_after_pull(transport);
8986        // §7.4.5: the reset is over once the first post-reset pull round
8987        // leaves no unit catching up: the tables are rebuilt and current.
8988        if self.upgrading && catching_up.is_empty() {
8989            self.set_upgrading(false);
8990        }
8991        report.bootstrapping = catching_up;
8992        SyncOutcome::Ok(report)
8993    }
8994
8995    /// §1.6 / §10.1: the failure an in-band `ERROR` frame reports.
8996    fn error_frame_failure(
8997        code: String,
8998        message: String,
8999        recommended_action: String,
9000        details: Option<RawJson>,
9001    ) -> (String, String, Option<String>, Option<Value>) {
9002        let recommended_action = (!recommended_action.is_empty()).then_some(recommended_action);
9003        let details = details
9004            .and_then(|details| serde_json::from_str::<Value>(&details.0).ok())
9005            .filter(Value::is_object);
9006        (code, message, recommended_action, details)
9007    }
9008
9009    /// §4.5 at `LIVE_END` of a pull: join rule J for an `active` section, the
9010    /// §4.6 replica reset for `reset`, nothing for `unregistered`; then the
9011    /// replica cursor, the echo digest and the digest of the list the server
9012    /// now stores.
9013    fn finish_live_section(
9014        &mut self,
9015        status: LiveStatus,
9016        reason_code: &str,
9017        through_seq: i64,
9018        echo_digest: Option<[u8; 32]>,
9019        registration_digest: &str,
9020        report: &mut SyncReport,
9021    ) -> Result<(), String> {
9022        let rollback = self.capture();
9023        let resets = report.resets.len();
9024        self.begin_observation("syncular_live_end")?;
9025        let finished = (|| {
9026            let mut batch = ChangeAccumulator::default();
9027            let before = self.stored(replica_cursor(&self.conn))?;
9028            let mut cursor = if status == LiveStatus::Unregistered {
9029                before
9030            } else {
9031                -1
9032            };
9033            let joined = before.max(through_seq);
9034            for unit in self.stored(load_units(&self.conn))? {
9035                if unit.status != UnitStatus::Active || unit.bootstrapping() {
9036                    continue;
9037                }
9038                match status {
9039                    // §4.6 replica reset: every live unit bootstraps again;
9040                    // post-bootstrap units keep their cursors.
9041                    LiveStatus::Reset if unit.cursor.is_none() => {
9042                        for (base_key, value) in self.stored(window_refs(&self.conn, &unit.id))? {
9043                            batch.window(&base_key, &value);
9044                        }
9045                        report.resets.push(unit.id.clone());
9046                        self.stored(save_unit(
9047                            &self.conn,
9048                            &UnitRecord {
9049                                reason_code: Some(reason_code.to_owned()),
9050                                cursor: Some(-1),
9051                                seeded: Vec::new(),
9052                                ..unit
9053                            },
9054                        ))?;
9055                    }
9056                    // §4.5 join rule J.
9057                    LiveStatus::Active => {
9058                        let next = unit.cursor.unwrap_or(joined).max(through_seq);
9059                        if next >= joined {
9060                            cursor = joined;
9061                        }
9062                        if unit.cursor.is_some() {
9063                            self.stored(save_unit(
9064                                &self.conn,
9065                                &UnitRecord {
9066                                    cursor: (next < joined).then_some(next),
9067                                    ..unit
9068                                },
9069                            ))?;
9070                        }
9071                    }
9072                    _ => {}
9073                }
9074            }
9075            self.set_meta(REPLICA_CURSOR_KEY, &cursor.to_string())?;
9076            self.set_meta(
9077                ECHO_DIGEST_KEY,
9078                &echo_digest
9079                    .map(|digest| bytes_to_hex(&digest))
9080                    .unwrap_or_default(),
9081            )?;
9082            if status == LiveStatus::Unregistered {
9083                self.delete_meta(REGISTERED_DIGEST_KEY)?;
9084            } else {
9085                self.set_meta(REGISTERED_DIGEST_KEY, registration_digest)?;
9086            }
9087            self.finish_observation("syncular_live_end", batch)
9088        })();
9089        if finished.is_err() {
9090            self.rollback_observation("syncular_live_end");
9091            self.restore(rollback);
9092            report.resets.truncate(resets);
9093        }
9094        finished
9095    }
9096
9097    // -- push results (§6.3, §7.2) ------------------------------------------------
9098
9099    fn handle_push_results(
9100        &mut self,
9101        frames: &[Frame],
9102        pushed_ids: &HashSet<&str>,
9103        rejection_details_by_commit: &HashMap<String, BTreeMap<i32, RejectionDetails>>,
9104        report: &mut SyncReport,
9105        last_final_push_result_id: Option<&str>,
9106    ) -> Result<(), String> {
9107        self.begin_observation("syncular_push_results")?;
9108        let rollback = self.capture();
9109        let conflict_count = self.conflicts.len();
9110        let applied_count = report.applied.len();
9111        let rejected_count = report.rejected.len();
9112        let retryable_count = report.retryable.len();
9113        let mut drained = Vec::new();
9114        let mut batch = ChangeAccumulator::default();
9115        let persisted = (|| {
9116            for frame in frames {
9117                let Frame::PushResult {
9118                    client_commit_id,
9119                    status,
9120                    results,
9121                    commit_seq,
9122                } = frame
9123                else {
9124                    return Err("unexpected acknowledgement frame".to_owned());
9125                };
9126                if !pushed_ids.contains(client_commit_id.as_str()) {
9127                    continue;
9128                }
9129                let Some(index) = self
9130                    .outbox
9131                    .iter()
9132                    .position(|commit| commit.client_commit_id == *client_commit_id)
9133                else {
9134                    continue;
9135                };
9136                let status = *status;
9137                let rejection_details = rejection_details_by_commit.get(client_commit_id);
9138                let operations = self.outbox[index].ops.clone();
9139                match status {
9140                    PushStatus::Applied | PushStatus::Cached => {
9141                        // §7.2: a lost ack replays as `cached` — proceed as if the
9142                        // ack had arrived.
9143                        let sequence =
9144                            commit_seq.ok_or_else(|| "ACK lacks a commit sequence".to_owned())?;
9145                        let affected: HashSet<_> = operations
9146                            .iter()
9147                            .map(|op| (op.table.as_str(), op.row_id.as_str()))
9148                            .collect();
9149                        let prior = self.acknowledged_operations(Some(&affected), None)?;
9150                        for (op_index, operation) in operations.iter().enumerate() {
9151                            let delivered: bool = self.conn.query_row("SELECT EXISTS(SELECT 1 FROM _syncular_row_deliveries WHERE tbl=?1 AND id=?2 AND commit_seq>=?3)", rusqlite::params![operation.table, operation.row_id, sequence], |row| row.get(0))
9152                                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9153                            if delivered {
9154                                continue;
9155                            }
9156                            let table = self
9157                                .schema
9158                                .table(&operation.table)
9159                                .ok_or_else(|| "sync.unknown_table".to_owned())?;
9160                            let mut intended: Map<String, Value> = self
9161                                .stored_row(table, &operation.row_id, true)?
9162                                .map(|(row, _)| {
9163                                    table
9164                                        .columns
9165                                        .iter()
9166                                        .zip(&row)
9167                                        .map(|(column, value)| {
9168                                            (column.name.clone(), column_value_to_json(value))
9169                                        })
9170                                        .collect()
9171                                })
9172                                .unwrap_or_default();
9173                            let mut scope_image = intended.clone();
9174                            for op in prior
9175                                .iter()
9176                                .chain(
9177                                    self.failed_commits
9178                                        .iter()
9179                                        .flat_map(|(commit, _)| &commit.ops),
9180                                )
9181                                .chain(
9182                                    self.outbox
9183                                        .iter()
9184                                        .take(index)
9185                                        .flat_map(|commit| &commit.ops),
9186                                )
9187                                .chain(operations.iter().take(op_index + 1))
9188                            {
9189                                if op.table != operation.table || op.row_id != operation.row_id {
9190                                    continue;
9191                                }
9192                                if let Some(values) = &op.values {
9193                                    intended.extend(values.clone());
9194                                } else {
9195                                    scope_image = intended.clone();
9196                                    intended.clear();
9197                                }
9198                            }
9199                            self.conn.execute("INSERT INTO _syncular_acked_rows(commit_id,idx,tbl,id,commit_seq,op_json,intent_json) VALUES (?1,?2,?3,?4,?5,?6,?7)",
9200                                rusqlite::params![client_commit_id, op_index as i64, operation.table, operation.row_id, sequence,
9201                                    serde_json::to_string(&CommitOperation::from(operation)).map_err(|_| "sync.local_corrupt".to_owned())?,
9202                                    serde_json::to_string(if operation.upsert { &intended } else { &scope_image }).map_err(|_| "sync.local_corrupt".to_owned())?])
9203                                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9204                        }
9205                        let journal_results = results
9206                            .iter()
9207                            .map(|result| {
9208                                let op_index = match result {
9209                                    OpResult::Applied { op_index }
9210                                    | OpResult::Conflict { op_index, .. }
9211                                    | OpResult::Error { op_index, .. } => *op_index,
9212                                };
9213                                CommitOperationOutcome::Applied { op_index }
9214                            })
9215                            .collect::<Vec<_>>();
9216                        let outcome_status = if status == PushStatus::Applied {
9217                            CommitOutcomeStatus::Applied
9218                        } else {
9219                            CommitOutcomeStatus::Cached
9220                        };
9221                        let persisted = self
9222                            .persist_commit_outcome(
9223                                client_commit_id,
9224                                outcome_status,
9225                                &journal_results,
9226                                None,
9227                            )
9228                            .and_then(|()| self.delete_outbox_persisted(client_commit_id));
9229                        persisted?;
9230                        self.sync_needed = true;
9231                        batch.status = true;
9232                        batch.outcomes = true;
9233                    }
9234                    PushStatus::Rejected => {
9235                        if results.iter().any(|result| {
9236                            matches!(
9237                                result,
9238                                OpResult::Error {
9239                                    code,
9240                                    retryable: true,
9241                                    ..
9242                                } if code == "sync.idempotency_cache_miss"
9243                            )
9244                        }) {
9245                            // §6.3/§7.2: a serving failure, not an outcome — keep the
9246                            // exact commit queued for an identical retry.
9247                            report.retryable.push(client_commit_id.to_owned());
9248                            continue;
9249                        }
9250
9251                        let mut journal_results = Vec::with_capacity(results.len());
9252                        let mut conflicts = Vec::new();
9253                        let mut rejections = Vec::new();
9254                        for result in results {
9255                            match result {
9256                                OpResult::Applied { op_index } => {
9257                                    journal_results.push(CommitOperationOutcome::Applied {
9258                                        op_index: *op_index,
9259                                    });
9260                                }
9261                                OpResult::Conflict {
9262                                    op_index,
9263                                    code,
9264                                    message,
9265                                    server_version,
9266                                    server_row,
9267                                    conflict_columns: conflict_columns_bitmap,
9268                                } => {
9269                                    let operation = operations
9270                                        .get(*op_index as usize)
9271                                        .map(CommitOperation::from);
9272                                    let (table, row_id) = operation
9273                                        .as_ref()
9274                                        .map(|op| (op.table.clone(), op.row_id.clone()))
9275                                        .unwrap_or_default();
9276                                    let server_row_json = self
9277                                        .schema
9278                                        .table(&table)
9279                                        .and_then(|t| {
9280                                            decode_row_bytes(t, server_row, &self.encryption)
9281                                                .ok()
9282                                                .map(|(row, _)| (t, row))
9283                                        })
9284                                        .map(|(t, row)| {
9285                                            let mut map = Map::new();
9286                                            for (i, column) in t.columns.iter().enumerate() {
9287                                                map.insert(
9288                                                    column.name.clone(),
9289                                                    column_value_to_json(
9290                                                        row.get(i).unwrap_or(&None),
9291                                                    ),
9292                                                );
9293                                            }
9294                                            map
9295                                        })
9296                                        .unwrap_or_default();
9297                                    // §6.3: decode the presence-layout bitmap
9298                                    // into the marked column names.
9299                                    let conflict_columns = self
9300                                        .schema
9301                                        .table(&table)
9302                                        .map(|t| {
9303                                            t.columns
9304                                                .iter()
9305                                                .enumerate()
9306                                                .filter(|(index, _)| {
9307                                                    conflict_columns_bitmap
9308                                                        .get(index / 8)
9309                                                        .is_some_and(|byte| {
9310                                                            byte & (1 << (index % 8)) != 0
9311                                                        })
9312                                                })
9313                                                .map(|(_, column)| column.name.clone())
9314                                                .collect::<Vec<_>>()
9315                                        })
9316                                        .unwrap_or_default();
9317                                    let conflict = ConflictRecord {
9318                                        client_commit_id: client_commit_id.to_owned(),
9319                                        op_index: *op_index,
9320                                        table,
9321                                        row_id,
9322                                        code: code.clone(),
9323                                        message: message.clone(),
9324                                        server_version: *server_version,
9325                                        server_row: server_row_json,
9326                                        conflict_columns,
9327                                        operation,
9328                                    };
9329                                    journal_results.push(CommitOperationOutcome::Conflict {
9330                                        conflict: conflict.clone(),
9331                                    });
9332                                    conflicts.push(conflict);
9333                                }
9334                                OpResult::Error {
9335                                    op_index,
9336                                    code,
9337                                    message,
9338                                    retryable,
9339                                } => {
9340                                    let rejection = RejectionRecord {
9341                                        client_commit_id: client_commit_id.to_owned(),
9342                                        op_index: *op_index,
9343                                        code: code.clone(),
9344                                        message: message.clone(),
9345                                        retryable: *retryable,
9346                                        details: rejection_details
9347                                            .and_then(|details| details.get(op_index))
9348                                            .cloned(),
9349                                        operation: operations
9350                                            .get(*op_index as usize)
9351                                            .map(CommitOperation::from),
9352                                    };
9353                                    journal_results.push(CommitOperationOutcome::Error {
9354                                        rejection: rejection.clone(),
9355                                    });
9356                                    rejections.push(rejection);
9357                                }
9358                            }
9359                        }
9360                        if self.retain_failed_commits {
9361                            self.retain_failed_commit(client_commit_id, &operations)?;
9362                        }
9363                        let outcome_status = if conflicts.is_empty() {
9364                            CommitOutcomeStatus::Rejected
9365                        } else {
9366                            CommitOutcomeStatus::Conflict
9367                        };
9368                        let persisted = self
9369                            .persist_commit_outcome(
9370                                client_commit_id,
9371                                outcome_status,
9372                                &journal_results,
9373                                Some(&operations),
9374                            )
9375                            .and_then(|()| self.delete_outbox_persisted(client_commit_id));
9376                        persisted?;
9377                        batch.conflicts = !conflicts.is_empty();
9378                        batch.rejections = !rejections.is_empty();
9379                        batch.status = true;
9380                        batch.outcomes = true;
9381                        self.conflicts.extend(conflicts);
9382                        self.rejections.extend(rejections);
9383                    }
9384                }
9385                drained.push(self.outbox.remove(index));
9386                if batch.status {
9387                    for operation in &operations {
9388                        if !self.record_row_scopes(
9389                            &mut batch,
9390                            &operation.table,
9391                            &operation.row_id,
9392                            false,
9393                        ) {
9394                            batch.table(&operation.table);
9395                        }
9396                    }
9397                }
9398                match status {
9399                    PushStatus::Applied | PushStatus::Cached => {
9400                        report.applied.push(client_commit_id.to_owned())
9401                    }
9402                    PushStatus::Rejected => report.rejected.push(client_commit_id.to_owned()),
9403                }
9404            }
9405            if batch.outcomes && frames.iter().any(|frame| matches!(frame, Frame::PushResult { client_commit_id, .. } if Some(client_commit_id.as_str()) == last_final_push_result_id)) {
9406                self.prune_commit_outcomes()?;
9407            }
9408            self.reconcile_rows(&drained)?;
9409            self.rebuild_overlay_if_dirty()?;
9410            self.finish_observation("syncular_push_results", batch)
9411        })();
9412        if let Err(error) = persisted {
9413            self.rollback_observation("syncular_push_results");
9414            self.restore(rollback);
9415            report.applied.truncate(applied_count);
9416            report.rejected.truncate(rejected_count);
9417            report.retryable.truncate(retryable_count);
9418            return Err(error);
9419        }
9420        if !rollback.sync_needed && self.sync_needed {
9421            self.sync_intent_queue.push_back(SyncIntent::Interactive);
9422        }
9423        report.conflicts += (self.conflicts.len() - conflict_count) as u32;
9424        Ok(())
9425    }
9426
9427    fn push_result_is_final(status: PushStatus, results: &[OpResult]) -> bool {
9428        status != PushStatus::Rejected
9429            || !results.iter().any(|result| {
9430                matches!(
9431                    result,
9432                    OpResult::Error {
9433                        code,
9434                        retryable: true,
9435                        ..
9436                    } if code == "sync.idempotency_cache_miss"
9437                )
9438            })
9439    }
9440
9441    // -- unit sections and the live section ------------------------------------------
9442
9443    /// One unit section (§4.4): apply its segments, then persist its
9444    /// `SUB_END` (§4.6 reset, §3.3 revocation and purge, the echo, cursor and
9445    /// resume token). A section of a unit the app released while the round was
9446    /// in flight is skipped; a fail-closed first page fails the unit and skips
9447    /// the rest of its section (§5.6).
9448    fn process_section(
9449        &mut self,
9450        transport: &mut dyn Transport,
9451        start: &SectionStart,
9452        body: Vec<Frame>,
9453        sub_end: Option<(i64, Option<String>)>,
9454        meta: &RequestMeta,
9455        report: &mut SyncReport,
9456    ) -> Result<(), SectionError> {
9457        let storage = |message| SectionError::Abort("storage.failed".to_owned(), message);
9458        let unit = self
9459            .stored(get_unit(&self.conn, &start.id))
9460            .map_err(storage)?
9461            .filter(|unit| unit.status == UnitStatus::Active);
9462        // §7.7: a section boundary ends the previous payload.
9463        self.progress.update(|p| {
9464            p.phase = ProgressPhase::Import;
9465            p.unit_id = unit.as_ref().map(|unit| unit.id.clone());
9466            p.table = None;
9467            p.segment_id = None;
9468            p.bytes_received = 0;
9469            p.bytes_total = None;
9470            p.rows_processed = 0;
9471            p.rows_total = None;
9472            p.import_write_ms = None;
9473            p.import_commit_ms = None;
9474        });
9475        let Some(unit) = unit else {
9476            return Ok(());
9477        };
9478        let echo: Vec<EchoRecord> = start
9479            .fenced
9480            .iter()
9481            .map(|record| EchoRecord {
9482                table: record.table.clone(),
9483                status: record.status.name().to_owned(),
9484                scopes: record.scopes.iter().cloned().collect(),
9485            })
9486            .collect();
9487        // §5.6: rows earlier attempts at this position applied.
9488        let applied_record = self
9489            .stored(applied_rows(&self.conn, &unit.id))
9490            .map_err(storage)?;
9491        // §5.6: a resumed bootstrap keeps the tables it already seeded.
9492        let mut seeded: BTreeSet<String> = if unit.bootstrap_state.is_none() {
9493            BTreeSet::new()
9494        } else {
9495            unit.seeded.iter().cloned().collect()
9496        };
9497        let applied = (|| {
9498            for frame in body {
9499                match frame {
9500                    Frame::SegmentInline { payload } => {
9501                        let segment = decode_rows_segment(&payload).map_err(|e| {
9502                            SectionError::Abort(e.code.as_str().to_owned(), e.detail)
9503                        })?;
9504                        self.progress.update(|p| {
9505                            p.phase = ProgressPhase::Import;
9506                            p.unit_id = Some(unit.id.clone());
9507                            p.table = Some(segment.table.clone());
9508                            p.segment_id = None;
9509                            p.bytes_received = payload.len() as u64;
9510                            p.bytes_total = Some(payload.len() as u64);
9511                            p.rows_processed = 0;
9512                            p.rows_total =
9513                                Some(segment.blocks.iter().map(|block| block.len() as u64).sum());
9514                            p.import_write_ms = None;
9515                            p.import_commit_ms = None;
9516                        });
9517                        let (pair, clear, effective) =
9518                            self.segment_pair(&unit, &echo, &seeded, &segment.table)?;
9519                        report.segment_rows_applied +=
9520                            self.apply_segment(&pair, &segment, clear, &effective, &unit.id)?;
9521                        seeded.insert(segment.table);
9522                    }
9523                    Frame::SegmentRef {
9524                        segment_id,
9525                        byte_length,
9526                        media_type,
9527                        table,
9528                        row_count,
9529                        as_of_commit_seq,
9530                        scope_digest,
9531                        row_cursor,
9532                        next_row_cursor,
9533                        url,
9534                        url_expires_at_ms,
9535                        ..
9536                    } => {
9537                        // §4.2: reject a descriptor whose mediaType was not
9538                        // advertised — never skip or guess.
9539                        let advertised = match media_type {
9540                            MediaType::Rows => {
9541                                meta.accept & ACCEPT_EXTERNAL_ROWS != 0
9542                                    || meta.accept & ACCEPT_INLINE_ROWS != 0
9543                            }
9544                            MediaType::Sqlite => meta.accept & ACCEPT_SQLITE != 0,
9545                        };
9546                        if !advertised {
9547                            return Err(SectionError::Abort(
9548                                "sync.invalid_request".to_owned(),
9549                                format!(
9550                                    "SEGMENT_REF mediaType {} was not advertised in accept (§4.2)",
9551                                    media_type.name()
9552                                ),
9553                            ));
9554                        }
9555                        // §5.6 applied-segment rule: a partly applied image
9556                        // already ran its table's first-page clear.
9557                        let resumed = applied_record.get(&segment_id).copied();
9558                        if resumed.is_some() {
9559                            seeded.insert(table.clone());
9560                        }
9561                        if resumed == Some(row_count) {
9562                            continue;
9563                        }
9564                        if meta
9565                            .deferred
9566                            .contains(&(unit.id.clone(), segment_id.clone()))
9567                        {
9568                            return Err(SectionError::Deferred);
9569                        }
9570                        self.progress.update(|p| {
9571                            p.phase = ProgressPhase::Download;
9572                            p.unit_id = Some(unit.id.clone());
9573                            p.table = Some(table.clone());
9574                            p.segment_id = Some(segment_id.clone());
9575                            p.bytes_received = 0;
9576                            p.bytes_total = u64::try_from(byte_length).ok();
9577                            p.rows_processed = 0;
9578                            p.rows_total = u64::try_from(row_count).ok();
9579                            p.import_write_ms = None;
9580                            p.import_commit_ms = None;
9581                        });
9582                        let observer = self.progress.clone();
9583                        let mut on_progress = |bytes| observer.update(|p| p.bytes_received = bytes);
9584                        let bytes = if let Some(url) = url {
9585                            // §5.4: a url-carrying descriptor MUST be fetched
9586                            // from that URL; failure invalidates the whole
9587                            // descriptor (no fall-through to §5.5 — re-pull
9588                            // recovers, §1.4 rule 5).
9589                            if meta.accept & ACCEPT_SIGNED_URLS == 0 {
9590                                return Err(SectionError::Abort(
9591                                    "sync.invalid_request".to_owned(),
9592                                    "SEGMENT_REF carries a url but accept bit 3 was not advertised (§5.4)"
9593                                        .to_owned(),
9594                                ));
9595                            }
9596                            // §5.4: MUST NOT start a fetch at/past expiry.
9597                            if url_expires_at_ms.is_some_and(|exp| {
9598                                exp <= transport
9599                                    .segment_fetch_started_at(&url)
9600                                    .unwrap_or_else(|| self.clock_now_ms())
9601                            }) {
9602                                return Err(SectionError::Abort(
9603                                    "sync.segment_expired".to_owned(),
9604                                    format!(
9605                                        "signed URL for segment {segment_id} expired before fetch — re-pull mints fresh descriptors (§5.4)"
9606                                    ),
9607                                ));
9608                            }
9609                            transport
9610                                .fetch_url(
9611                                    &url,
9612                                    u64::try_from(byte_length).unwrap_or(0),
9613                                    &mut on_progress,
9614                                )
9615                                .map_err(|e| self.segment_fetch_failed(e))?
9616                        } else {
9617                            // §5.5: the owning unit's scope map, each value a one-element list.
9618                            let requested_scopes_json =
9619                                canonical_scope_json(&unit_scope_lists(&unit.scopes));
9620                            transport
9621                                .download_segment(
9622                                    &SegmentRequest {
9623                                        segment_id: segment_id.clone(),
9624                                        table: table.clone(),
9625                                        requested_scopes_json,
9626                                        byte_length: u64::try_from(byte_length).unwrap_or(0),
9627                                    },
9628                                    &mut on_progress,
9629                                )
9630                                .map_err(|e| {
9631                                    // §5.5: access changed since the response;
9632                                    // the next round's SUB_START settles the unit.
9633                                    if e.code == "sync.forbidden" {
9634                                        SectionError::Refused
9635                                    } else {
9636                                        self.segment_fetch_failed(e)
9637                                    }
9638                                })?
9639                        };
9640                        // §5.4: the descriptor's byteLength caps the body before
9641                        // §5.1 verifies the content address; the client discards
9642                        // the segment and re-pulls on either failure.
9643                        if bytes.len() as i64 > byte_length {
9644                            self.schedule_background_retry();
9645                            return Err(SectionError::Abort(
9646                                "sync.invalid_request".to_owned(),
9647                                "segment exceeds its byteLength".to_owned(),
9648                            ));
9649                        }
9650                        on_progress(bytes.len() as u64);
9651                        let digest = Sha256::digest(&bytes);
9652                        let expected = segment_id
9653                            .strip_prefix("sha256:")
9654                            .unwrap_or(segment_id.as_str());
9655                        if bytes_to_hex(&digest) != expected {
9656                            // §5.1: the client discards the segment and re-pulls.
9657                            self.schedule_background_retry();
9658                            return Err(SectionError::Abort(
9659                                "sync.invalid_request".to_owned(),
9660                                "segment bytes do not match the content address (§5.1)".to_owned(),
9661                            ));
9662                        }
9663                        self.progress.update(|p| p.phase = ProgressPhase::Import);
9664                        if media_type == MediaType::Sqlite {
9665                            // §5.3: a range image keeps its position in the resume
9666                            // token; a descriptor row cursor is invalid.
9667                            if row_cursor.is_some() || next_row_cursor.is_some() {
9668                                return Err(SectionError::Abort(
9669                                    "sync.invalid_request".to_owned(),
9670                                    "sqlite segments are whole-table: rowCursor/nextRowCursor must be absent (§5.3)"
9671                                        .to_owned(),
9672                                ));
9673                            }
9674                            let (pair, clear, effective) =
9675                                self.segment_pair(&unit, &echo, &seeded, &table)?;
9676                            report.segment_rows_applied += self.apply_sqlite_segment(
9677                                &pair,
9678                                &bytes,
9679                                clear,
9680                                &effective,
9681                                &unit.id,
9682                                &segment_id,
9683                                resumed.unwrap_or(0),
9684                                row_count,
9685                                as_of_commit_seq,
9686                                &scope_digest,
9687                            )?;
9688                            seeded.insert(table);
9689                        } else {
9690                            let segment = decode_rows_segment(&bytes).map_err(|e| {
9691                                SectionError::Abort(e.code.as_str().to_owned(), e.detail)
9692                            })?;
9693                            let (pair, clear, effective) =
9694                                self.segment_pair(&unit, &echo, &seeded, &segment.table)?;
9695                            report.segment_rows_applied +=
9696                                self.apply_segment(&pair, &segment, clear, &effective, &unit.id)?;
9697                            seeded.insert(segment.table);
9698                            // An image records its rows with each chunk.
9699                            self.stored(record_applied_rows(
9700                                &self.conn,
9701                                &unit.id,
9702                                &segment_id,
9703                                row_count,
9704                            ))
9705                            .map_err(storage)?;
9706                        }
9707                    }
9708                    _ => {
9709                        return Err(SectionError::Abort(
9710                            "sync.invalid_request".to_owned(),
9711                            "unexpected frame inside a unit section".to_owned(),
9712                        ));
9713                    }
9714                }
9715            }
9716            Ok(())
9717        })();
9718        match applied {
9719            Ok(()) => {}
9720            // The unit keeps catching up with its cursor unpersisted.
9721            Err(SectionError::Refused | SectionError::Deferred) => return Ok(()),
9722            Err(SectionError::FailClosed) => {
9723                // §5.6: the failure is unit-local; the rest of the response
9724                // still applies and the failed unit leaves the list.
9725                self.begin_observation("syncular_section_failure")
9726                    .map_err(storage)?;
9727                let failed = (|| {
9728                    let mut batch = ChangeAccumulator::default();
9729                    for (base_key, value) in self.stored(window_refs(&self.conn, &unit.id))? {
9730                        batch.window(&base_key, &value);
9731                    }
9732                    batch.status = true;
9733                    self.stored(save_unit(
9734                        &self.conn,
9735                        &UnitRecord {
9736                            status: UnitStatus::Failed,
9737                            reason_code: Some("sync.scope_revoked".to_owned()),
9738                            ..unit.clone()
9739                        },
9740                    ))?;
9741                    self.finish_observation("syncular_section_failure", batch)
9742                })();
9743                if let Err(message) = failed {
9744                    self.rollback_observation("syncular_section_failure");
9745                    return Err(storage(message));
9746                }
9747                report.failed.push(unit.id);
9748                return Ok(());
9749            }
9750            Err(error) => return Err(error),
9751        }
9752        let Some((next_cursor, bootstrap_state)) = sub_end else {
9753            return Ok(());
9754        };
9755        let rollback = self.capture();
9756        let counts = (
9757            report.resets.len(),
9758            report.revoked.len(),
9759            report.failed.len(),
9760        );
9761        self.begin_observation("syncular_sub_end")
9762            .map_err(storage)?;
9763        let finished = self
9764            .finish_section(start, echo, next_cursor, bootstrap_state, seeded, report)
9765            .and_then(|batch| {
9766                self.rebuild_overlay_if_dirty()?;
9767                self.finish_observation("syncular_sub_end", batch)
9768            });
9769        if let Err(message) = finished {
9770            self.rollback_observation("syncular_sub_end");
9771            self.restore(rollback);
9772            report.resets.truncate(counts.0);
9773            report.revoked.truncate(counts.1);
9774            report.failed.truncate(counts.2);
9775            return Err(storage(message));
9776        }
9777        Ok(())
9778    }
9779
9780    /// §5.6: the table, first-page flag and effective scopes of one segment
9781    /// of `unit`'s section.
9782    #[allow(clippy::type_complexity)]
9783    fn segment_pair(
9784        &self,
9785        unit: &UnitRecord,
9786        echo: &[EchoRecord],
9787        seeded: &BTreeSet<String>,
9788        name: &str,
9789    ) -> Result<(TableSchema, bool, Vec<(String, Vec<String>)>), SectionError> {
9790        let table = self.schema.table(name).cloned().ok_or_else(|| {
9791            SectionError::Abort(
9792                "sync.schema_mismatch".to_owned(),
9793                format!("segment table {name:?} is not in the client schema"),
9794            )
9795        })?;
9796        let effective = covers(&table, &unit.scopes)
9797            .then(|| pair_scopes(&unit.scopes, name, Some(echo)))
9798            .flatten()
9799            .ok_or_else(|| {
9800                SectionError::Abort(
9801                    "sync.invalid_request".to_owned(),
9802                    "a segment names no active pair of its unit".to_owned(),
9803                )
9804            })?;
9805        Ok((table, !seeded.contains(name), effective))
9806    }
9807
9808    /// §4.4 `SUB_END` of an active, revoked or reset section, inside the
9809    /// caller's observation: a reset discards cursor, resume token and
9810    /// first-page record and keeps the echo; a revoked pair purges by its last
9811    /// echo (§3.3); otherwise the echo replaces the persisted one, a revoked
9812    /// fenced record that turns active again re-bootstraps the unit, and a
9813    /// bootstrap section persists its pin, resume token and seeded tables.
9814    fn finish_section(
9815        &mut self,
9816        start: &SectionStart,
9817        echo: Vec<EchoRecord>,
9818        next_cursor: i64,
9819        bootstrap_state: Option<String>,
9820        seeded: BTreeSet<String>,
9821        report: &mut SyncReport,
9822    ) -> Result<ChangeAccumulator, String> {
9823        #[cfg(feature = "bench-internals")]
9824        let _phase = self.benchmark_phases.start(Phase::CursorPersist);
9825        let mut batch = ChangeAccumulator::default();
9826        // §4.4: SUB_END writes only into a unit that is still registered.
9827        let Some(unit) = self
9828            .stored(get_unit(&self.conn, &start.id))?
9829            .filter(|unit| unit.status == UnitStatus::Active)
9830        else {
9831            return Ok(batch);
9832        };
9833        for (base_key, value) in self.stored(window_refs(&self.conn, &unit.id))? {
9834            batch.window(&base_key, &value);
9835        }
9836        if start.status == SubStatus::Reset {
9837            // §4.6: staleness, never a purge; the rows stay until the
9838            // bootstrap replaces them.
9839            report.resets.push(unit.id.clone());
9840            self.stored(save_unit(
9841                &self.conn,
9842                &UnitRecord {
9843                    reason_code: Some(start.reason_code.clone()),
9844                    cursor: Some(-1),
9845                    bootstrap_state: None,
9846                    seeded: Vec::new(),
9847                    ..unit
9848                },
9849            ))?;
9850            return Ok(batch);
9851        }
9852        let revoked = start.status == SubStatus::Revoked;
9853        let mut units = None;
9854        let mut failed = false;
9855        let mut rearmed = false;
9856        for table in self.schema.tables.clone() {
9857            if !covers(&table, &unit.scopes) {
9858                continue;
9859            }
9860            let last = unit
9861                .echo
9862                .as_deref()
9863                .and_then(|last| pair_scopes(&unit.scopes, &table.name, Some(last)));
9864            let next = (!revoked)
9865                .then(|| pair_scopes(&unit.scopes, &table.name, Some(&echo)))
9866                .flatten();
9867            // §3.3: a revoked fenced record that turns active again re-bootstraps.
9868            rearmed |= unit.echo.is_some() && last.is_none() && next.is_some();
9869            // §3.3: purge a pair the response revokes, keyed on the echo the
9870            // client persisted while the pair was active.
9871            if let (Some(last), None) = (&last, &next) {
9872                let units = match units {
9873                    Some(ref units) => units,
9874                    None => units.insert(self.stored(load_units(&self.conn))?),
9875                };
9876                let owners = unit_owners(units, &table, &unit.id);
9877                failed |= !self.purge_pair(&mut batch, &table, last, &owners)?;
9878            }
9879        }
9880        if revoked || failed {
9881            if revoked {
9882                report.revoked.push(unit.id.clone());
9883            }
9884            if failed {
9885                report.failed.push(unit.id.clone());
9886            }
9887            batch.status = true;
9888            self.stored(save_unit(
9889                &self.conn,
9890                &UnitRecord {
9891                    status: if failed {
9892                        UnitStatus::Failed
9893                    } else {
9894                        UnitStatus::Revoked
9895                    },
9896                    reason_code: Some(if failed {
9897                        "sync.scope_revoked".to_owned()
9898                    } else {
9899                        start.reason_code.clone()
9900                    }),
9901                    cursor: Some(-1),
9902                    bootstrap_state: None,
9903                    seeded: Vec::new(),
9904                    echo: None,
9905                    ..unit
9906                },
9907            ))?;
9908            return Ok(batch);
9909        }
9910        let completed = start.bootstrap && !rearmed && bootstrap_state.is_none();
9911        let (cursor, bootstrap_state, seeded) = if rearmed {
9912            (Some(-1), None, Vec::new())
9913        } else if start.bootstrap {
9914            let seeded = if bootstrap_state.is_none() {
9915                Vec::new()
9916            } else {
9917                seeded.into_iter().collect()
9918            };
9919            (Some(next_cursor), bootstrap_state, seeded)
9920        } else {
9921            (
9922                unit.cursor,
9923                unit.bootstrap_state.clone(),
9924                unit.seeded.clone(),
9925            )
9926        };
9927        self.stored(save_unit(
9928            &self.conn,
9929            &UnitRecord {
9930                reason_code: None,
9931                echo: Some(echo.clone()),
9932                cursor,
9933                bootstrap_state,
9934                seeded,
9935                ..unit.clone()
9936            },
9937        ))?;
9938        if !completed {
9939            return Ok(batch);
9940        }
9941        // §7.1: a completed bootstrap covers acknowledged intent of its pairs
9942        // at or below its pin.
9943        for table in self.schema.tables.clone() {
9944            let Some(effective) = covers(&table, &unit.scopes)
9945                .then(|| pair_scopes(&unit.scopes, &table.name, Some(&echo)))
9946                .flatten()
9947            else {
9948                continue;
9949            };
9950            let mut statement = self
9951                .conn
9952                .prepare("SELECT id,intent_json FROM _syncular_acked_rows WHERE tbl=?1 AND commit_seq<=?2")
9953                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9954            let rows = statement
9955                .query_map(rusqlite::params![table.name, next_cursor], |row| {
9956                    Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
9957                })
9958                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9959            let mut covered = HashSet::new();
9960            for row in rows {
9961                let (id, intent) =
9962                    row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9963                let values: Map<String, Value> =
9964                    serde_json::from_str(&intent).map_err(|_| "sync.local_corrupt".to_owned())?;
9965                if owns_local_row(&table, &effective, &[], &values) {
9966                    covered.insert(id);
9967                }
9968            }
9969            drop(statement);
9970            for id in covered {
9971                self.conn
9972                    .execute(
9973                        "DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2 AND commit_seq<=?3",
9974                        rusqlite::params![table.name, id, next_cursor],
9975                    )
9976                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9977                self.overlay_dirty.table(&table.name);
9978                batch.table(&table.name);
9979            }
9980        }
9981        Ok(batch)
9982    }
9983
9984    /// §3.3: purge one revoked pair inside the caller's observation: delete
9985    /// the rows matching its last effective scopes that no owner holds, drop
9986    /// the retained intent and every pending commit that provably writes into
9987    /// them, and delete now-unauthorized blob bodies. `Ok(false)` when the
9988    /// table has no local scope-column mapping (fail closed: nothing deleted).
9989    fn purge_pair(
9990        &mut self,
9991        batch: &mut ChangeAccumulator,
9992        table: &TableSchema,
9993        last: &[(String, Vec<String>)],
9994        owners: &[Vec<(String, Vec<String>)>],
9995    ) -> Result<bool, String> {
9996        if local_scope_predicate(table, last, owners).is_err() {
9997            return Ok(false);
9998        }
9999        for commit in self.drop_failed_commits(|name, values| {
10000            name == table.name && owns_local_row(table, last, owners, values)
10001        })? {
10002            batch.conflicts = true;
10003            batch.rejections = true;
10004            batch.outcomes = true;
10005            for operation in commit.ops {
10006                batch.table(&operation.table);
10007            }
10008        }
10009        if !self.purge_scope_rows(&table.name, last, owners)? {
10010            return Err("storage.failed: the revoked scope purge failed".to_owned());
10011        }
10012        self.record_scope_map(batch, &table.name, last);
10013        if self.drop_doomed_outbox(table, last, owners)? {
10014            batch.status = true;
10015            batch.rejections = true;
10016            batch.outcomes = true;
10017        }
10018        // §5.9.7 B2: revocation deletes now-unauthorized blob bodies.
10019        self.delete_unreferenced_cached_blobs()?;
10020        Ok(true)
10021    }
10022
10023    /// §4.5 client application: apply one live-section or delta `COMMIT`
10024    /// with only the changes that match the persisted echo of a receiving
10025    /// unit; every applied change then goes through the version guard.
10026    /// Returns whether any change of the frame applied.
10027    fn apply_live_commit(
10028        &mut self,
10029        receivers: &[UnitRecord],
10030        tables: &[String],
10031        changes: Vec<ssp2::model::Change>,
10032        commit_seq: i64,
10033        source: AppliedServerCommitSource,
10034    ) -> Result<bool, TransportError> {
10035        let changes: Vec<ssp2::model::Change> = changes
10036            .into_iter()
10037            .filter(|change| {
10038                // An unknown table fails loud in the apply below.
10039                let Some(table) = tables
10040                    .get(change.table_index as usize)
10041                    .and_then(|name| self.schema.table(name))
10042                else {
10043                    return true;
10044                };
10045                receivers.iter().any(|unit| {
10046                    covers(table, &unit.scopes)
10047                        && pair_scopes(&unit.scopes, &table.name, unit.echo.as_deref()).is_some_and(
10048                            |scopes| {
10049                                scopes.iter().all(|(variable, values)| {
10050                                    change
10051                                        .scopes
10052                                        .iter()
10053                                        .find(|(key, _)| key == variable)
10054                                        .is_some_and(|(_, value)| values.contains(value))
10055                                })
10056                            },
10057                        )
10058                })
10059            })
10060            .collect();
10061        if changes.is_empty() {
10062            return Ok(false);
10063        }
10064        let affected_tables = changes
10065            .iter()
10066            .filter_map(|change| tables.get(change.table_index as usize))
10067            .cloned()
10068            .collect::<BTreeSet<_>>();
10069        let previous_batches = self.change_queue.len();
10070        self.apply_commit_frame(tables, &changes, commit_seq)?;
10071        if self.change_queue.len() > previous_batches {
10072            if let Some(batch) = self.change_queue.back_mut() {
10073                batch
10074                    .applied_commits
10075                    .extend(
10076                        affected_tables
10077                            .into_iter()
10078                            .map(|table| AppliedServerCommit {
10079                                table,
10080                                commit_seq,
10081                                source,
10082                            }),
10083                    );
10084            }
10085        }
10086        Ok(true)
10087    }
10088
10089    // Both pull and realtime frames use the same durable observation boundary.
10090    fn apply_commit_frame(
10091        &mut self,
10092        tables: &[String],
10093        changes: &[ssp2::model::Change],
10094        sequence: i64,
10095    ) -> Result<(), TransportError> {
10096        #[cfg(feature = "bench-internals")]
10097        let _phase = self.benchmark_phases.start(Phase::CommitApply);
10098        let rollback = self.capture();
10099        self.coalesce_unit(changes.len())
10100            .and_then(|()| self.begin_observation("syncular_commit"))
10101            .map_err(|message| TransportError::new("storage.failed", message))?;
10102        let applied = (|| {
10103            let mut batch = ChangeAccumulator::default();
10104            self.record_commit_changes(&mut batch, tables, changes);
10105            // §7.1 chained base: a change names the version an acknowledged own
10106            // commit wrote; pending upserts chained on it move to that version.
10107            for change in changes {
10108                let Some(table) = tables.get(change.table_index as usize) else {
10109                    continue;
10110                };
10111                let acknowledged = self
10112                    .conn
10113                    .prepare_cached("SELECT op_json FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2 AND commit_seq=?3")
10114                    .and_then(|mut statement| {
10115                        statement
10116                            .query_map(rusqlite::params![table, change.row_id, sequence], |row| row.get::<_, String>(0))?
10117                            .collect::<Result<Vec<_>, _>>()
10118                    })
10119                    .map_err(|error| TransportError::new("storage.failed", Self::sqlite_failure(&self.storage_failure, error)))?
10120                    .iter()
10121                    .map(|op| serde_json::from_str::<CommitOperation>(op).map(OutboxOp::from))
10122                    .collect::<Result<Vec<_>, _>>()
10123                    .map_err(|_| TransportError::new("storage.failed", "sync.local_corrupt: invalid acknowledged operation".to_owned()))?;
10124                for index in 0..self.outbox.len() {
10125                    if !self.outbox[index]
10126                        .ops
10127                        .iter()
10128                        .any(|op| acknowledged.iter().any(|earlier| chains_on(earlier, op)))
10129                    {
10130                        continue;
10131                    }
10132                    let commit = Arc::make_mut(&mut self.outbox[index]);
10133                    for op in &mut commit.ops {
10134                        if acknowledged.iter().any(|earlier| chains_on(earlier, op)) {
10135                            op.base_version = Some(change.row_version);
10136                        }
10137                    }
10138                    let ops = serde_json::to_string(
10139                        &commit
10140                            .ops
10141                            .iter()
10142                            .map(CommitOperation::from)
10143                            .collect::<Vec<_>>(),
10144                    )
10145                    .map_err(|_| {
10146                        TransportError::new(
10147                            "storage.failed",
10148                            "sync.local_corrupt: unencodable outbox".to_owned(),
10149                        )
10150                    })?;
10151                    let id = commit.client_commit_id.clone();
10152                    self.conn
10153                        .execute(
10154                            "UPDATE _syncular_outbox SET ops_json=?1 WHERE commit_id=?2",
10155                            rusqlite::params![ops, id],
10156                        )
10157                        .map_err(|error| {
10158                            TransportError::new(
10159                                "storage.failed",
10160                                Self::sqlite_failure(&self.storage_failure, error),
10161                            )
10162                        })?;
10163                }
10164            }
10165            {
10166                let storage_failed = |error| {
10167                    TransportError::new(
10168                        "storage.failed",
10169                        Self::sqlite_failure(&self.storage_failure, error),
10170                    )
10171                };
10172                let pinned: HashSet<(&str, &str)> = self
10173                    .outbox
10174                    .iter()
10175                    .flat_map(|commit| &commit.ops)
10176                    .map(|op| (op.table.as_str(), op.row_id.as_str()))
10177                    .collect();
10178                for change in changes {
10179                    let table = tables.get(change.table_index as usize).ok_or_else(|| {
10180                        TransportError::new(
10181                            "sync.invalid_request",
10182                            "change tableIndex out of range".to_owned(),
10183                        )
10184                    })?;
10185                    if pinned.contains(&(table.as_str(), change.row_id.as_str())) {
10186                        self.conn.prepare_cached("INSERT INTO _syncular_row_deliveries(tbl,id,commit_seq) VALUES (?1,?2,?3) ON CONFLICT(tbl,id) DO UPDATE SET commit_seq=MAX(commit_seq,excluded.commit_seq)")
10187                            .and_then(|mut statement| statement.execute(rusqlite::params![table, change.row_id, sequence]))
10188                            .map_err(storage_failed)?;
10189                    }
10190                    self.conn.prepare_cached("DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2 AND commit_seq<=?3")
10191                        .and_then(|mut statement| statement.execute(rusqlite::params![table, change.row_id, sequence]))
10192                        .map_err(storage_failed)?;
10193                }
10194            }
10195            self.apply_commit_changes(tables, changes)?;
10196            self.rebuild_overlay_if_dirty()
10197                .map_err(|message| TransportError::new("storage.failed", message))?;
10198            self.finish_observation("syncular_commit", batch)
10199                .map_err(|message| TransportError::new("storage.failed", message))
10200        })();
10201        if applied.is_err() {
10202            self.rollback_observation("syncular_commit");
10203            self.restore(rollback);
10204        }
10205        applied
10206    }
10207
10208    fn apply_commit_changes(
10209        &mut self,
10210        tables: &[String],
10211        changes: &[ssp2::model::Change],
10212    ) -> Result<(), TransportError> {
10213        // A clean overlay can reconcile changed primary keys independently
10214        // when their tables have no secondary unique constraints. Such a
10215        // constraint can change whether another row's write succeeds, so a
10216        // table with one and any local intent takes the table rebuild.
10217        let changed_tables: BTreeSet<&str> = changes
10218            .iter()
10219            .filter_map(|change| tables.get(change.table_index as usize))
10220            .map(String::as_str)
10221            .collect();
10222        let mut mirror_visible = !self.overlay_dirty.get();
10223        for name in &changed_tables {
10224            if mirror_visible
10225                && self
10226                    .schema
10227                    .table(name)
10228                    .is_some_and(|table| table.indexes.iter().any(|index| index.unique))
10229            {
10230                let (pending, protected) = self
10231                    .table_intent(name)
10232                    .map_err(|message| TransportError::new("storage.failed", message))?;
10233                mirror_visible = !pending && !protected;
10234            }
10235        }
10236        for change in changes {
10237            let table_name = tables.get(change.table_index as usize).ok_or_else(|| {
10238                TransportError::new(
10239                    "sync.invalid_request",
10240                    "change tableIndex out of range".to_owned(),
10241                )
10242            })?;
10243            let table = self.schema.table(table_name).ok_or_else(|| {
10244                TransportError::new(
10245                    "sync.schema_mismatch",
10246                    format!("change targets unknown table {table_name:?}"),
10247                )
10248            })?;
10249            let landed = match change.op {
10250                Op::Upsert => {
10251                    let payload = change.row.as_ref().ok_or_else(|| {
10252                        TransportError::new(
10253                            "sync.invalid_request",
10254                            "upsert change without row payload".to_owned(),
10255                        )
10256                    })?;
10257                    // §5.11: decrypt encrypted columns on apply, or seal the row.
10258                    let (row, sealed) = {
10259                        #[cfg(feature = "bench-internals")]
10260                        let _phase = self.benchmark_phases.start(Phase::RowDecode);
10261                        // §5.11: a decrypt failure keeps its own identity.
10262                        decode_row_bytes(table, payload, &self.encryption)?
10263                    };
10264                    let table_name = table.name.clone();
10265                    // §4.5 version guard: only a newer upsert lands.
10266                    let landed = self
10267                        .write_base_row(&table_name, &row, change.row_version, sealed.as_ref())
10268                        .map_err(|m| TransportError::new("sync.invalid_request", m))?;
10269                    if mirror_visible && landed {
10270                        self.write_row(
10271                            &visible_table(&table_name),
10272                            &table_name,
10273                            &row,
10274                            change.row_version,
10275                            sealed.as_ref(),
10276                        )
10277                        .map_err(|m| TransportError::new("sync.invalid_request", m))?;
10278                    }
10279                    landed
10280                }
10281                // §4.5 version guard: a delete at or below the local version
10282                // is skipped, a delete of an absent row is a no-op.
10283                Op::Delete => self
10284                    .delete_base_row(table_name, &change.row_id, change.row_version)
10285                    .map_err(|m| TransportError::new("sync.invalid_request", m))?,
10286            };
10287            // The change dropped the row's acknowledged intent either way
10288            // (§7.1), so the overlay of a skipped change rebuilds from the
10289            // unchanged base.
10290            self.overlay_dirty.table(table_name);
10291            if mirror_visible && !(landed && change.op == Op::Upsert) {
10292                let predicate = row_id_predicate(table);
10293                let visible = visible_table(table_name);
10294                let mut statements = vec![format!("DELETE FROM {visible} WHERE {predicate}")];
10295                if !landed {
10296                    statements.push(format!(
10297                        "INSERT INTO {visible} SELECT * FROM {} WHERE {predicate}",
10298                        base_table(table_name)
10299                    ));
10300                }
10301                for statement in &statements {
10302                    self.conn
10303                        .prepare_cached(statement)
10304                        .and_then(|mut statement| {
10305                            statement.execute(rusqlite::params![change.row_id])
10306                        })
10307                        .map_err(|error| {
10308                            TransportError::new("sync.invalid_request", error.to_string())
10309                        })?;
10310                }
10311            }
10312        }
10313        if mirror_visible {
10314            let changed_rows: HashSet<(&str, &str)> = changes
10315                .iter()
10316                .map(|change| {
10317                    (
10318                        tables[change.table_index as usize].as_str(),
10319                        change.row_id.as_str(),
10320                    )
10321                })
10322                .collect();
10323            self.replay_overlay(&changed_tables, Some(&changed_rows))
10324                .map_err(|message| TransportError::new("storage.failed", message))?;
10325            self.overlay_dirty.set(false);
10326        }
10327        Ok(())
10328    }
10329
10330    /// §5.6 segment application: validate against the generated schema,
10331    /// clear the pair's scope on the table's first page (fail closed
10332    /// without a mapping), then upsert each row with its segment-carried
10333    /// server version under the §4.5 version guard (§5.2).
10334    fn apply_segment(
10335        &mut self,
10336        table: &TableSchema,
10337        segment: &RowsSegment,
10338        first_page: bool,
10339        effective: &[(String, Vec<String>)],
10340        unit_id: &str,
10341    ) -> Result<u32, SectionError> {
10342        // §5.2: the column table validates against the generated schema —
10343        // order, names, types, nullability; mismatch is fatal. §5.11: the
10344        // server sends the WIRE types (bytes for an encrypted column), so
10345        // validate against wire_columns.
10346        let column = if segment.columns.len() != table.wire_columns.len() {
10347            Some(Value::from("count"))
10348        } else {
10349            segment
10350                .columns
10351                .iter()
10352                .zip(table.wire_columns.iter())
10353                .position(|(a, b)| a.name != b.name || a.ty != b.ty || a.nullable != b.nullable)
10354                .map(Value::from)
10355        };
10356        if segment.table != table.name
10357            || segment.schema_version != self.schema.version
10358            || column.is_some()
10359        {
10360            let mut details = Map::new();
10361            details.insert("table".to_owned(), Value::from(table.name.as_str()));
10362            if let Some(column) = column {
10363                details.insert("column".to_owned(), column);
10364            }
10365            return Err(SectionError::Transport(TransportError {
10366                code: "sync.schema_mismatch".to_owned(),
10367                message: "rows segment columns do not match the generated schema (§5.2)".to_owned(),
10368                details: Some(Value::Object(details)),
10369            }));
10370        }
10371        let mut applied = 0u32;
10372        let empty = Vec::new();
10373        let blocks = if segment.blocks.is_empty() {
10374            std::slice::from_ref(&empty)
10375        } else {
10376            &segment.blocks
10377        };
10378        for (index, block) in blocks.iter().enumerate() {
10379            let clear = first_page && index == 0;
10380            if !clear && block.is_empty() {
10381                continue;
10382            }
10383            let rollback = self.capture();
10384            let (pending, protected) = self
10385                .table_intent(&table.name)
10386                .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10387            let reconcile_pending =
10388                !protected && pending && table.indexes.iter().any(|index| index.unique);
10389            let mirror_visible = !protected && !reconcile_pending;
10390            self.coalesce_unit(block.len())
10391                .and_then(|()| self.begin_observation("syncular_segment_block"))
10392                .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10393            let outcome = (|| {
10394                self.rebuild_overlay_if_dirty()
10395                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10396                let mut changed_keys = Vec::new();
10397                let mut batch = ChangeAccumulator::default();
10398                if clear && self.clear_scope(table, effective, unit_id)? || !block.is_empty() {
10399                    batch.table(&table.name);
10400                }
10401                for (row_index, row) in block.iter().enumerate() {
10402                    let decrypted;
10403                    let mut sealed = None;
10404                    let values = if table.has_encrypted_columns() {
10405                        let mut values = row.values.clone();
10406                        sealed = crate::values::decrypt_segment_row(
10407                            table,
10408                            &mut values,
10409                            &self.encryption,
10410                        )
10411                        .map_err(SectionError::Transport)?;
10412                        decrypted = values;
10413                        &decrypted
10414                    } else {
10415                        &row.values
10416                    };
10417                    if !self
10418                        .write_base_row(&table.name, values, row.server_version, sealed.as_ref())
10419                        .map_err(|message| {
10420                            SectionError::Abort("sync.invalid_request".into(), message)
10421                        })?
10422                    {
10423                        continue;
10424                    }
10425                    if reconcile_pending {
10426                        changed_keys.push(
10427                            owned_sql_value(RowParam::Cell(&values[table.pk_index])).map_err(
10428                                |message| SectionError::Abort("storage.failed".into(), message),
10429                            )?,
10430                        );
10431                    }
10432                    if mirror_visible {
10433                        self.write_row(
10434                            &visible_table(&table.name),
10435                            &table.name,
10436                            values,
10437                            row.server_version,
10438                            sealed.as_ref(),
10439                        )
10440                        .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10441                    }
10442                    let processed = u64::from(applied) + row_index as u64 + 1;
10443                    if processed.is_multiple_of(1024) {
10444                        self.progress.update(|p| p.rows_processed = processed);
10445                    }
10446                }
10447                self.reconcile_import(table, &mut changed_keys, protected, reconcile_pending)
10448                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10449                self.finish_observation("syncular_segment_block", batch)
10450                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))
10451            })();
10452            if let Err(error) = outcome {
10453                self.rollback_observation("syncular_segment_block");
10454                self.restore(rollback);
10455                return Err(error);
10456            }
10457            applied += block.len() as u32;
10458            self.progress
10459                .update(|p| p.rows_processed = u64::from(applied));
10460            std::thread::yield_now();
10461        }
10462        self.progress
10463            .update(|p| p.rows_processed = u64::from(applied));
10464        Ok(applied)
10465    }
10466
10467    /// §5.3 sqlite-image application: validate the in-file metadata
10468    /// against the descriptor, validate column names/order against the
10469    /// generated schema, run the §5.6 first-page clear when fresh, then
10470    /// replace-or-upsert every image row with its `_syncular_version`.
10471    /// Mechanics: the image lands in a temp file read through a second
10472    /// rusqlite connection, equivalent to ATTACH + INSERT…SELECT.
10473    /// Each image chunk commits its writes and revision before readers continue.
10474    #[allow(clippy::too_many_arguments)]
10475    fn apply_sqlite_segment(
10476        &mut self,
10477        table: &TableSchema,
10478        bytes: &[u8],
10479        first_page: bool,
10480        effective: &[(String, Vec<String>)],
10481        unit_id: &str,
10482        segment_id: &str,
10483        resumed: i64,
10484        row_count: i64,
10485        as_of_commit_seq: i64,
10486        scope_digest: &str,
10487    ) -> Result<u32, SectionError> {
10488        let invalid = |detail: &str| {
10489            SectionError::Abort(
10490                "sync.invalid_request".to_owned(),
10491                format!("sqlite segment rejected: {detail} (§5.3)"),
10492            )
10493        };
10494        // §5.11: an image bypasses the decrypt seam, so a server cannot serve
10495        // one for a table with encrypted columns.
10496        if table.has_encrypted_columns() {
10497            return Err(invalid("table has encrypted columns"));
10498        }
10499
10500        // The image lands beside the replica so it shares the replica's
10501        // directory permissions and volume; an in-memory replica has no
10502        // directory and uses the system temp dir. The file is owner-only.
10503        let directory = self
10504            .previous_version_replica_path()
10505            .and_then(|replica| std::path::Path::new(&replica).parent().map(Into::into))
10506            .unwrap_or_else(std::env::temp_dir);
10507        let path = directory.join(format!(".syncular-image-{}.tmp", uuid::Uuid::new_v4()));
10508        let outcome = (|| {
10509            let mut options = std::fs::OpenOptions::new();
10510            options.write(true).create_new(true);
10511            #[cfg(unix)]
10512            std::os::unix::fs::OpenOptionsExt::mode(&mut options, 0o600);
10513            std::io::Write::write_all(
10514                &mut options
10515                    .open(&path)
10516                    .map_err(|_| invalid("image temp file write failed"))?,
10517                bytes,
10518            )
10519            .map_err(|_| invalid("image temp file write failed"))?;
10520            let img = rusqlite::Connection::open_with_flags(
10521                &path,
10522                rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY,
10523            )
10524            .map_err(|_| invalid("bytes do not open as a SQLite database"))?;
10525            self.apply_sqlite_image(
10526                &img,
10527                table,
10528                first_page,
10529                effective,
10530                unit_id,
10531                segment_id,
10532                resumed,
10533                row_count,
10534                as_of_commit_seq,
10535                scope_digest,
10536            )
10537        })();
10538        let _ = std::fs::remove_file(&path);
10539        outcome
10540    }
10541
10542    #[allow(clippy::too_many_arguments)]
10543    fn apply_sqlite_image(
10544        &mut self,
10545        img: &rusqlite::Connection,
10546        table: &crate::schema::TableSchema,
10547        first_fresh_page: bool,
10548        effective: &[(String, Vec<String>)],
10549        unit_id: &str,
10550        segment_id: &str,
10551        resumed: i64,
10552        row_count: i64,
10553        as_of_commit_seq: i64,
10554        scope_digest: &str,
10555    ) -> Result<u32, SectionError> {
10556        let invalid = |detail: String| {
10557            SectionError::Abort(
10558                "sync.invalid_request".to_owned(),
10559                format!("sqlite segment rejected: {detail} (§5.3)"),
10560            )
10561        };
10562
10563        // 1. Metadata vs descriptor + client state (§5.3 rule 2; exactly
10564        //    one row).
10565        type MetaRow = (i64, String, i64, i64, String, i64, i64);
10566        let meta: MetaRow = img
10567            .query_row(
10568                "SELECT format, \"table\", \"schemaVersion\", \"asOfCommitSeq\",
10569                        \"scopeDigest\", \"rowCount\",
10570                        (SELECT count(*) FROM _syncular_segment)
10571                 FROM _syncular_segment",
10572                [],
10573                |row| {
10574                    Ok((
10575                        row.get(0)?,
10576                        row.get(1)?,
10577                        row.get(2)?,
10578                        row.get(3)?,
10579                        row.get(4)?,
10580                        row.get(5)?,
10581                        row.get(6)?,
10582                    ))
10583                },
10584            )
10585            .map_err(|_| invalid("missing or unreadable _syncular_segment metadata".to_owned()))?;
10586        let (format, meta_table, schema_version, pin, digest, meta_rows, meta_count) = meta;
10587        if meta_count != 1 {
10588            return Err(invalid(format!(
10589                "_syncular_segment must contain exactly one row, found {meta_count}"
10590            )));
10591        }
10592        if format != 1 {
10593            return Err(invalid(format!("format {format}")));
10594        }
10595        if meta_table != table.name {
10596            return Err(invalid(format!("image table {meta_table:?}")));
10597        }
10598        if schema_version != i64::from(self.schema.version) {
10599            return Err(invalid(format!("schemaVersion {schema_version}")));
10600        }
10601        if pin != as_of_commit_seq {
10602            return Err(invalid(format!("asOfCommitSeq {pin}")));
10603        }
10604        if digest != scope_digest {
10605            return Err(invalid("scopeDigest mismatch".to_owned()));
10606        }
10607        if meta_rows != row_count {
10608            return Err(invalid(format!("rowCount {meta_rows}")));
10609        }
10610
10611        // 2. Column names and order vs the generated schema (§5.3 rule 3).
10612        let mut names: Vec<String> = Vec::new();
10613        let mut primary_keys: Vec<String> = Vec::new();
10614        {
10615            let mut stmt = img
10616                .prepare(&format!("PRAGMA table_info({})", quote_ident(&table.name)))
10617                .map_err(|_| invalid("image data table missing".to_owned()))?;
10618            let mut rows = stmt
10619                .query([])
10620                .map_err(|_| invalid("image data table unreadable".to_owned()))?;
10621            while let Some(row) = rows
10622                .next()
10623                .map_err(|_| invalid("image data table unreadable".to_owned()))?
10624            {
10625                if row
10626                    .get::<_, i64>(5)
10627                    .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?
10628                    > 0
10629                {
10630                    primary_keys.push(row.get(1).map_err(|error| {
10631                        invalid(Self::sqlite_failure(&self.storage_failure, error))
10632                    })?);
10633                }
10634                names.push(
10635                    row.get::<_, String>(1)
10636                        .map_err(|_| invalid("image data table unreadable".to_owned()))?,
10637                );
10638            }
10639        }
10640        let mut expected: Vec<&str> = table.columns.iter().map(|c| c.name.as_str()).collect();
10641        expected.push("_syncular_version");
10642        if names.len() != expected.len() || names.iter().zip(expected.iter()).any(|(a, b)| a != b) {
10643            return Err(SectionError::Transport(TransportError {
10644                code: "sync.schema_mismatch".to_owned(),
10645                message: "sqlite segment columns do not match the generated schema (§5.3)"
10646                    .to_owned(),
10647                details: Some(serde_json::json!({ "table": table.name })),
10648            }));
10649        }
10650
10651        if primary_keys != vec![table.primary_key.clone()] {
10652            return Err(invalid(
10653                "image primary key does not match generated schema".into(),
10654            ));
10655        }
10656        let actual_count: i64 = img
10657            .query_row(
10658                &format!("SELECT count(*) FROM {}", quote_ident(&table.name)),
10659                [],
10660                |row| row.get(0),
10661            )
10662            .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
10663        if actual_count != row_count {
10664            return Err(invalid("image row count does not match descriptor".into()));
10665        }
10666        let insert = self.insert_row_sql(&base_table(&table.name), table);
10667        let visible_insert = self.insert_row_sql(&visible_table(&table.name), table);
10668        let column_list = names
10669            .iter()
10670            .map(|n| quote_ident(n))
10671            .collect::<Vec<_>>()
10672            .join(", ");
10673        // §5.6: an earlier attempt at this position committed the first
10674        // `resumed` rows in primary-key order; the content address pins them.
10675        let mut stmt = img
10676            .prepare(&format!(
10677                "SELECT {column_list} FROM {} ORDER BY {} LIMIT -1 OFFSET ?1",
10678                quote_ident(&table.name),
10679                quote_ident(&table.primary_key)
10680            ))
10681            .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
10682        let mut rows = stmt
10683            .query([resumed])
10684            .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
10685        let resumed = u32::try_from(resumed)
10686            .ok()
10687            .filter(|rows| i64::from(*rows) <= actual_count)
10688            .ok_or_else(|| invalid("applied-segment record exceeds the image".into()))?;
10689        let mut applied = resumed;
10690        let mut limit = if first_fresh_page {
10691            1024
10692        } else {
10693            self.image_chunk_rows
10694                .get(&table.name)
10695                .copied()
10696                .unwrap_or(1024)
10697        };
10698        let (mut write, mut commit) = (std::time::Duration::ZERO, std::time::Duration::ZERO);
10699        loop {
10700            let clear = applied == 0 && first_fresh_page;
10701            let rollback = self.capture();
10702            let (pending, protected) = self
10703                .table_intent(&table.name)
10704                .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10705            let reconcile_pending =
10706                !protected && pending && table.indexes.iter().any(|index| index.unique);
10707            let mirror_visible = !protected && !reconcile_pending;
10708            // §5.3: a table's first chunk holds 1,024 rows; a later range image
10709            // starts at the size its previous range reached. A later chunk holds at
10710            // least 1,024 rows, at most twice the rows of the one before, at
10711            // most 65,536, and at most the rows the one before applied per
10712            // 250 ms from its start to its COMMIT; a chunk that replays local
10713            // writes holds at most 1,024.
10714            let chunk = if pending || protected {
10715                limit.min(1024)
10716            } else {
10717                limit
10718            };
10719            self.coalesce_unit(chunk as usize)
10720                .and_then(|()| self.begin_observation("syncular_image_chunk"))
10721                .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10722            let started = std::time::Instant::now();
10723            let result = (|| {
10724                self.rebuild_overlay_if_dirty()
10725                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10726                let mut changed_keys = Vec::new();
10727                let mut batch = ChangeAccumulator::default();
10728                if clear && self.clear_scope(table, effective, unit_id)? {
10729                    batch.table(&table.name);
10730                }
10731                let mut processed = 0u32;
10732                {
10733                    let mut ins = self.conn.prepare_cached(&insert).map_err(|error| {
10734                        invalid(Self::sqlite_failure(&self.storage_failure, error))
10735                    })?;
10736                    let mut mirror = if mirror_visible {
10737                        Some(self.conn.prepare_cached(&visible_insert).map_err(|error| {
10738                            invalid(Self::sqlite_failure(&self.storage_failure, error))
10739                        })?)
10740                    } else {
10741                        None
10742                    };
10743                    for _ in 0..chunk {
10744                        let Some(row) = rows.next().map_err(|error| {
10745                            invalid(Self::sqlite_failure(&self.storage_failure, error))
10746                        })?
10747                        else {
10748                            break;
10749                        };
10750                        for (i, column) in table.columns.iter().enumerate() {
10751                            let cell = row.get_ref(i).map_err(|error| {
10752                                invalid(Self::sqlite_failure(&self.storage_failure, error))
10753                            })?;
10754                            let param = image_cell_param(column, cell).map_err(invalid)?;
10755                            ins.raw_bind_parameter(i + 1, &param).map_err(|error| {
10756                                invalid(Self::sqlite_failure(&self.storage_failure, error))
10757                            })?;
10758                            if let Some(mirror) = &mut mirror {
10759                                mirror.raw_bind_parameter(i + 1, &param).map_err(|error| {
10760                                    invalid(Self::sqlite_failure(&self.storage_failure, error))
10761                                })?;
10762                            }
10763                        }
10764                        let version_index = table.columns.len();
10765                        let version: i64 = row.get(version_index).map_err(|error| {
10766                            invalid(Self::sqlite_failure(&self.storage_failure, error))
10767                        })?;
10768                        if version < 1 {
10769                            return Err(invalid("image row version must be positive".into()));
10770                        }
10771                        ins.raw_bind_parameter(version_index + 1, version)
10772                            .map_err(|error| {
10773                                invalid(Self::sqlite_failure(&self.storage_failure, error))
10774                            })?;
10775                        // §4.5 version guard: the base upsert decides.
10776                        let landed = ins.raw_execute().map_err(|error| {
10777                            invalid(Self::sqlite_failure(&self.storage_failure, error))
10778                        })? > 0;
10779                        processed += 1;
10780                        if processed.is_multiple_of(1024) && processed < chunk {
10781                            self.progress
10782                                .update(|p| p.rows_processed = u64::from(applied + processed));
10783                        }
10784                        if !landed {
10785                            continue;
10786                        }
10787                        if let Some(mirror) = &mut mirror {
10788                            mirror
10789                                .raw_bind_parameter(version_index + 1, version)
10790                                .map_err(|error| {
10791                                    invalid(Self::sqlite_failure(&self.storage_failure, error))
10792                                })?;
10793                            mirror.raw_execute().map_err(|error| {
10794                                invalid(Self::sqlite_failure(&self.storage_failure, error))
10795                            })?;
10796                        }
10797                        if reconcile_pending {
10798                            changed_keys.push(row.get::<_, SqlValue>(table.pk_index).map_err(
10799                                |error| invalid(Self::sqlite_failure(&self.storage_failure, error)),
10800                            )?);
10801                        }
10802                    }
10803                }
10804                if processed > 0 {
10805                    batch.table(&table.name);
10806                    self.overlay_dirty.table(&table.name);
10807                }
10808                self.stored(record_applied_rows(
10809                    &self.conn,
10810                    unit_id,
10811                    segment_id,
10812                    i64::from(applied + processed),
10813                ))
10814                .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10815                self.reconcile_import(table, &mut changed_keys, protected, reconcile_pending)
10816                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10817                let written = std::time::Instant::now();
10818                self.finish_observation("syncular_image_chunk", batch)
10819                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10820                Ok((processed, written))
10821            })();
10822            let (processed, written) = match result {
10823                Ok(done) => done,
10824                Err(error) => {
10825                    self.rollback_observation("syncular_image_chunk");
10826                    self.restore(rollback);
10827                    return Err(error);
10828                }
10829            };
10830            // The chunk commits before the yield, also inside a coalesced apply.
10831            let coalesced = self.coalesced.is_some();
10832            if coalesced {
10833                self.commit_coalesced_apply()
10834                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10835            }
10836            write += written - started;
10837            commit += written.elapsed();
10838            applied += processed;
10839            let last = processed < chunk || i64::from(applied) == actual_count;
10840            // A file replica's WAL keeps its high-water size until truncated.
10841            if last && self.conn.is_autocommit() {
10842                self.conn
10843                    .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))
10844                    .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
10845            }
10846            if coalesced {
10847                self.begin_coalesced_apply()
10848                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10849            }
10850            self.progress.update(|p| {
10851                p.rows_processed = u64::from(applied);
10852                p.import_write_ms = u64::try_from(write.as_millis()).ok();
10853                p.import_commit_ms = u64::try_from(commit.as_millis()).ok();
10854            });
10855            let duration = started.elapsed().as_secs_f64() * 1000.0;
10856            #[cfg(test)]
10857            let duration = self.image_chunk_ms.get().unwrap_or(duration);
10858            std::thread::yield_now();
10859            if last {
10860                break;
10861            }
10862            // A zero duration saturates the cast and leaves the doubling.
10863            limit = (processed * 2)
10864                .min(65_536)
10865                .min((f64::from(processed) * 250.0 / duration) as u32)
10866                .max(1024);
10867            self.image_chunk_rows.insert(table.name.clone(), limit);
10868        }
10869        if i64::from(applied) != actual_count {
10870            return Err(invalid(format!(
10871                "image yielded {applied} rows, expected {actual_count}"
10872            )));
10873        }
10874        Ok(applied - resumed)
10875    }
10876
10877    /// §7.1 keyed reconciliation of the rows `drained` commits touch: restore
10878    /// them from the base and replay their remaining intent. A table with a
10879    /// secondary unique index restores and replays every row with intent,
10880    /// since one row's value can admit or defer another row's write.
10881    fn reconcile_rows(&self, drained: &[Arc<OutboxCommit>]) -> Result<(), String> {
10882        let names: BTreeSet<&str> = drained
10883            .iter()
10884            .flat_map(|commit| &commit.ops)
10885            .map(|op| op.table.as_str())
10886            .collect();
10887        for name in names {
10888            let Some(table) = self.schema.table(name) else {
10889                continue;
10890            };
10891            let ops = drained.iter().flat_map(|commit| &commit.ops);
10892            let tables = BTreeSet::from([name]);
10893            let rows: HashSet<_> = ops
10894                .clone()
10895                .map(|op| (op.table.as_str(), op.row_id.as_str()))
10896                .collect();
10897            let unique = table.indexes.iter().any(|index| index.unique);
10898            let acknowledged = if unique {
10899                self.acknowledged_operations(None, Some(&tables))?
10900            } else {
10901                Vec::new()
10902            };
10903            let peers = acknowledged
10904                .iter()
10905                .chain(
10906                    self.failed_commits
10907                        .iter()
10908                        .flat_map(|(commit, _)| &commit.ops),
10909                )
10910                .chain(self.outbox.iter().flat_map(|commit| &commit.ops))
10911                .filter(|_| unique);
10912            self.restore_base_rows(table, &mut Vec::new(), peers.chain(ops.clone()))?;
10913            self.replay_overlay(&tables, (!unique).then_some(&rows))?;
10914        }
10915        Ok(())
10916    }
10917
10918    /// Whether `table` has pending outbox intent, and whether it has protected
10919    /// (acknowledged or retained failed) intent.
10920    fn table_intent(&self, table: &str) -> Result<(bool, bool), String> {
10921        let touches = |op: &OutboxOp| op.table == table;
10922        Ok((
10923            self.outbox
10924                .iter()
10925                .flat_map(|commit| &commit.ops)
10926                .any(touches),
10927            self.failed_commits
10928                .iter()
10929                .flat_map(|(commit, _)| &commit.ops)
10930                .any(touches)
10931                || self
10932                    .conn
10933                    .query_row(
10934                        "SELECT EXISTS(SELECT 1 FROM _syncular_acked_rows WHERE tbl=?1)",
10935                        [table],
10936                        |row| row.get(0),
10937                    )
10938                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
10939        ))
10940    }
10941
10942    /// §7.1 reconciliation after an import block or chunk. Keyed replay covers
10943    /// only pending intent, so a table with protected intent rebuilds; a
10944    /// `restore` restores the imported `keys` and every pending row first.
10945    fn reconcile_import(
10946        &mut self,
10947        table: &TableSchema,
10948        keys: &mut Vec<SqlValue>,
10949        protected: bool,
10950        restore: bool,
10951    ) -> Result<(), String> {
10952        if protected {
10953            return self.rebuild_overlay_if_dirty();
10954        }
10955        let pending = self.outbox.iter().flat_map(|commit| &commit.ops);
10956        if restore {
10957            self.restore_base_rows(table, keys, pending.clone())?;
10958        }
10959        self.apply_outbox_ops(pending.filter(|op| op.table == table.name))?;
10960        self.overlay_dirty.set(false);
10961        Ok(())
10962    }
10963
10964    /// Restore the visible rows of `keys` and of the rows `ops` touch in
10965    /// `table` from the base before the caller replays their intent. The
10966    /// caller owns the transaction and publishes its revision after replay.
10967    fn restore_base_rows<'a>(
10968        &self,
10969        table: &TableSchema,
10970        keys: &mut Vec<SqlValue>,
10971        ops: impl IntoIterator<Item = &'a OutboxOp>,
10972    ) -> Result<(), String> {
10973        let visible = visible_table(&table.name);
10974        let base = base_table(&table.name);
10975        let pk = quote_ident(&table.primary_key);
10976        let predicate = row_id_predicate(table);
10977        let mut lookup = self.conn.prepare_cached(&format!(
10978            "SELECT {pk} FROM {base} WHERE {predicate} UNION SELECT {pk} FROM {visible} WHERE {predicate}"
10979        )).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
10980        let mut deleted = HashSet::new();
10981        for op in ops.into_iter().filter(|op| op.table == table.name) {
10982            if op.upsert {
10983                if let Some(values) = &op.values {
10984                    let key = json_to_column_value(
10985                        &table.columns[table.pk_index],
10986                        values.get(&table.primary_key),
10987                    )?;
10988                    keys.push(owned_sql_value(RowParam::Cell(&key))?);
10989                }
10990            } else if deleted.insert(&op.row_id) {
10991                keys.extend(
10992                    lookup
10993                        .query_map([&op.row_id], |row| row.get::<_, SqlValue>(0))
10994                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
10995                        .collect::<Result<Vec<_>, _>>()
10996                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
10997                );
10998            }
10999        }
11000        // Delete the complete affected set before restoring any base value, so
11001        // old optimistic unique values cannot collide with the new server rows.
11002        for chunk in keys.chunks(400) {
11003            let holes = vec!["?"; chunk.len()].join(",");
11004            self.conn
11005                .prepare_cached(&format!("DELETE FROM {visible} WHERE {pk} IN ({holes})"))
11006                .and_then(|mut statement| statement.execute(rusqlite::params_from_iter(chunk)))
11007                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11008        }
11009        let updates = table
11010            .columns
11011            .iter()
11012            .map(|column| column.name.as_str())
11013            .chain(std::iter::once("_syncular_version"))
11014            .chain(table.sealed_columns().iter().copied())
11015            .filter(|name| *name != table.primary_key)
11016            .map(|name| {
11017                let name = quote_ident(name);
11018                format!("{name}=excluded.{name}")
11019            })
11020            .collect::<Vec<_>>()
11021            .join(",");
11022        for chunk in keys.chunks(400) {
11023            let holes = vec!["?"; chunk.len()].join(",");
11024            self.conn.prepare_cached(&format!("INSERT INTO {visible} SELECT * FROM {base} WHERE {pk} IN ({holes}) ON CONFLICT ({pk}) DO UPDATE SET {updates}"))
11025                .and_then(|mut statement| statement.execute(rusqlite::params_from_iter(chunk)))
11026                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11027        }
11028        Ok(())
11029    }
11030
11031    // -- application-authorized local purge ---------------------------------------
11032
11033    fn compile_local_data_purge(
11034        &self,
11035        input: &LocalDataPurgeInput,
11036    ) -> Result<(Vec<CompiledLocalDataPurgeTarget>, String), String> {
11037        let invalid = |message: String| format!("sync.invalid_request: {message}");
11038        if input.purge_id.is_empty()
11039            || input.purge_id.len() > 128
11040            || !is_local_operation_code_like(&input.purge_id)
11041        {
11042            return Err(invalid(
11043                "local purge purgeId must be a 1–128 character code-like identifier".to_owned(),
11044            ));
11045        }
11046        if input.targets.is_empty() || input.targets.len() > MAX_LOCAL_PURGE_TARGETS {
11047            return Err(invalid(format!(
11048                "local purge needs between 1 and {MAX_LOCAL_PURGE_TARGETS} targets"
11049            )));
11050        }
11051
11052        let mut deduplicated: BTreeMap<
11053            String,
11054            (CompiledLocalDataPurgeTarget, LocalDataPurgeTarget),
11055        > = BTreeMap::new();
11056        for target in &input.targets {
11057            let table = self.schema.table(&target.table).ok_or_else(|| {
11058                invalid(format!(
11059                    "local purge names unknown table {:?}",
11060                    target.table
11061                ))
11062            })?;
11063            if target.selectors.is_empty() || target.selectors.len() > MAX_LOCAL_PURGE_SELECTORS {
11064                return Err(invalid(format!(
11065                    "local purge target {:?} needs between 1 and {MAX_LOCAL_PURGE_SELECTORS} selectors",
11066                    target.table
11067                )));
11068            }
11069            let mut selectors = Vec::with_capacity(target.selectors.len());
11070            let mut canonical_selectors = BTreeMap::new();
11071            for (column_name, raw_values) in &target.selectors {
11072                let Some((column_index, column)) = table
11073                    .columns
11074                    .iter()
11075                    .enumerate()
11076                    .find(|(_, column)| column.name == *column_name)
11077                else {
11078                    return Err(invalid(format!(
11079                        "local purge target {:?} names unknown column {:?}",
11080                        target.table, column_name
11081                    )));
11082                };
11083                let encrypted = table
11084                    .encrypted_columns
11085                    .iter()
11086                    .any(|candidate| candidate.index == column_index);
11087                if column.ty != ColumnType::String || encrypted {
11088                    return Err(invalid(format!(
11089                        "local purge selector {:?}.{:?} must be a plaintext string column",
11090                        target.table, column_name
11091                    )));
11092                }
11093                if raw_values.is_empty() || raw_values.len() > MAX_LOCAL_PURGE_VALUES {
11094                    return Err(invalid(format!(
11095                        "local purge selector {:?}.{:?} needs between 1 and {MAX_LOCAL_PURGE_VALUES} values",
11096                        target.table, column_name
11097                    )));
11098                }
11099                let mut values = raw_values.clone();
11100                values.sort();
11101                values.dedup();
11102                if values.iter().any(|value| {
11103                    value.is_empty()
11104                        || value.len() > MAX_LOCAL_PURGE_VALUE_LENGTH
11105                        || !is_local_operation_code_like(value)
11106                }) {
11107                    return Err(invalid(format!(
11108                        "local purge selector values must be 1–{MAX_LOCAL_PURGE_VALUE_LENGTH} character code-like identifiers"
11109                    )));
11110                }
11111                selectors.push((column_name.clone(), values.clone()));
11112                canonical_selectors.insert(column_name.clone(), values);
11113            }
11114            let canonical = LocalDataPurgeTarget {
11115                table: target.table.clone(),
11116                selectors: canonical_selectors,
11117            };
11118            let key = serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
11119            deduplicated.insert(
11120                key,
11121                (
11122                    CompiledLocalDataPurgeTarget {
11123                        table: target.table.clone(),
11124                        selectors,
11125                    },
11126                    canonical,
11127                ),
11128            );
11129        }
11130        let targets = deduplicated
11131            .values()
11132            .map(|(compiled, _)| compiled.clone())
11133            .collect::<Vec<_>>();
11134        let canonical = deduplicated
11135            .values()
11136            .map(|(_, target)| target.clone())
11137            .collect::<Vec<_>>();
11138        let canonical_plan =
11139            serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
11140        Ok((targets, canonical_plan))
11141    }
11142
11143    fn local_purge_base_row_ids(
11144        &self,
11145        targets: &[CompiledLocalDataPurgeTarget],
11146    ) -> Result<BTreeMap<String, BTreeSet<String>>, String> {
11147        let mut by_table: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
11148        for target in targets {
11149            let table = self
11150                .schema
11151                .table(&target.table)
11152                .ok_or_else(|| format!("sync.invalid_request: unknown table {:?}", target.table))?;
11153            let mut clauses = Vec::with_capacity(target.selectors.len());
11154            let mut params = Vec::new();
11155            for (column, values) in &target.selectors {
11156                clauses.push(format!(
11157                    "{} IN ({})",
11158                    quote_ident(column),
11159                    values.iter().map(|_| "?").collect::<Vec<_>>().join(", ")
11160                ));
11161                params.extend(values.iter().cloned().map(SqlValue::Text));
11162            }
11163            let sql = format!(
11164                "SELECT CAST({} AS TEXT) FROM {} WHERE {}",
11165                quote_ident(&table.primary_key),
11166                base_table(&target.table),
11167                clauses.join(" AND ")
11168            );
11169            let mut statement = self
11170                .conn
11171                .prepare(&sql)
11172                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11173            let rows = statement
11174                .query_map(rusqlite::params_from_iter(params), |row| {
11175                    let value = sql_ref_to_json(&table.columns[table.pk_index], row.get_ref(0)?);
11176                    render_row_id_json(Some(&value))
11177                        .map_err(|message| rusqlite::Error::ToSqlConversionFailure(message.into()))
11178                })
11179                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11180            let ids = by_table.entry(target.table.clone()).or_default();
11181            for row in rows {
11182                ids.insert(
11183                    row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
11184                );
11185            }
11186        }
11187        Ok(by_table)
11188    }
11189
11190    fn local_purge_values_match(
11191        target: &CompiledLocalDataPurgeTarget,
11192        values: &Map<String, Value>,
11193    ) -> bool {
11194        target.selectors.iter().all(|(column, allowed)| {
11195            values
11196                .get(column)
11197                .and_then(Value::as_str)
11198                .is_some_and(|value| allowed.iter().any(|candidate| candidate == value))
11199        })
11200    }
11201
11202    /// Apply one host-authorized local security purge. The host MUST gate the
11203    /// corresponding subscriptions first; this primitive owns local SQLite
11204    /// cleanup and never grants/revokes server authority by itself.
11205    pub fn purge_local_data(
11206        &mut self,
11207        input: &LocalDataPurgeInput,
11208    ) -> Result<LocalDataPurgeResult, String> {
11209        let (targets, canonical_plan) = self.compile_local_data_purge(input)?;
11210        let meta_key = format!("localPurge:{}", input.purge_id);
11211        let applied_plan = self
11212            .conn
11213            .query_row(
11214                "SELECT value FROM _syncular_meta WHERE key = ?1",
11215                rusqlite::params![meta_key],
11216                |row| row.get::<_, String>(0),
11217            )
11218            .optional()
11219            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11220        if let Some(applied_plan) = applied_plan {
11221            if applied_plan != canonical_plan {
11222                return Err(format!(
11223                    "sync.invalid_request: local purge id {:?} was already used with a different plan",
11224                    input.purge_id
11225                ));
11226            }
11227            // A retry after a failed rewrite finishes it.
11228            self.erase_purged_pages()?;
11229            return Ok(LocalDataPurgeResult {
11230                already_applied: true,
11231                purged_rows: 0,
11232                dropped_commits: 0,
11233            });
11234        }
11235
11236        // RFC 0005 D7: the container file cannot be a purge target (its name is
11237        // not in the running schema), so purge drops it as a fixed step, before
11238        // any mirror row is touched and unconditionally.
11239        let _ = self.drop_previous_version();
11240
11241        let rollback = self.capture();
11242        self.begin_observation("syncular_local_purge")?;
11243        let applied = (|| -> Result<(ChangeAccumulator, LocalDataPurgeResult), String> {
11244            let row_ids = self.local_purge_base_row_ids(&targets)?;
11245            let retained_dropped = self.drop_failed_commits(|table, values| {
11246                targets.iter().any(|target| {
11247                    target.table == table && Self::local_purge_values_match(target, values)
11248                })
11249            })?;
11250            let doomed = self
11251                .outbox
11252                .iter()
11253                .filter(|commit| {
11254                    commit.ops.iter().any(|operation| {
11255                        let matching_targets = targets
11256                            .iter()
11257                            .filter(|target| target.table == operation.table)
11258                            .collect::<Vec<_>>();
11259                        if matching_targets.is_empty() {
11260                            return false;
11261                        }
11262                        if row_ids
11263                            .get(&operation.table)
11264                            .is_some_and(|ids| ids.contains(&operation.row_id))
11265                        {
11266                            return true;
11267                        }
11268                        operation.values.as_ref().is_some_and(|values| {
11269                            matching_targets
11270                                .iter()
11271                                .any(|target| Self::local_purge_values_match(target, values))
11272                        })
11273                    })
11274                })
11275                .cloned()
11276                .collect::<Vec<_>>();
11277            let doomed_ids = doomed
11278                .iter()
11279                .map(|commit| commit.client_commit_id.clone())
11280                .collect::<BTreeSet<_>>();
11281            let mut batch = ChangeAccumulator::default();
11282            for commit in &retained_dropped {
11283                batch.conflicts = true;
11284                batch.rejections = true;
11285                batch.outcomes = true;
11286                for operation in &commit.ops {
11287                    batch.table(&operation.table);
11288                }
11289            }
11290            let mut rejections = Vec::new();
11291            // §5.11 purge erasure: the journal keeps a dropped commit's identity and
11292            // codes, never its operations, so no purged value survives in it.
11293            for commit in &doomed {
11294                for operation in &commit.ops {
11295                    batch.table(&operation.table);
11296                    self.overlay_dirty.table(&operation.table);
11297                }
11298                let results = (0..commit.ops.len())
11299                    .map(|op_index| {
11300                        let rejection = RejectionRecord {
11301                            client_commit_id: commit.client_commit_id.clone(),
11302                            op_index: op_index as i32,
11303                            code: "client.local_data_purged".to_owned(),
11304                            message: "the commit was dropped by an application-authorized local data purge".to_owned(),
11305                            retryable: false,
11306                            details: None,
11307                            operation: None,
11308                        };
11309                        rejections.push(rejection.clone());
11310                        CommitOperationOutcome::Error { rejection }
11311                    })
11312                    .collect::<Vec<_>>();
11313                self.persist_commit_outcome(
11314                    &commit.client_commit_id,
11315                    CommitOutcomeStatus::Rejected,
11316                    &results,
11317                    None,
11318                )?;
11319                self.delete_outbox_persisted(&commit.client_commit_id)?;
11320            }
11321            if !doomed.is_empty() {
11322                self.outbox
11323                    .retain(|commit| !doomed_ids.contains(&commit.client_commit_id));
11324                self.rejections.extend(rejections);
11325                self.prune_commit_outcomes()?;
11326                batch.status = true;
11327                batch.rejections = true;
11328                batch.outcomes = true;
11329            }
11330
11331            let mut purged_rows = 0usize;
11332            for (table, ids) in &row_ids {
11333                if ids.is_empty() {
11334                    continue;
11335                }
11336                batch.table(table);
11337                self.overlay_dirty.table(table);
11338                for row_id in ids {
11339                    self.delete_base_row(table, row_id, i64::MAX)?;
11340                    purged_rows += 1;
11341                }
11342            }
11343            // §5.11 purge erasure: a journaled outcome with an operation or
11344            // conflict row the purge reaches keeps its identity, status, ids
11345            // and codes only.
11346            let touches = |table: &str, row_id: &str, values: Option<&Map<String, Value>>| {
11347                targets.iter().any(|target| {
11348                    target.table == table
11349                        && (row_ids.get(table).is_some_and(|ids| ids.contains(row_id))
11350                            || values.is_some_and(|values| {
11351                                Self::local_purge_values_match(target, values)
11352                            }))
11353                })
11354            };
11355            let entries = self
11356                .conn
11357                .prepare("SELECT client_commit_id, results_json, operations_json FROM _syncular_commit_outcomes")
11358                .and_then(|mut statement| {
11359                    statement
11360                        .query_map([], |row| {
11361                            Ok((
11362                                row.get::<_, String>(0)?,
11363                                row.get::<_, String>(1)?,
11364                                row.get::<_, Option<String>>(2)?,
11365                            ))
11366                        })?
11367                        .collect::<Result<Vec<_>, _>>()
11368                })
11369                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11370            for (client_commit_id, results_json, operations_json) in entries {
11371                let corrupt = |_| "sync.local_corrupt: persisted commit outcome is invalid".to_owned();
11372                let mut results: Vec<CommitOperationOutcome> =
11373                    serde_json::from_str(&results_json).map_err(corrupt)?;
11374                let operations: Vec<CommitOperation> = operations_json
11375                    .map(|json| serde_json::from_str(&json))
11376                    .transpose()
11377                    .map_err(corrupt)?
11378                    .unwrap_or_default();
11379                let named = operations
11380                    .iter()
11381                    .chain(results.iter().filter_map(|result| match result {
11382                        CommitOperationOutcome::Conflict { conflict } => conflict.operation.as_ref(),
11383                        CommitOperationOutcome::Error { rejection } => rejection.operation.as_ref(),
11384                        CommitOperationOutcome::Applied { .. } => None,
11385                    }))
11386                    .any(|operation| touches(&operation.table, &operation.row_id, operation.values.as_ref()))
11387                    || results.iter().any(|result| {
11388                        matches!(result, CommitOperationOutcome::Conflict { conflict }
11389                            if touches(&conflict.table, &conflict.row_id, Some(&conflict.server_row)))
11390                    });
11391                if !named {
11392                    continue;
11393                }
11394                for result in &mut results {
11395                    match result {
11396                        CommitOperationOutcome::Conflict { conflict } => {
11397                            conflict.server_row = Map::new();
11398                            conflict.conflict_columns.clear();
11399                            conflict.operation = None;
11400                        }
11401                        CommitOperationOutcome::Error { rejection } => {
11402                            rejection.details = None;
11403                            rejection.operation = None;
11404                        }
11405                        CommitOperationOutcome::Applied { .. } => {}
11406                    }
11407                }
11408                let results_json = serde_json::to_string(&results).map_err(|error| error.to_string())?;
11409                self.conn
11410                    .execute(
11411                        "UPDATE _syncular_commit_outcomes SET results_json = ?1, operations_json = NULL WHERE client_commit_id = ?2",
11412                        rusqlite::params![results_json, client_commit_id],
11413                    )
11414                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11415                batch.outcomes = true;
11416            }
11417            self.rebuild_overlay_if_dirty()?;
11418            self.delete_unreferenced_cached_blobs()?;
11419            self.conn
11420                .execute(
11421                    "INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
11422                    rusqlite::params![meta_key, canonical_plan],
11423                )
11424                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11425            Ok((
11426                batch,
11427                LocalDataPurgeResult {
11428                    already_applied: false,
11429                    purged_rows,
11430                    dropped_commits: doomed.len(),
11431                },
11432            ))
11433        })()
11434        .and_then(|(batch, result)| {
11435            self.finish_observation("syncular_local_purge", batch)
11436                .map(|()| result)
11437        });
11438        let result = match applied {
11439            Ok(result) => result,
11440            Err(error) => {
11441                self.rollback_observation("syncular_local_purge");
11442                self.restore(rollback);
11443                return Err(error);
11444            }
11445        };
11446        self.cancel_sync_round();
11447        self.erase_purged_pages()?;
11448        Ok(result)
11449    }
11450
11451    /// §5.11 purge erasure: SQLite leaves a deleted value in the freed pages of the
11452    /// database file, and the WAL keeps the frames written before the purge,
11453    /// until either is rewritten. VACUUM rebuilds the file from the remaining
11454    /// rows, and the checkpoint truncates the WAL.
11455    fn erase_purged_pages(&self) -> Result<(), String> {
11456        let erased = self
11457            .conn
11458            .execute_batch("PRAGMA secure_delete = ON; VACUUM;")
11459            .and_then(|()| {
11460                self.conn
11461                    .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))
11462            });
11463        let reset = self.conn.execute_batch("PRAGMA secure_delete = OFF");
11464        erased
11465            .and(reset)
11466            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
11467    }
11468
11469    /// Application-authorized recovery of the replicated projection. Unlike a
11470    /// local security purge, this retains the entire outbox, device identity,
11471    /// lease, outcomes, unit registrations, and protected bookkeeping.
11472    /// The projection reset, unit rewind, optimistic replay, and
11473    /// idempotency marker share one savepoint for interruption safety.
11474    pub fn rebootstrap_local_data(
11475        &mut self,
11476        input: &LocalDataRebootstrapInput,
11477    ) -> Result<LocalDataRebootstrapResult, String> {
11478        if input.rebootstrap_id.is_empty()
11479            || input.rebootstrap_id.len() > 128
11480            || !is_local_operation_code_like(&input.rebootstrap_id)
11481        {
11482            return Err(
11483                "sync.invalid_request: local rebootstrap rebootstrapId must be a 1–128 character code-like identifier"
11484                    .to_owned(),
11485            );
11486        }
11487        let meta_key = format!("localRebootstrap:{}", input.rebootstrap_id);
11488        if let Some(persisted) = meta_get(&self.conn, &meta_key).map_err(|_| {
11489            "sync.local_corrupt: persisted local rebootstrap receipt is unreadable".to_owned()
11490        })? {
11491            let (retained_commits, reset_subscriptions) =
11492                decode_local_rebootstrap_receipt(&persisted)?;
11493            return Ok(LocalDataRebootstrapResult {
11494                already_applied: true,
11495                retained_commits,
11496                reset_subscriptions,
11497            });
11498        }
11499        if self.stopped || self.schema_floor.is_some() {
11500            return Err(
11501                "sync.invalid_request: local rebootstrap cannot bypass an active schema-floor stop; update the application first"
11502                    .to_owned(),
11503            );
11504        }
11505
11506        let retained_commits = self.outbox.len();
11507        let reset_subscriptions = self.stored(load_units(&self.conn))?.len();
11508        let receipt = encode_local_rebootstrap_receipt(retained_commits, reset_subscriptions)?;
11509        let rollback = self.capture();
11510        self.begin_observation("syncular_local_rebootstrap")?;
11511        let mut batch = ChangeAccumulator::default();
11512        let applied = (|| -> Result<(), String> {
11513            self.run_schema_reset_observed(&mut batch, false)?;
11514            self.conn
11515                .execute(
11516                    "INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
11517                    rusqlite::params![meta_key, receipt],
11518                )
11519                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11520            self.sync_needed = true;
11521            self.sync_intent_queue.push_back(SyncIntent::Interactive);
11522            batch.status = true;
11523            Ok(())
11524        })()
11525        .and_then(|()| self.finish_observation("syncular_local_rebootstrap", batch));
11526        if let Err(error) = applied {
11527            self.rollback_observation("syncular_local_rebootstrap");
11528            self.restore(rollback);
11529            return Err(error);
11530        }
11531        self.cancel_sync_round();
11532
11533        Ok(LocalDataRebootstrapResult {
11534            already_applied: false,
11535            retained_commits,
11536            reset_subscriptions,
11537        })
11538    }
11539
11540    // -- scope purge + doomed outbox (§3.3) ----------------------------------------
11541
11542    /// Delete base and visible rows owned by the effective scopes
11543    /// ([`local_scope_predicate`]); `Ok(false)` = no local scope-column
11544    /// mapping for a key (the fail-closed case). A SQLite failure is an error.
11545    fn purge_scope_rows(
11546        &mut self,
11547        table_name: &str,
11548        effective: &[(String, Vec<String>)],
11549        owners: &[Vec<(String, Vec<String>)>],
11550    ) -> Result<bool, String> {
11551        let Some(table) = self.schema.table(table_name) else {
11552            return Ok(false);
11553        };
11554        let Ok(predicate) = local_scope_predicate(table, effective, owners) else {
11555            return Ok(false);
11556        };
11557        let Some((predicate, params)) = predicate else {
11558            return Ok(true);
11559        };
11560        self.overlay_dirty.table(table_name);
11561        for source in [base_table(table_name), visible_table(table_name)] {
11562            self.conn
11563                .execute(
11564                    &format!("DELETE FROM {source} WHERE {predicate}"),
11565                    rusqlite::params_from_iter(&params),
11566                )
11567                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11568        }
11569        Ok(true)
11570    }
11571
11572    /// §3.3: drop pending commits whose upserts provably land in the
11573    /// revoked effective scopes and no other owner covers (row ownership)
11574    /// — whole-commit, never per-operation.
11575    fn drop_doomed_outbox(
11576        &mut self,
11577        table: &TableSchema,
11578        effective: &[(String, Vec<String>)],
11579        owners: &[Vec<(String, Vec<String>)>],
11580    ) -> Result<bool, String> {
11581        let doomed: Vec<_> =
11582            self.outbox
11583                .iter()
11584                .filter(|commit| {
11585                    commit.ops.iter().any(|op| {
11586                        op.upsert
11587                            && op.table == table.name
11588                            && op.values.as_ref().is_some_and(|values| {
11589                                owns_local_row(table, effective, owners, values)
11590                            })
11591                    })
11592                })
11593                .cloned()
11594                .collect();
11595        if doomed.is_empty() {
11596            return Ok(false);
11597        }
11598        let mut rejections = Vec::new();
11599        for commit in &doomed {
11600            let results = commit
11601                .ops
11602                .iter()
11603                .enumerate()
11604                .map(|(op_index, operation)| {
11605                    self.overlay_dirty.table(&operation.table);
11606                    let rejection = RejectionRecord {
11607                        client_commit_id: commit.client_commit_id.clone(),
11608                        op_index: op_index as i32,
11609                        code: "sync.scope_revoked".to_owned(),
11610                        message: "the commit was dropped because its effective scope was revoked"
11611                            .to_owned(),
11612                        retryable: false,
11613                        details: None,
11614                        operation: Some(CommitOperation::from(operation)),
11615                    };
11616                    rejections.push(rejection.clone());
11617                    CommitOperationOutcome::Error { rejection }
11618                })
11619                .collect::<Vec<_>>();
11620            self.persist_commit_outcome(
11621                &commit.client_commit_id,
11622                CommitOutcomeStatus::Rejected,
11623                &results,
11624                Some(&commit.ops),
11625            )?;
11626            self.delete_outbox_persisted(&commit.client_commit_id)?;
11627        }
11628        self.prune_commit_outcomes()?;
11629        let doomed_ids = doomed
11630            .iter()
11631            .map(|commit| commit.client_commit_id.as_str())
11632            .collect::<BTreeSet<_>>();
11633        self.outbox
11634            .retain(|commit| !doomed_ids.contains(commit.client_commit_id.as_str()));
11635        self.rejections.extend(rejections);
11636        Ok(true)
11637    }
11638
11639    // -- blobs (§5.9) ----------------------------------------------------------------
11640
11641    /// §5.9.7: hash bytes into the content address, cache them, queue the
11642    /// upload (flushed before the next push, B4). Returns the canonical
11643    /// BlobRef JSON `{blobId, byteLength, mediaType?}` for a `blob_ref`
11644    /// column value.
11645    pub fn upload_blob(
11646        &mut self,
11647        bytes: &[u8],
11648        media_type: Option<String>,
11649        name: Option<String>,
11650    ) -> Result<Value, String> {
11651        let blob_id = blob_id_for(bytes);
11652        let now = self.clock_now_ms();
11653        let tx = self
11654            .conn
11655            .transaction()
11656            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11657        tx.execute(
11658            "INSERT INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES (?,?,?,?,?)
11659             ON CONFLICT(blob_id) DO NOTHING",
11660            rusqlite::params![blob_id, bytes, bytes.len() as i64, media_type, now],
11661        )
11662        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11663        tx.execute(
11664            "INSERT OR IGNORE INTO _syncular_blob_uploads(blob_id, media_type, created_at_ms) VALUES (?,?,?)",
11665            rusqlite::params![blob_id, media_type, now],
11666        )
11667        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11668        tx.commit()
11669            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11670        self.enforce_blob_cache_cap();
11671        let mut obj = Map::new();
11672        obj.insert("blobId".to_owned(), Value::from(blob_id));
11673        obj.insert("byteLength".to_owned(), Value::from(bytes.len() as i64));
11674        if let Some(mt) = media_type {
11675            obj.insert("mediaType".to_owned(), Value::from(mt));
11676        }
11677        if let Some(n) = name {
11678            obj.insert("name".to_owned(), Value::from(n));
11679        }
11680        Ok(Value::Object(obj))
11681    }
11682
11683    /// §5.9.7: resolve an owned blob body without driver encoding. A
11684    /// content-addressed cache hit serves with no fetch (B1); a miss downloads
11685    /// (§5.9.5), verifies the address, caches the bytes, and returns them.
11686    pub fn fetch_blob_bytes(
11687        &mut self,
11688        transport: &mut dyn Transport,
11689        blob_id_or_ref: &str,
11690    ) -> Result<FetchedBlob, (String, String)> {
11691        let simple = |m: String| ("client.failed".to_owned(), m);
11692        // A BlobRef declares the size that caps the download (§5.9.5); a bare
11693        // blobId declares none.
11694        let (blob_id, byte_length) = if blob_id_or_ref.starts_with("sha256:") {
11695            (blob_id_or_ref.to_owned(), None)
11696        } else {
11697            let value: Value = serde_json::from_str(blob_id_or_ref)
11698                .map_err(|_| simple("blob ref is not JSON".to_owned()))?;
11699            (
11700                value
11701                    .get("blobId")
11702                    .and_then(Value::as_str)
11703                    .ok_or_else(|| simple("blob ref has no blobId".to_owned()))?
11704                    .to_owned(),
11705                value.get("byteLength").and_then(Value::as_u64),
11706            )
11707        };
11708        if let Some(cached) = self.get_cached_blob_bytes(&blob_id).map_err(simple)? {
11709            return Ok(cached);
11710        }
11711        if !self.transport_enabled {
11712            return Err(("sync.offline".into(), "transport gate is closed".into()));
11713        }
11714        // §5.9.5: propagate the server's blob.* code (blob.forbidden /
11715        // blob.not_found) verbatim so the harness can assert on it. The
11716        // authorized endpoint serves bytes inline OR (always-issue, presign
11717        // configured) a signed url the client fetches directly — no host auth,
11718        // no fall-through: failure => re-request (the caller's next fetch_blob_bytes).
11719        #[cfg(feature = "bench-internals")]
11720        let download_phase = self.benchmark_phases.start(Phase::BlobDownload);
11721        let bytes = match transport
11722            .blob_download(&blob_id, byte_length)
11723            .map_err(|e| (e.code, e.message))?
11724        {
11725            BlobDownload::Bytes(bytes) => bytes,
11726            BlobDownload::Url {
11727                url,
11728                url_expires_at_ms,
11729            } => {
11730                // §5.9.5: MUST NOT start a fetch at/past expiry.
11731                if url_expires_at_ms.is_some_and(|exp| exp <= self.clock_now_ms()) {
11732                    return Err((
11733                        "sync.segment_expired".to_owned(),
11734                        format!(
11735                            "blob url for {blob_id} expired before fetch — re-request mints a fresh url (§5.9.5)"
11736                        ),
11737                    ));
11738                }
11739                transport
11740                    .fetch_blob_url(&url, byte_length)
11741                    .map_err(|e| (e.code, e.message))?
11742            }
11743        };
11744        #[cfg(feature = "bench-internals")]
11745        drop(download_phase);
11746        #[cfg(feature = "bench-internals")]
11747        let validate_phase = self.benchmark_phases.start(Phase::BlobValidate);
11748        // §5.9.5 inherits §5.1: the declared size caps the body before the
11749        // content address is verified.
11750        if byte_length.is_some_and(|cap| bytes.len() as u64 > cap) {
11751            return Err((
11752                "sync.invalid_request".to_owned(),
11753                "blob exceeds its byteLength".to_owned(),
11754            ));
11755        }
11756        if blob_id_for(&bytes) != blob_id {
11757            return Err(simple(format!(
11758                "blob content address mismatch for {blob_id}"
11759            )));
11760        }
11761        #[cfg(feature = "bench-internals")]
11762        drop(validate_phase);
11763        #[cfg(feature = "bench-internals")]
11764        let insert_phase = self.benchmark_phases.start(Phase::BlobCacheInsert);
11765        let now = self.clock_now_ms();
11766        self.conn
11767            .execute(
11768                "INSERT OR IGNORE INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES (?,?,?,NULL,?)",
11769                rusqlite::params![blob_id, bytes, bytes.len() as i64, now],
11770            )
11771            .map_err(|e| simple(e.to_string()))?;
11772        self.conn
11773            .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))
11774            .map_err(|e| simple(e.to_string()))?;
11775        self.enforce_blob_cache_cap();
11776        #[cfg(feature = "bench-internals")]
11777        drop(insert_phase);
11778        self.get_cached_blob_bytes(&blob_id)
11779            .map_err(simple)?
11780            .ok_or_else(|| simple("blob cache write failed".to_owned()))
11781    }
11782
11783    fn get_cached_blob_bytes(&self, blob_id: &str) -> Result<Option<FetchedBlob>, String> {
11784        #[cfg(feature = "bench-internals")]
11785        let _phase = self.benchmark_phases.start(Phase::BlobCacheRead);
11786        let mut stmt = self
11787            .conn
11788            .prepare("SELECT bytes, byte_length, media_type FROM _syncular_blobs WHERE blob_id = ?")
11789            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11790        let mut rows = stmt
11791            .query(rusqlite::params![blob_id])
11792            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11793        if let Some(row) = rows
11794            .next()
11795            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
11796        {
11797            let bytes: Vec<u8> = row
11798                .get(0)
11799                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11800            let byte_length: i64 = row
11801                .get(1)
11802                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11803            let media_type: Option<String> = row
11804                .get(2)
11805                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11806            return Ok(Some(FetchedBlob {
11807                blob_id: blob_id.to_owned(),
11808                byte_length,
11809                bytes,
11810                media_type,
11811            }));
11812        }
11813        Ok(None)
11814    }
11815
11816    /// §5.9.7 B4: upload every queued blob before push.
11817    fn pending_blob_uploads(&self) -> Result<Vec<crate::round::PendingUpload>, TransportError> {
11818        let pending: Vec<(String, Option<String>)> = {
11819            let mut stmt = self
11820                .conn
11821                .prepare(
11822                    "SELECT blob_id, media_type FROM (
11823                      SELECT blob_id, media_type, created_at_ms FROM _syncular_blob_uploads
11824                      UNION ALL
11825                      SELECT DISTINCT r.blob_id, b.media_type, 9223372036854775807 AS created_at_ms
11826                      FROM _syncular_blob_commit_refs r LEFT JOIN _syncular_blobs b ON b.blob_id = r.blob_id
11827                      WHERE NOT EXISTS (SELECT 1 FROM _syncular_blob_uploads u WHERE u.blob_id = r.blob_id)
11828                    ) ORDER BY created_at_ms, blob_id",
11829                )
11830                .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
11831            let rows = stmt
11832                .query_map([], |row| {
11833                    Ok((row.get::<_, String>(0)?, row.get::<_, Option<String>>(1)?))
11834                })
11835                .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
11836            rows.collect::<Result<_, _>>()
11837                .map_err(|error| match error {
11838                    rusqlite::Error::InvalidColumnType(..)
11839                    | rusqlite::Error::FromSqlConversionFailure(..) => TransportError::new(
11840                        "sync.local_corrupt",
11841                        "Pending blob upload metadata is invalid",
11842                    ),
11843                    _ => TransportError::new("client.failed", error.to_string()),
11844                })?
11845        };
11846        let mut uploads = Vec::new();
11847        for (blob_id, media_type) in pending {
11848            let bytes = (|| {
11849                let stored: Option<(SqlValue, SqlValue)> = self
11850                    .conn
11851                    .query_row(
11852                        "SELECT bytes, byte_length FROM _syncular_blobs WHERE blob_id = ?",
11853                        rusqlite::params![blob_id],
11854                        |row| Ok((row.get(0)?, row.get(1)?)),
11855                    )
11856                    .optional()
11857                    .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
11858                let Some((SqlValue::Blob(bytes), SqlValue::Integer(byte_length))) = stored else {
11859                    return Err(TransportError::new(
11860                        "sync.local_corrupt",
11861                        "Pending blob upload body is missing or corrupt",
11862                    ));
11863                };
11864                if byte_length != bytes.len() as i64 || blob_id_for(&bytes) != blob_id {
11865                    return Err(TransportError::new(
11866                        "sync.local_corrupt",
11867                        "Pending blob upload body is missing or corrupt",
11868                    ));
11869                }
11870                Ok(bytes)
11871            })();
11872            let failed = bytes.is_err();
11873            uploads.push(crate::round::PendingUpload {
11874                id: blob_id,
11875                bytes,
11876                media_type,
11877            });
11878            if failed {
11879                break;
11880            }
11881        }
11882        Ok(uploads)
11883    }
11884
11885    fn visible_blob_ids_sql(&self) -> String {
11886        let mut selects = Vec::new();
11887        for table in &self.schema.tables {
11888            for column in table
11889                .columns
11890                .iter()
11891                .filter(|column| column.ty == ColumnType::BlobRef)
11892            {
11893                let identifier = quote_ident(&column.name);
11894                selects.push(format!(
11895                    "SELECT json_extract({identifier}, '$.blobId') AS blob_id FROM {} \
11896                     WHERE typeof({identifier}) = 'text' AND json_valid({identifier}) \
11897                     AND json_type({identifier}, '$.blobId') = 'text'",
11898                    quote_ident(&table.name),
11899                ));
11900            }
11901        }
11902        if selects.is_empty() {
11903            "SELECT NULL AS blob_id WHERE 0".to_owned()
11904        } else {
11905            selects.join(" UNION ")
11906        }
11907    }
11908
11909    /// §5.9.7 B1 size cap. Visible rows, pending commits, and
11910    /// staged uploads pin bodies directly; no derived refcount is stored.
11911    fn enforce_blob_cache_cap(&self) {
11912        let Some(max_bytes) = self.limits.blob_cache_max_bytes else {
11913            return;
11914        };
11915        let mut total: i64 = self
11916            .conn
11917            .query_row(
11918                "SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs",
11919                [],
11920                |row| row.get(0),
11921            )
11922            .unwrap_or(0);
11923        if total <= max_bytes {
11924            return;
11925        }
11926        let query = format!(
11927            "SELECT blob_id, byte_length FROM _syncular_blobs
11928             WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
11929               AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
11930               AND blob_id NOT IN ({})
11931             ORDER BY created_at_ms ASC, blob_id ASC",
11932            self.visible_blob_ids_sql(),
11933        );
11934        let candidates: Vec<(String, i64)> = {
11935            let Ok(mut stmt) = self.conn.prepare(&query) else {
11936                return;
11937            };
11938            let Ok(rows) = stmt.query_map([], |row| {
11939                Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?))
11940            }) else {
11941                return;
11942            };
11943            rows.filter_map(Result::ok).collect()
11944        };
11945        for (blob_id, byte_length) in candidates {
11946            if total <= max_bytes {
11947                break;
11948            }
11949            let _ = self.conn.execute(
11950                "DELETE FROM _syncular_blobs WHERE blob_id = ?",
11951                rusqlite::params![blob_id],
11952            );
11953            total -= byte_length;
11954        }
11955    }
11956
11957    fn delete_unreferenced_cached_blobs(&mut self) -> Result<(), String> {
11958        #[cfg(feature = "bench-internals")]
11959        let _phase = self.benchmark_phases.start(Phase::BlobRetention);
11960        if !self.schema_has_blobs() {
11961            return Ok(());
11962        }
11963        self.rebuild_overlay_if_dirty()?;
11964        self.conn
11965            .execute(
11966                &format!(
11967                    "DELETE FROM _syncular_blobs
11968                 WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
11969                   AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
11970                   AND blob_id NOT IN ({})",
11971                    self.visible_blob_ids_sql(),
11972                ),
11973                [],
11974            )
11975            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11976        Ok(())
11977    }
11978
11979    // -- local row storage ----------------------------------------------------------
11980
11981    /// The cached per-table primary-key upsert SQL for `full_table` (see
11982    /// the `insert_sql` field: built once, reused per row).
11983    fn insert_row_sql(&self, full_table: &str, table: &crate::schema::TableSchema) -> String {
11984        if let Some(sql) = self.insert_sql.borrow().get(full_table) {
11985            return sql.clone();
11986        }
11987        let mut columns: Vec<String> = table.columns.iter().map(|c| quote_ident(&c.name)).collect();
11988        columns.push(quote_ident("_syncular_version"));
11989        columns.extend(table.sealed_columns().iter().map(|name| quote_ident(name)));
11990        let placeholders: Vec<&str> = columns.iter().map(|_| "?").collect();
11991        let primary_key = quote_ident(&table.primary_key);
11992        let updates = columns
11993            .iter()
11994            .filter(|column| **column != primary_key)
11995            .map(|column| format!("{column}=excluded.{column}"))
11996            .collect::<Vec<_>>()
11997            .join(", ");
11998        // §4.5 version guard: server state lands in the base table only above
11999        // the stored version; the visible overlay takes every write.
12000        let guard = if full_table == base_table(&table.name) {
12001            format!(" WHERE excluded._syncular_version > {full_table}._syncular_version")
12002        } else {
12003            String::new()
12004        };
12005        let sql = format!(
12006            "INSERT INTO {full_table} ({}) VALUES ({}) ON CONFLICT ({primary_key}) DO UPDATE SET {updates}{guard}",
12007            columns.join(", "),
12008            placeholders.join(", ")
12009        );
12010        self.insert_sql
12011            .borrow_mut()
12012            .insert(full_table.to_owned(), sql.clone());
12013        sql
12014    }
12015
12016    /// §4.5: write a server row under the version guard; returns whether it
12017    /// landed. `sealed` is its §5.11 sealed state, `None` for an unsealed row.
12018    fn write_base_row(
12019        &self,
12020        table_name: &str,
12021        row: &Row,
12022        version: i64,
12023        sealed: Option<&SealedRow>,
12024    ) -> Result<bool, String> {
12025        let landed = self.write_row(&base_table(table_name), table_name, row, version, sealed)?;
12026        if landed {
12027            self.overlay_dirty.table(table_name);
12028        }
12029        Ok(landed)
12030    }
12031
12032    fn write_row(
12033        &self,
12034        full_table: &str,
12035        table_name: &str,
12036        row: &Row,
12037        version: i64,
12038        sealed: Option<&SealedRow>,
12039    ) -> Result<bool, String> {
12040        #[cfg(feature = "bench-internals")]
12041        let _phase = self.benchmark_phases.start(Phase::RowWrite);
12042        let table = self
12043            .schema
12044            .table(table_name)
12045            .ok_or_else(|| format!("unknown table {table_name:?}"))?;
12046        let sql = self.insert_row_sql(full_table, table);
12047        let mut stmt = self
12048            .conn
12049            .prepare_cached(&sql)
12050            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12051        let params = row
12052            .iter()
12053            .map(RowParam::Cell)
12054            .chain(std::iter::once(RowParam::Version(version)))
12055            .chain(
12056                [
12057                    sealed.map(|sealed| sealed.key_id.as_str()),
12058                    sealed.map(|sealed| sealed.envelopes.as_str()),
12059                ]
12060                .into_iter()
12061                .take(table.sealed_columns().len())
12062                .map(RowParam::Text),
12063            );
12064        stmt.execute(rusqlite::params_from_iter(params))
12065            .map(|changed| changed > 0)
12066            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
12067    }
12068
12069    /// §4.5: delete a server row whose stored version lies below `version`;
12070    /// returns whether a row left.
12071    fn delete_base_row(
12072        &self,
12073        table_name: &str,
12074        row_id: &str,
12075        version: i64,
12076    ) -> Result<bool, String> {
12077        let table = self
12078            .schema
12079            .table(table_name)
12080            .ok_or_else(|| format!("unknown table {table_name:?}"))?;
12081        let sql = format!(
12082            "DELETE FROM {} WHERE {} AND _syncular_version < ?2",
12083            base_table(table_name),
12084            row_id_predicate(table)
12085        );
12086        let mut stmt = self
12087            .conn
12088            .prepare_cached(&sql)
12089            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12090        let landed = stmt
12091            .execute(rusqlite::params![row_id, version])
12092            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
12093            > 0;
12094        if landed {
12095            self.overlay_dirty.table(table_name);
12096        }
12097        Ok(landed)
12098    }
12099
12100    fn has_acknowledged_rows(&self) -> Result<bool, String> {
12101        self.conn
12102            .query_row(
12103                "SELECT EXISTS(SELECT 1 FROM _syncular_acked_rows)",
12104                [],
12105                |row| row.get(0),
12106            )
12107            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
12108    }
12109
12110    fn acknowledged_operations(
12111        &self,
12112        rows: Option<&HashSet<(&str, &str)>>,
12113        tables: Option<&BTreeSet<&str>>,
12114    ) -> Result<Vec<OutboxOp>, String> {
12115        let keys = rows.map(|rows| serde_json::to_string(&rows).expect("row keys"));
12116        let mut statement = self
12117            .conn
12118            .prepare_cached(if keys.is_some() {
12119                ACKED_ROWS_BY_ROW
12120            } else {
12121                "SELECT op_json,intent_json FROM _syncular_acked_rows WHERE ?1 IS NULL AND (?2 IS NULL OR tbl IN(SELECT value FROM json_each(?2))) ORDER BY commit_seq,idx"
12122            })
12123            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12124        let rows = statement
12125            .query_map(
12126                rusqlite::params![
12127                    keys,
12128                    tables.map(|tables| serde_json::to_string(tables).expect("table keys"))
12129                ],
12130                |row| {
12131                    #[cfg(test)]
12132                    self.acknowledged_replay_count
12133                        .set(self.acknowledged_replay_count.get() + 1);
12134                    Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
12135                },
12136            )
12137            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12138        rows.map(|row| {
12139            let (op, intent) =
12140                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12141            let op: CommitOperation =
12142                serde_json::from_str(&op).map_err(|_| "sync.local_corrupt".to_owned())?;
12143            let table = self
12144                .schema
12145                .table(&op.table)
12146                .ok_or_else(|| "sync.unknown_table".to_owned())?;
12147            let values = if op.op == "upsert" && self.visible_row(table, &op.row_id)?.is_none() {
12148                Some(serde_json::from_str(&intent).map_err(|_| "sync.local_corrupt".to_owned())?)
12149            } else {
12150                op.values.clone()
12151            };
12152            Ok(OutboxOp {
12153                values,
12154                ..op.into()
12155            })
12156        })
12157        .collect()
12158    }
12159
12160    fn drop_acknowledged_rows(
12161        &self,
12162        matches: impl Fn(&str, &Map<String, Value>) -> bool,
12163    ) -> Result<Vec<OutboxCommit>, String> {
12164        let mut statement = self
12165            .conn
12166            .prepare("SELECT commit_id,tbl,intent_json,id FROM _syncular_acked_rows")
12167            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12168        let rows = statement
12169            .query_map([], |row| {
12170                Ok((
12171                    row.get::<_, String>(0)?,
12172                    row.get::<_, String>(1)?,
12173                    row.get::<_, String>(2)?,
12174                    row.get::<_, String>(3)?,
12175                ))
12176            })
12177            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12178        let mut doomed = HashSet::new();
12179        for row in rows {
12180            let (id, table, intent, row_id) =
12181                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12182            let values =
12183                serde_json::from_str(&intent).map_err(|_| "sync.local_corrupt".to_owned())?;
12184            let base_matches = self
12185                .schema
12186                .table(&table)
12187                .map(|schema| self.stored_row(schema, &row_id, true))
12188                .transpose()?
12189                .flatten()
12190                .is_some_and(|(base, _)| {
12191                    let schema = self.schema.table(&table).expect("acknowledged table");
12192                    let values = schema
12193                        .columns
12194                        .iter()
12195                        .zip(base.iter())
12196                        .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
12197                        .collect();
12198                    matches(&table, &values)
12199                });
12200            if matches(&table, &values) || base_matches {
12201                doomed.insert(id);
12202            }
12203        }
12204        let mut dropped = Vec::new();
12205        for id in doomed {
12206            let mut statement = self
12207                .conn
12208                .prepare("SELECT op_json FROM _syncular_acked_rows WHERE commit_id=?1 ORDER BY idx")
12209                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12210            let operations = statement
12211                .query_map([&id], |row| row.get::<_, String>(0))
12212                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12213            let mut ops = Vec::new();
12214            for operation in operations {
12215                let json = operation
12216                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12217                let op: CommitOperation =
12218                    serde_json::from_str(&json).map_err(|_| "sync.local_corrupt".to_owned())?;
12219                self.overlay_dirty.table(&op.table);
12220                ops.push(OutboxOp {
12221                    table: op.table,
12222                    row_id: op.row_id,
12223                    upsert: op.op == "upsert",
12224                    base_version: op.base_version,
12225                    values: op.values,
12226                    insert: op.insert,
12227                });
12228            }
12229            dropped.push(OutboxCommit {
12230                client_commit_id: id.clone(),
12231                ops,
12232            });
12233            self.conn
12234                .execute("DELETE FROM _syncular_acked_rows WHERE commit_id=?1", [id])
12235                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12236        }
12237        Ok(dropped)
12238    }
12239
12240    /// [`Self::rebuild_overlay`], skipped when neither the base tables nor
12241    /// the outbox changed since the last rebuild (the no-op sync round).
12242    fn rebuild_overlay_if_dirty(&mut self) -> Result<(), String> {
12243        if self.overlay_dirty.get() {
12244            self.rebuild_overlay()?;
12245        }
12246        Ok(())
12247    }
12248
12249    /// §7.1: local reads see outbox state applied optimistically — rebuild
12250    /// every visible table as (base server state) + (pending outbox replay
12251    /// on top). Optimistic rows carry version `-1`.
12252    fn rebuild_overlay(&mut self) -> Result<(), String> {
12253        #[cfg(feature = "bench-internals")]
12254        let _phase = self.benchmark_phases.start(Phase::OverlayRebuild);
12255        #[cfg(test)]
12256        self.overlay_rebuild_count
12257            .set(self.overlay_rebuild_count.get() + 1);
12258        let dirty = self.overlay_dirty.snapshot();
12259        let tables: Vec<_> = self
12260            .schema
12261            .tables
12262            .iter()
12263            .filter(|table| {
12264                dirty
12265                    .as_ref()
12266                    .is_none_or(|names| names.is_empty() || names.contains(&table.name))
12267            })
12268            .cloned()
12269            .collect();
12270        let names: BTreeSet<_> = tables.iter().map(|table| table.name.as_str()).collect();
12271        self.begin_observation("syncular_overlay")?;
12272        let rebuilt = (|| {
12273            for table in &tables {
12274                for index in &table.fts_indexes {
12275                    self.drop_fts_triggers(index)?;
12276                }
12277                let visible = visible_table(&table.name);
12278                let base = base_table(&table.name);
12279                self.conn
12280                    .execute_batch(&format!(
12281                        "DELETE FROM {visible}; INSERT INTO {visible} SELECT * FROM {base}"
12282                    ))
12283                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12284            }
12285            self.replay_overlay(&names, None)?;
12286            for table in &tables {
12287                for index in &table.fts_indexes {
12288                    self.rebuild_fts_projection(table, index)?;
12289                    self.create_fts_triggers(table, index)?;
12290                }
12291            }
12292            self.conn
12293                .execute_batch("RELEASE syncular_overlay")
12294                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12295            Ok(())
12296        })();
12297        if rebuilt.is_err() {
12298            self.conn
12299                .execute_batch("ROLLBACK TO syncular_overlay; RELEASE syncular_overlay")
12300                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12301        } else {
12302            self.overlay_dirty.set(false);
12303        }
12304        rebuilt
12305    }
12306
12307    /// §7.1 replay of acknowledged, retained failed and pending intent, in
12308    /// that order, for the operations of `tables` on `rows` (every row for
12309    /// `None`) over their current visible state.
12310    fn replay_overlay(
12311        &self,
12312        tables: &BTreeSet<&str>,
12313        rows: Option<&HashSet<(&str, &str)>>,
12314    ) -> Result<(), String> {
12315        let replays = |op: &&OutboxOp| {
12316            tables.contains(op.table.as_str())
12317                && rows.is_none_or(|rows| rows.contains(&(op.table.as_str(), op.row_id.as_str())))
12318        };
12319        self.apply_outbox_ops(&self.acknowledged_operations(rows, Some(tables))?)?;
12320        for (commit, initial) in self.failed_commits.iter() {
12321            for operation in commit.ops.iter().filter(replays) {
12322                let Some(table) = self.schema.table(&operation.table) else {
12323                    continue;
12324                };
12325                if operation.upsert
12326                    && self.visible_row(table, &operation.row_id)?.is_none()
12327                    && operation.presented_values(table).is_some_and(|values| {
12328                        table
12329                            .columns
12330                            .iter()
12331                            .all(|column| values.contains_key(&column.name))
12332                    })
12333                {
12334                    let key = serde_json::to_string(&(&operation.table, &operation.row_id))
12335                        .expect("row key serialization");
12336                    if let Some(Value::Object(values)) = initial.get(&key) {
12337                        self.apply_outbox_ops(&[OutboxOp {
12338                            values: Some(values.clone()),
12339                            ..operation.clone()
12340                        }])?;
12341                    }
12342                }
12343            }
12344            self.apply_outbox_ops(commit.ops.iter().filter(replays))?;
12345        }
12346        self.apply_outbox_ops(
12347            self.outbox
12348                .iter()
12349                .flat_map(|commit| &commit.ops)
12350                .filter(replays),
12351        )
12352    }
12353
12354    /// §7.1 replay: every operation is applied over the current visible
12355    /// state. An operation that no longer applies (a server row now holds its
12356    /// unique value) stays pending and invisible until a later replay admits
12357    /// it or the server answers its push.
12358    fn apply_outbox_ops<'a>(
12359        &self,
12360        ops: impl IntoIterator<Item = &'a OutboxOp>,
12361    ) -> Result<(), String> {
12362        #[cfg(feature = "bench-internals")]
12363        let mut phase = self.benchmark_phases.start(Phase::PendingReplay);
12364        for op in ops {
12365            #[cfg(feature = "bench-internals")]
12366            if let Some(phase) = &mut phase {
12367                phase.unit();
12368            }
12369            if op.upsert {
12370                if let Some((table, values)) = self
12371                    .schema
12372                    .table(&op.table)
12373                    .filter(|table| table.indexes.iter().any(|index| index.unique))
12374                    .and_then(|table| Some((table, op.presented_values(table)?)))
12375                {
12376                    let mut intended: Map<String, Value> = self
12377                        .visible_row(table, &op.row_id)?
12378                        .map(|(row, _)| {
12379                            table
12380                                .columns
12381                                .iter()
12382                                .zip(&row)
12383                                .map(|(column, value)| {
12384                                    (column.name.clone(), column_value_to_json(value))
12385                                })
12386                                .collect()
12387                        })
12388                        .unwrap_or_default();
12389                    intended.extend(values.into_owned());
12390                    if !self.unique_conflicts(table, &intended, false)?.is_empty() {
12391                        continue;
12392                    }
12393                }
12394            }
12395            self.apply_outbox_op(op)?;
12396        }
12397        Ok(())
12398    }
12399
12400    fn apply_outbox_op(&self, op: &OutboxOp) -> Result<(), String> {
12401        let Some(table) = self.schema.table(&op.table) else {
12402            return Ok(());
12403        };
12404        let visible = visible_table(&table.name);
12405        if !op.upsert {
12406            let sql = format!("DELETE FROM {visible} WHERE {}", row_id_predicate(table));
12407            self.conn
12408                .prepare_cached(&sql)
12409                .and_then(|mut statement| statement.execute(rusqlite::params![op.row_id]))
12410                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12411            return Ok(());
12412        }
12413        // §7.1: an insert-intent operation that lacks a non-nullable column
12414        // cannot encode; it stays invisible until the send-time prepass drops
12415        // it as sync.outbox_incompatible (§7.4.4).
12416        let Some(values) = op.presented_values(table) else {
12417            return Ok(());
12418        };
12419        // §7.1: the overlay applies the operation's PRESENT columns
12420        // over the current local row. A partial operation over an
12421        // absent local row leaves it absent — the server answers it
12422        // with sync.row_deleted or sync.row_missing (§6.2). An
12423        // insert-intent operation presents every column.
12424        let full = table
12425            .columns
12426            .iter()
12427            .all(|column| values.contains_key(&column.name));
12428        let local = self.visible_row(table, &op.row_id)?;
12429        if local.is_none() && !full {
12430            return Ok(());
12431        }
12432        let mut row: Row = Vec::with_capacity(table.columns.len());
12433        for (index, column) in table.columns.iter().enumerate() {
12434            let incoming = match values.get(&column.name) {
12435                Some(raw) => json_to_column_value(
12436                    if raw.get("$bytes").is_some()
12437                        && table
12438                            .encrypted_columns
12439                            .iter()
12440                            .any(|encrypted| encrypted.index == index)
12441                    {
12442                        &table.wire_columns[index]
12443                    } else {
12444                        column
12445                    },
12446                    Some(raw),
12447                ),
12448                None => Ok(local
12449                    .as_ref()
12450                    .and_then(|(values, _)| values.get(index).cloned())
12451                    .flatten()),
12452            };
12453            row.push(incoming.map_err(|_| {
12454                "sync.local_corrupt: persisted optimistic row value is invalid".to_owned()
12455            })?);
12456        }
12457        let version = local.as_ref().map_or(-1, |(_, version)| *version);
12458        self.write_row(&visible, &table.name, &row, version, None)
12459            .map(|_| ())
12460            .map_err(|error| {
12461                if error.starts_with("UNIQUE constraint failed:") {
12462                    "sync.constraint_violation: local write violates a unique constraint".to_owned()
12463                } else {
12464                    error
12465                }
12466            })
12467    }
12468
12469    /// Whether the row is present in the overlay. Existence reads only the
12470    /// key: locked encrypted values stay opaque, and decoding the complete row
12471    /// would reject a valid plain write.
12472    fn visible_row_exists(&self, table: &TableSchema, row_id: &str) -> Result<bool, String> {
12473        self.conn
12474            .query_row(
12475                &format!(
12476                    "SELECT EXISTS(SELECT 1 FROM {} WHERE {})",
12477                    visible_table(&table.name),
12478                    row_id_predicate(table)
12479                ),
12480                rusqlite::params![row_id],
12481                |row| row.get(0),
12482            )
12483            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
12484    }
12485
12486    /// §5.11: a sealed row is read-only; a write could otherwise send NULL
12487    /// over ciphertext this device cannot see.
12488    fn refuse_sealed_row(&self, table: &TableSchema, row_id: &str) -> Result<(), ClientError> {
12489        if !table.has_encrypted_columns() {
12490            return Ok(());
12491        }
12492        let key_id: Option<String> = self
12493            .conn
12494            .query_row(
12495                &format!(
12496                    "SELECT \"_sync_sealed_key\" FROM {} WHERE {}",
12497                    visible_table(&table.name),
12498                    row_id_predicate(table)
12499                ),
12500                rusqlite::params![row_id],
12501                |row| row.get(0),
12502            )
12503            .optional()
12504            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
12505            .flatten();
12506        match key_id {
12507            None => Ok(()),
12508            Some(key_id) => Err(ClientError {
12509                code: "client.encryption_key_missing".to_owned(),
12510                message: "the row is sealed under a key this device does not hold".to_owned(),
12511                details: Some(json!({ "table": table.name, "keyId": key_id })),
12512                retryable: false,
12513            }),
12514        }
12515    }
12516
12517    /// §5.11 reveal: decrypt every sealed base row whose envelope keys the
12518    /// keyring now holds into its columns and clear both reserved columns,
12519    /// one change batch per table. `client.decrypt_failed` fails the caller.
12520    fn reveal_sealed_rows(&mut self) -> Result<(), TransportError> {
12521        let storage = |message: String| TransportError::new("storage.failed", message);
12522        let tables: Vec<TableSchema> = self
12523            .schema
12524            .tables
12525            .iter()
12526            .filter(|table| table.has_encrypted_columns())
12527            .cloned()
12528            .collect();
12529        for table in &tables {
12530            let base = base_table(&table.name);
12531            let pk = quote_ident(&table.primary_key);
12532            let sealed: Vec<(SqlValue, i64, String)> = self
12533                .conn
12534                .prepare_cached(&format!(
12535                    "SELECT {pk}, \"_syncular_version\", \"_sync_sealed\" FROM {base} WHERE \"_sync_sealed_key\" IS NOT NULL"
12536                ))
12537                .and_then(|mut statement| {
12538                    statement
12539                        .query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))?
12540                        .collect()
12541                })
12542                .map_err(|error| storage(Self::sqlite_failure(&self.storage_failure, error)))?;
12543            let mut revealed = Vec::new();
12544            for (key, version, envelopes) in sealed {
12545                if let Some(row) =
12546                    crate::values::reveal_sealed_row(table, &envelopes, &self.encryption)?
12547                {
12548                    revealed.push((key, version, row));
12549                }
12550            }
12551            if revealed.is_empty() {
12552                continue;
12553            }
12554            self.begin_observation("syncular_reveal").map_err(storage)?;
12555            let outcome = (|| {
12556                let assignments = table
12557                    .encrypted_columns
12558                    .iter()
12559                    .map(|enc| format!("{}=?", quote_ident(&table.columns[enc.index].name)))
12560                    .collect::<Vec<_>>()
12561                    .join(",");
12562                let mut statement = self
12563                    .conn
12564                    .prepare_cached(&format!(
12565                        "UPDATE {base} SET {assignments},\"_sync_sealed_key\"=NULL,\"_sync_sealed\"=NULL WHERE {pk}=? AND \"_syncular_version\"=?"
12566                    ))
12567                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12568                for (key, version, row) in &revealed {
12569                    let mut params = table
12570                        .encrypted_columns
12571                        .iter()
12572                        .map(|enc| owned_sql_value(RowParam::Cell(&row[enc.index])))
12573                        .collect::<Result<Vec<_>, _>>()?;
12574                    params.push(key.clone());
12575                    params.push(SqlValue::Integer(*version));
12576                    statement
12577                        .execute(rusqlite::params_from_iter(params))
12578                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12579                }
12580                drop(statement);
12581                self.overlay_dirty.table(&table.name);
12582                self.rebuild_overlay()?;
12583                let mut batch = ChangeAccumulator::default();
12584                batch.table(&table.name);
12585                self.finish_observation("syncular_reveal", batch)
12586            })();
12587            if outcome.is_err() {
12588                self.rollback_observation("syncular_reveal");
12589            }
12590            outcome.map_err(storage)?;
12591        }
12592        Ok(())
12593    }
12594
12595    /// The visible (optimistic) row's values plus version, or `None` when the
12596    /// row is absent from the overlay.
12597    fn visible_row(&self, table: &TableSchema, row_id: &str) -> Result<Option<(Row, i64)>, String> {
12598        self.stored_row(table, row_id, false)
12599    }
12600
12601    fn stored_row(
12602        &self,
12603        table: &TableSchema,
12604        row_id: &str,
12605        base: bool,
12606    ) -> Result<Option<(Row, i64)>, String> {
12607        let sql = format!(
12608            "SELECT * FROM {} WHERE {}",
12609            if base {
12610                base_table(&table.name)
12611            } else {
12612                visible_table(&table.name)
12613            },
12614            row_id_predicate(table)
12615        );
12616        let mut stmt = self
12617            .conn
12618            .prepare_cached(&sql)
12619            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12620        let mut rows = stmt
12621            .query(rusqlite::params![row_id])
12622            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12623        let Some(row) = rows
12624            .next()
12625            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
12626        else {
12627            return Ok(None);
12628        };
12629        let mut values = Vec::with_capacity(table.columns.len());
12630        for (index, column) in table.columns.iter().enumerate() {
12631            let raw = row
12632                .get_ref(index)
12633                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12634            if let rusqlite::types::ValueRef::Blob(bytes) = raw {
12635                if table
12636                    .encrypted_columns
12637                    .iter()
12638                    .any(|encrypted| encrypted.index == index)
12639                {
12640                    values.push(Some(ColumnValue::Bytes(bytes.to_vec())));
12641                    continue;
12642                }
12643            }
12644            let json = sql_ref_to_json(column, raw);
12645            values.push(json_to_column_value(column, Some(&json)).map_err(|_| {
12646                "sync.local_corrupt: persisted local row value is invalid".to_owned()
12647            })?);
12648        }
12649        let version: i64 = row
12650            .get(table.columns.len())
12651            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12652        Ok(Some((values, version)))
12653    }
12654
12655    // -- realtime (§8) ---------------------------------------------------------------
12656
12657    /// §8.8: the configured realtime policy.
12658    #[must_use]
12659    pub fn realtime_policy(&self) -> RealtimePolicy {
12660        self.realtime_policy
12661    }
12662
12663    /// §8.8: the explicit realtime availability state.
12664    #[must_use]
12665    pub fn realtime_state(&self) -> RealtimeState {
12666        self.realtime_state
12667    }
12668
12669    /// §8.8: set the realtime policy. Refused while the socket is connected:
12670    /// a policy change never silently re-routes an ownership state.
12671    pub fn set_realtime_policy(&mut self, policy: RealtimePolicy) -> Result<(), String> {
12672        if self.realtime_state == RealtimeState::Connected && policy != self.realtime_policy {
12673            return Err(
12674                "sync.invalid_request: disconnect realtime before changing the realtime policy"
12675                    .to_owned(),
12676            );
12677        }
12678        self.realtime_policy = policy;
12679        match policy {
12680            RealtimePolicy::Off => self.set_realtime_state(RealtimeState::Disabled, None),
12681            RealtimePolicy::Required | RealtimePolicy::Optional
12682                if self.realtime_state == RealtimeState::Disabled =>
12683            {
12684                self.set_realtime_state(RealtimeState::Disconnected, None);
12685            }
12686            RealtimePolicy::Required | RealtimePolicy::Optional => {}
12687        }
12688        Ok(())
12689    }
12690
12691    /// Record an availability transition, keeping the reason code bounded and
12692    /// code-like (§7.6) and clearing it where it cannot apply.
12693    fn set_realtime_state(&mut self, state: RealtimeState, reason_code: Option<&str>) {
12694        self.realtime_state = state;
12695        self.realtime_reason_code = reason_code.map(Self::diagnostic_code);
12696        if matches!(state, RealtimeState::Connected | RealtimeState::Disabled) {
12697            self.realtime_reason_code = None;
12698            self.realtime_retry_delay_ms = None;
12699        }
12700    }
12701
12702    #[must_use]
12703    fn realtime_connected(&self) -> bool {
12704        self.realtime_state == RealtimeState::Connected
12705    }
12706
12707    pub fn connect_realtime(&mut self, transport: &mut dyn Transport) -> Result<(), String> {
12708        if !self.transport_enabled {
12709            return Err("sync.offline: transport gate is closed".into());
12710        }
12711        if self.realtime_policy == RealtimePolicy::Off {
12712            return Err(
12713                "sync.invalid_request: realtime is off; realtime connect is refused".to_owned(),
12714            );
12715        }
12716        if self.realtime_connected() {
12717            return Ok(());
12718        }
12719        self.realtime_state = RealtimeState::Connecting;
12720        match transport.realtime_connect(&self.client_id) {
12721            Ok(()) => {
12722                self.set_realtime_state(RealtimeState::Connected, None);
12723                Ok(())
12724            }
12725            Err(error) => {
12726                self.set_realtime_state(RealtimeState::Refused, Some(&error.code));
12727                Err(format!("{}: {}", error.code, error.message))
12728            }
12729        }
12730    }
12731
12732    pub fn disconnect_realtime(&mut self, transport: &mut dyn Transport) {
12733        self.cancel_sync_round();
12734        // A deliberately disconnected or policy-disabled client has nothing
12735        // to release; a `lost` socket still does (the host owns the handle).
12736        if self.realtime_state == RealtimeState::Disconnected
12737            || self.realtime_state == RealtimeState::Disabled
12738        {
12739            return;
12740        }
12741        let _ = transport.realtime_close();
12742        self.set_realtime_state(
12743            if self.realtime_policy == RealtimePolicy::Off {
12744                RealtimeState::Disabled
12745            } else {
12746                RealtimeState::Disconnected
12747            },
12748            None,
12749        );
12750        self.presence.clear(); // §8.6.1: presence is per-connection
12751    }
12752
12753    /// §8.8: the host observed the socket end without disconnecting it (a
12754    /// peer close, an I/O error, a failed send; the native transport reports
12755    /// these as `Inbound::Disconnected`). Releases the transport, moves a
12756    /// `Connected` state to `Lost` with the transport's code as the reason, and schedules
12757    /// one background retry intent so the host reconnects and catches up.
12758    pub fn realtime_lost(&mut self, transport: &mut dyn Transport, error: &TransportError) {
12759        if self.realtime_state != RealtimeState::Connected {
12760            return;
12761        }
12762        self.cancel_sync_round();
12763        let _ = transport.realtime_close();
12764        self.set_realtime_state(RealtimeState::Lost, Some(&error.code));
12765        self.realtime_retry_delay_ms = Some(self.schedule_background_retry());
12766        self.presence.clear();
12767    }
12768
12769    /// §8.6.2: publish (or clear, `doc: None`) this client's presence
12770    /// document for `scope_key`. Requires a live socket; the document is
12771    /// ephemeral (lost on disconnect). Authorization is the connection's
12772    /// registration (§8.6.3) — an unheld key is rejected loudly by the
12773    /// server with `presence.forbidden`.
12774    pub fn set_presence(
12775        &mut self,
12776        transport: &mut dyn Transport,
12777        scope_key: &str,
12778        doc: Option<&Value>,
12779    ) -> Result<(), String> {
12780        if !self.transport_enabled {
12781            return Err("sync.offline: transport gate is closed".into());
12782        }
12783        if !self.realtime_connected() {
12784            return Err("setPresence requires a connected realtime socket (§8.6)".to_string());
12785        }
12786        let text = encode_presence_publish(scope_key, doc);
12787        transport
12788            .realtime_send(&text)
12789            .map_err(|e| format!("{}: {}", e.code, e.message))
12790    }
12791
12792    /// §8.6: the peers currently present on a scope key (ephemeral).
12793    pub fn presence(&self, scope_key: &str) -> Vec<PresencePeer> {
12794        self.presence
12795            .get(scope_key)
12796            .map(|peers| peers.values().cloned().collect())
12797            .unwrap_or_default()
12798    }
12799
12800    /// §8.6 apply an inbound presence fanout to the local map.
12801    fn apply_presence(
12802        &mut self,
12803        scope_key: String,
12804        kind: Option<PresenceKind>,
12805        actor_id: Option<String>,
12806        client_id: Option<String>,
12807        doc: Option<Value>,
12808        error: Option<String>,
12809    ) {
12810        // A refused publish (§8.6.2) changes no peer; the native host surfaces
12811        // it as a `presence` event carrying the server's `error` code.
12812        if error.is_some() {
12813            return;
12814        }
12815        let (Some(kind), Some(actor_id), Some(client_id)) = (kind, actor_id, client_id) else {
12816            return;
12817        };
12818        let peer_key = format!("{actor_id} {client_id}");
12819        match kind {
12820            PresenceKind::Leave => {
12821                if let Some(peers) = self.presence.get_mut(&scope_key) {
12822                    peers.remove(&peer_key);
12823                    if peers.is_empty() {
12824                        self.presence.remove(&scope_key);
12825                    }
12826                }
12827            }
12828            _ => {
12829                let doc = match doc {
12830                    Some(Value::Object(_)) => doc.unwrap(),
12831                    _ => return,
12832                };
12833                self.presence.entry(scope_key).or_default().insert(
12834                    peer_key,
12835                    PresencePeer {
12836                        actor_id,
12837                        client_id,
12838                        doc,
12839                    },
12840                );
12841            }
12842        }
12843    }
12844
12845    /// Inbound JSON control message (§8.1). Unknown events are tolerated.
12846    pub fn on_realtime_text(&mut self, text: &str) {
12847        if self.stopped || !self.transport_enabled || self.security_preflight {
12848            return;
12849        }
12850        match parse_control(text) {
12851            Ok(ControlMessage::Hello { requires_sync, .. }) => {
12852                if requires_sync {
12853                    // §8.1: pull before trusting the socket for continuity.
12854                    self.set_sync_needed(true, true);
12855                }
12856            }
12857            Ok(ControlMessage::Presence {
12858                scope_key,
12859                kind,
12860                actor_id,
12861                client_id,
12862                doc,
12863                error,
12864                ..
12865            }) => {
12866                self.apply_presence(scope_key, kind, actor_id, client_id, doc, error);
12867            }
12868            Ok(ControlMessage::Wake { .. }) => {
12869                // §8.3: any wake-up means "run a pull soon", never data.
12870                self.set_sync_needed(true, true);
12871            }
12872            Ok(ControlMessage::Scopes { .. }) => {
12873                // §8.7: surfaced to the app; no sync-needed, no round.
12874                self.scopes_changed += 1;
12875            }
12876            _ => {}
12877        }
12878    }
12879
12880    /// Inbound binary delta: a complete SSP2 response (§8.2), applied like
12881    /// a pull response per section; an unapplied delta is a wake-up.
12882    /// Mailbox hosts send durable acknowledgements on their I/O executor.
12883    pub fn on_realtime_binary_queued(&mut self, bytes: &[u8]) -> Vec<String> {
12884        let mut transport = crate::round::DownloadResults::default();
12885        self.on_realtime_binary(&mut transport, bytes);
12886        transport.controls
12887    }
12888
12889    /// §8.2: a delta applies exactly like a live section and sets the replica
12890    /// cursor to its `throughSeq`; one at or below the replica cursor is a
12891    /// duplicate. A delta the client does not apply (no live unit, a status
12892    /// other than `active`, a decode or apply failure) is a drop and raises
12893    /// sync-needed.
12894    pub fn on_realtime_binary(&mut self, transport: &mut dyn Transport, bytes: &[u8]) {
12895        if self.stopped || !self.transport_enabled || self.security_preflight {
12896            return;
12897        }
12898        #[cfg(feature = "bench-internals")]
12899        let decode_phase = self.benchmark_phases.start(Phase::ResponseDecode);
12900        let message = match decode_message(bytes) {
12901            Ok(m) if m.msg_kind == MsgKind::Response => m,
12902            _ => {
12903                self.set_sync_needed(true, true);
12904                return;
12905            }
12906        };
12907        #[cfg(feature = "bench-internals")]
12908        drop(decode_phase);
12909        // §8.2: a delta's header means what a pull response's header means. A
12910        // schema floor (§1.6) is recorded, stops syncing and applies nothing.
12911        if let Some(Frame::RespHeader {
12912            required_schema_version: Some(required),
12913            latest_schema_version,
12914            ..
12915        }) = message.frames.first()
12916        {
12917            let floor = SchemaFloor {
12918                required_schema_version: Some(*required),
12919                latest_schema_version: *latest_schema_version,
12920            };
12921            if self.set_schema_floor(Some(floor)).is_err() {
12922                self.set_sync_needed(true, true);
12923            }
12924            return;
12925        }
12926        let through = message.frames.iter().find_map(|frame| match frame {
12927            Frame::LiveEnd { through_seq, .. } => Some(*through_seq),
12928            _ => None,
12929        });
12930        let epoch = self.get_meta(LOG_EPOCH_KEY).ok().flatten();
12931        let mut dropped = !matches!(message.frames.first(), Some(Frame::RespHeader {
12932                reset_required: false,
12933                log_epoch,
12934                ..
12935            }) if Some(log_epoch) == epoch.as_ref());
12936        let mut applied: Option<i64> = None;
12937        dropped |= self.begin_coalesced_apply().is_err();
12938        let mut receivers: Option<Vec<UnitRecord>> = None;
12939        for frame in message.frames {
12940            if dropped {
12941                break;
12942            }
12943            match frame {
12944                Frame::LiveStart { status, .. } => {
12945                    let Ok(cursor) = self.stored(replica_cursor(&self.conn)) else {
12946                        dropped = true;
12947                        break;
12948                    };
12949                    if status != LiveStatus::Active || cursor < 0 {
12950                        dropped = true;
12951                    } else if !through.is_some_and(|through| through <= cursor) {
12952                        // A catching-up unit receives no delta (§8.2).
12953                        receivers = self.stored(load_units(&self.conn)).ok().map(|units| {
12954                            units
12955                                .into_iter()
12956                                .filter(|unit| {
12957                                    unit.status == UnitStatus::Active
12958                                        && unit.echo.is_some()
12959                                        && unit.cursor.is_none()
12960                                })
12961                                .collect()
12962                        });
12963                        dropped = receivers.is_none();
12964                    }
12965                }
12966                Frame::Commit {
12967                    commit_seq,
12968                    tables,
12969                    changes,
12970                    ..
12971                } => {
12972                    let Some(receivers) = receivers.clone() else {
12973                        continue;
12974                    };
12975                    dropped = self
12976                        .apply_live_commit(
12977                            &receivers,
12978                            &tables,
12979                            changes,
12980                            commit_seq,
12981                            AppliedServerCommitSource::Delta,
12982                        )
12983                        .is_err();
12984                }
12985                Frame::LiveEnd { through_seq, .. } if receivers.is_some() => {
12986                    dropped = self
12987                        .set_meta(REPLICA_CURSOR_KEY, &through_seq.to_string())
12988                        .is_err();
12989                    applied = Some(through_seq);
12990                }
12991                Frame::Error { .. } => dropped = true,
12992                _ => {}
12993            }
12994        }
12995        // §1.4 rule 4: the ack follows the COMMIT. A failed COMMIT restored
12996        // the replica cursor, so the delta counts as a drop.
12997        if self.finish_coalesced_apply().is_err() || dropped {
12998            applied = None;
12999            dropped = true;
13000        }
13001        if let Some(cursor) = applied {
13002            // §8.2 ack point: the delta's throughSeq.
13003            let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
13004            let _ = transport.realtime_send(&ack);
13005        }
13006        if dropped {
13007            self.set_sync_needed(true, true);
13008        }
13009    }
13010
13011    /// §8.2 ack point after a pull round on a live connection: the replica
13012    /// cursor while it is at least `0`.
13013    fn ack_after_pull(&mut self, transport: &mut dyn Transport) {
13014        if !self.realtime_connected() {
13015            return;
13016        }
13017        if let Ok(cursor @ 0..) = self.stored(replica_cursor(&self.conn)) {
13018            let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
13019            let _ = transport.realtime_send(&ack);
13020        }
13021    }
13022}
13023
13024#[cfg(test)]
13025mod previous_version_wiring_tests;
13026
13027#[cfg(test)]
13028mod identity_tests;
13029
13030#[cfg(test)]
13031mod insert_intent_tests;
13032
13033#[cfg(test)]
13034mod robustness_tests;
13035
13036#[cfg(test)]
13037mod range_image_tests;