Skip to main content

syncular_client/
client.rs

1//! The Syncular v2 Rust client core (SPEC.md client-behavior contract):
2//! rusqlite local storage, §3.2/§3.3 effective-scope persistence + purge,
3//! §4 pull/cursor/bootstrap (§4.7 resume, §5.6 segment application), §6
4//! push with outbox order, §7 optimistic apply / rollback / replay-on-top,
5//! §2.3 clientCommitId idempotency, §8 realtime client rules, §10 errors.
6//!
7//! Built from `SPEC.md` and the committed `ssp2` codec alone — no
8//! reference to the v1 Rust tree or the v2 TypeScript client.
9
10use crate::{ProgressObserver, ProgressPhase, ProgressState};
11use ssp2::encode_message;
12use std::borrow::Cow;
13#[cfg(test)]
14use std::cell::Cell;
15use std::cell::RefCell;
16use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
17use std::sync::Arc;
18
19use rusqlite::types::{ToSqlOutput, Value as SqlValue, ValueRef};
20use rusqlite::{Connection, OpenFlags, OptionalExtension};
21use serde::{Deserialize, Serialize};
22use serde_json::{json, Map, Value};
23use sha2::{Digest, Sha256};
24use ssp2::decode::WIRE_VERSION;
25use ssp2::model::{
26    FencedRecord, Frame, LiveStatus, MediaType, Message, MsgKind, Op, OpResult, PushStatus,
27    SubStatus,
28};
29use ssp2::primitives::RawJson;
30use ssp2::registration::canonical_scope_json;
31use ssp2::segment::{decode_rows_segment, Column, ColumnType, ColumnValue, Row, RowsSegment};
32use ssp2::{decode_message, encode_presence_publish, parse_control, ControlMessage, PresenceKind};
33
34use crate::api::{
35    AppliedServerCommit, AppliedServerCommitSource, ClientChangeBatch, ClientDiagnosticsHost,
36    ClientDiagnosticsLease, ClientDiagnosticsReplica, ClientDiagnosticsSchema,
37    ClientDiagnosticsSnapshot, ClientDiagnosticsStorage, ClientError, ClientLimits, CommandEffects,
38    CommitDelivery, CommitOperation, CommitOperationOutcome, CommitOutcome, CommitOutcomeQuery,
39    CommitOutcomeResolution, CommitOutcomeStatus, ConflictRecord, CoverageSnapshot,
40    CurrentBaseProof, CurrentBaseProofReason, DiagnosticLastChange, DiagnosticLastRound,
41    DiagnosticQueryFailure, DiagnosticRoundCounters, DiagnosticUnits, EchoRecord, FetchedBlob,
42    LeaseState, LocalDataPurgeInput, LocalDataPurgeResult, LocalDataPurgeTarget,
43    LocalDataRebootstrapInput, LocalDataRebootstrapResult, Mutation, PresencePeer,
44    PreviousVersionStatus, QueryOwner, QueryReadFailure, QueryRow, QuerySnapshot, QueryValue,
45    RejectionDetails, RejectionRecord, ResolveCommitOutcomeInput, RetainedCommitRow,
46    RetainedUniqueConflict, RowState, SchemaFloor, SnapshotRead, SyncIntent, SyncOutcome,
47    SyncReport, SyncStatusSnapshot, TableChange, UnitCatchup, UnitRecord, UnitScopes, UnitStatus,
48    WindowBase, WindowChange, WindowCoverage, WindowState, WindowUnitRef,
49    CLIENT_DIAGNOSTICS_VERSION, MAX_DIAGNOSTIC_QUERY_FAILURES,
50};
51use crate::api::{RealtimePolicy, RealtimeState, REALTIME_UNAVAILABLE_CODE};
52#[cfg(feature = "bench-internals")]
53use crate::bench::{Phase, Recorder};
54use crate::previous_version::{
55    capture_previous_version_from_replica, first_incompatibility,
56    reconcile_previous_version_at_boot, set_local_schema_descriptor,
57    sweep_previous_version_container, PendingCommitAudit, PreviousVersionContextConfig,
58    PreviousVersionLifecycle, PreviousVersionReadSpec, PreviousVersionSnapshot,
59};
60use crate::schema::{parse_schema_json, ClientSchema, FtsIndexSchema, TableSchema};
61use crate::transport::{BlobDownload, SegmentRequest, Transport, TransportError};
62use crate::values::{
63    bytes_to_hex, column_value_to_json, decode_row_bytes, encode_sparse_row_json, full_row_values,
64    is_canonical_bytes_envelope, json_to_column_value, json_to_scope_map, normalize_values_casing,
65    render_row_id_json, scope_map_to_json, SealedRow,
66};
67
68/// §4.2 default: rows and SQLite images, both applied in committed chunks.
69/// Bit 3 (signed URLs, §5.4) is
70/// added per transport capability at request-build time.
71const DEFAULT_ACCEPT: u8 = 0b0111;
72const ACCEPT_INLINE_ROWS: u8 = 1 << 0;
73const ACCEPT_EXTERNAL_ROWS: u8 = 1 << 1;
74const ACCEPT_SQLITE: u8 = 1 << 2;
75const ACCEPT_SIGNED_URLS: u8 = 1 << 3;
76const MAX_DIAGNOSTIC_DOMAINS: usize = 256;
77
78/// §10.1 `recommendedAction` carried by a server HTTP error body.
79fn recommended_action_of(details: &Option<Value>) -> Option<String> {
80    details
81        .as_ref()?
82        .get("recommendedAction")?
83        .as_str()
84        .map(str::to_owned)
85}
86
87/// §7.4.1 persisted local schema-version marker (`_syncular_meta` key).
88pub(crate) const LOCAL_SCHEMA_VERSION_KEY: &str = "localSchemaVersion";
89const LOCAL_REVISION_KEY: &str = "localRevision";
90/// §4.5 replica cursor; absent means `-1`.
91const REPLICA_CURSOR_KEY: &str = "replicaCursor";
92/// Hex of the last pull `LIVE_END.echoDigest` (§4.2); absent means empty.
93const ECHO_DIGEST_KEY: &str = "echoDigest";
94/// Hex of the registration digest of the list the server stores (§4.1).
95const REGISTERED_DIGEST_KEY: &str = "registrationDigest";
96/// §7.4.3: table name → generation, persisted with the declaration evidence.
97const TABLE_GENERATIONS_KEY: &str = "localTableGenerations";
98/// §7.1: the identity of the outbox sequence.
99const OUTBOX_ID_KEY: &str = "outboxId";
100const CLIENT_ID_KEY: &str = "clientId";
101const LEASE_STATE_KEY: &str = "leaseState";
102const SCHEMA_FLOOR_KEY: &str = "schemaFloor";
103pub(crate) const LOG_EPOCH_KEY: &str = "logEpoch";
104/// Persisted fail-closed quarantine marker: present while a security preflight
105/// is pending and `activateSecurity` has yet to run. Storing it in the replica
106/// keeps the gate with the data it protects, so a rebuilt host handle reopening
107/// the same file returns in preflight even though its in-memory state is fresh.
108const SECURITY_PREFLIGHT_PENDING_KEY: &str = "securityPreflightPending";
109const LOCAL_REBOOTSTRAP_RECEIPT_VERSION: u8 = 2;
110const MAX_JS_SAFE_INTEGER: u64 = 9_007_199_254_740_991;
111/// §7.4.4 client-local code: a pending outbox commit cannot re-encode under
112/// the new schema after a bump. Never a wire code (§10.3).
113const OUTBOX_INCOMPATIBLE_CODE: &str = "sync.outbox_incompatible";
114/// §5.11 client-local code: an encode at the push seam resolved no usable key
115/// id or named an unknown key. Never a wire code (§10.3).
116const ENCRYPT_FAILED_CODE: &str = "client.encrypt_failed";
117const INVALID_REQUEST_CODE: &str = "sync.invalid_request:";
118/// §7.4.4: static text for a persisted commit the current codec refuses.
119const LEGACY_VALUE_REFUSAL_MESSAGE: &str =
120    "the persisted commit carries values the current codec refuses";
121/// §7.4.4: bounded reason on every intrinsic value drop.
122const INVALID_STORED_VALUES_REASON: &str = "invalid_stored_values";
123
124#[derive(Debug, Serialize, Deserialize)]
125#[serde(rename_all = "camelCase", deny_unknown_fields)]
126struct PersistedLocalDataRebootstrapReceipt {
127    version: u8,
128    retained_commits: u64,
129    reset_subscriptions: u64,
130}
131
132fn invalid_local_rebootstrap_receipt() -> String {
133    "sync.local_corrupt: persisted local rebootstrap receipt is invalid".to_owned()
134}
135
136fn encode_local_rebootstrap_receipt(
137    retained_commits: usize,
138    reset_subscriptions: usize,
139) -> Result<String, String> {
140    let retained_commits =
141        u64::try_from(retained_commits).map_err(|_| invalid_local_rebootstrap_receipt())?;
142    let reset_subscriptions =
143        u64::try_from(reset_subscriptions).map_err(|_| invalid_local_rebootstrap_receipt())?;
144    if retained_commits > MAX_JS_SAFE_INTEGER || reset_subscriptions > MAX_JS_SAFE_INTEGER {
145        return Err(invalid_local_rebootstrap_receipt());
146    }
147    serde_json::to_string(&PersistedLocalDataRebootstrapReceipt {
148        version: LOCAL_REBOOTSTRAP_RECEIPT_VERSION,
149        retained_commits,
150        reset_subscriptions,
151    })
152    .map_err(|_| invalid_local_rebootstrap_receipt())
153}
154
155fn decode_local_rebootstrap_receipt(value: &str) -> Result<(usize, usize), String> {
156    // Pre-0.15.36 markers proved application but did not retain the receipt.
157    if value == "v1" {
158        return Ok((0, 0));
159    }
160    let receipt: PersistedLocalDataRebootstrapReceipt =
161        serde_json::from_str(value).map_err(|_| invalid_local_rebootstrap_receipt())?;
162    if receipt.version != LOCAL_REBOOTSTRAP_RECEIPT_VERSION
163        || receipt.retained_commits > MAX_JS_SAFE_INTEGER
164        || receipt.reset_subscriptions > MAX_JS_SAFE_INTEGER
165    {
166        return Err(invalid_local_rebootstrap_receipt());
167    }
168    Ok((
169        usize::try_from(receipt.retained_commits)
170            .map_err(|_| invalid_local_rebootstrap_receipt())?,
171        usize::try_from(receipt.reset_subscriptions)
172            .map_err(|_| invalid_local_rebootstrap_receipt())?,
173    ))
174}
175
176#[cfg(test)]
177mod observation_tests;
178
179/// Internal state for hosts that perform network I/O outside the client owner.
180#[doc(hidden)]
181pub struct CurrentBasePreparation {
182    client_commit_id: String,
183    sequence: i64,
184    units: Vec<UnitRecord>,
185    transport_epoch: u64,
186    registration_epoch: u64,
187    security_epoch: u64,
188}
189
190#[cfg(test)]
191mod units_tests;
192
193#[derive(Debug, Clone)]
194struct OutboxOp {
195    upsert: bool,
196    table: String,
197    row_id: String,
198    base_version: Option<i64>,
199    /// Schema-agnostic local form (§0): driver JSON values, encoded with
200    /// the current codec at send time. A key's presence IS the §6.1 sparse
201    /// presence set — both the overlay and the wire payload derive from it.
202    values: Option<Map<String, Value>>,
203    /// §7.1 insert intent: `mutate` recorded this upsert while the row was
204    /// absent from the visible replica. Replay and send-time encoding present
205    /// every current-schema column; a column `values` lacks is NULL.
206    insert: bool,
207}
208
209impl OutboxOp {
210    /// The values this upsert presents under the current `table` (§7.1). An
211    /// insert-intent operation presents every column, a missing one as NULL;
212    /// `None` when a missing column is non-nullable, which leaves the commit
213    /// unencodable (§7.4.4), or when the operation carries no values.
214    fn presented_values(&self, table: &TableSchema) -> Option<Cow<'_, Map<String, Value>>> {
215        let values = self.values.as_ref()?;
216        if !self.insert || table.columns.iter().all(|c| values.contains_key(&c.name)) {
217            return Some(Cow::Borrowed(values));
218        }
219        full_row_values(table, values.clone()).ok().map(Cow::Owned)
220    }
221}
222
223impl From<CommitOperation> for OutboxOp {
224    fn from(operation: CommitOperation) -> Self {
225        Self {
226            upsert: operation.op == "upsert",
227            table: operation.table,
228            row_id: operation.row_id,
229            base_version: operation.base_version,
230            values: operation.values,
231            insert: operation.insert,
232        }
233    }
234}
235
236/// §7.5 static authoring refusal; the table and column travel in details.
237fn invalid_authoring(message: &str, details: Value) -> ClientError {
238    ClientError {
239        code: "sync.invalid_request".to_owned(),
240        message: message.to_owned(),
241        details: Some(details),
242        retryable: false,
243    }
244}
245
246/// §7.1 chained base: `later` repeats `earlier`'s positive base on the same
247/// row and presents only columns `earlier` presents.
248fn chains_on(earlier: &OutboxOp, later: &OutboxOp) -> bool {
249    // Only an upsert carries values, and it always carries its primary key.
250    earlier.row_id == later.row_id
251        && earlier.table == later.table
252        && later.base_version.is_some_and(|base| base > 0)
253        && earlier.base_version == later.base_version
254        && later.values.as_ref().is_some_and(|values| {
255            values.keys().all(|column| {
256                earlier
257                    .values
258                    .as_ref()
259                    .is_some_and(|earlier| earlier.contains_key(column))
260            })
261        })
262}
263
264impl From<&OutboxOp> for CommitOperation {
265    fn from(operation: &OutboxOp) -> Self {
266        Self {
267            table: operation.table.clone(),
268            row_id: operation.row_id.clone(),
269            op: if operation.upsert { "upsert" } else { "delete" }.to_owned(),
270            base_version: operation.base_version,
271            values: operation.values.clone(),
272            insert: operation.insert,
273        }
274    }
275}
276
277#[derive(Debug, Clone)]
278struct OutboxCommit {
279    client_commit_id: String,
280    ops: Vec<OutboxOp>,
281}
282
283/// One commit's `(table, value keys, insert intent)` per upsert operation — the classification
284/// shape the §7.4.4 send-time drop and the RFC 0005 D6 audit share, so the two
285/// can never disagree about which commits cannot re-encode.
286fn commit_audit_operations(commit: &OutboxCommit) -> Vec<(&str, Vec<&str>, bool)> {
287    commit
288        .ops
289        .iter()
290        .filter(|operation| operation.upsert)
291        .map(|operation| {
292            (
293                operation.table.as_str(),
294                operation
295                    .values
296                    .as_ref()
297                    .map(|values| values.keys().map(String::as_str).collect())
298                    .unwrap_or_default(),
299                operation.insert,
300            )
301        })
302        .collect()
303}
304
305type FailedCommit = (OutboxCommit, Map<String, Value>);
306
307#[derive(Debug, Clone)]
308struct CompiledLocalDataPurgeTarget {
309    table: String,
310    selectors: Vec<(String, Vec<String>)>,
311}
312
313struct StoredCommitOutcomeRow {
314    sequence: i64,
315    client_commit_id: String,
316    status: String,
317    recorded_at_ms: i64,
318    results_json: String,
319    operations_json: Option<String>,
320    resolution: String,
321    resolved_at_ms: Option<i64>,
322    replacement_client_commit_id: Option<String>,
323}
324
325impl StoredCommitOutcomeRow {
326    fn from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result<Self> {
327        Ok(Self {
328            sequence: row.get(0)?,
329            client_commit_id: row.get(1)?,
330            status: row.get(2)?,
331            recorded_at_ms: row.get(3)?,
332            results_json: row.get(4)?,
333            operations_json: row.get(5)?,
334            resolution: row.get(6)?,
335            resolved_at_ms: row.get(7)?,
336            replacement_client_commit_id: row.get(8)?,
337        })
338    }
339}
340
341/// Section outcome distinguishing the §5.6 unit-local fail-closed
342/// path from a round-aborting failure (§1.4 rule 5).
343/// §4.4 a unit section's `SUB_START`.
344struct SectionStart {
345    id: String,
346    status: SubStatus,
347    reason_code: String,
348    bootstrap: bool,
349    fenced: Vec<FencedRecord>,
350}
351
352enum SectionError {
353    FailClosed,
354    /// §5.5: the server refused a segment with `sync.forbidden`.
355    Refused,
356    /// §5.3 client memory: the round deferred this image's body to a later
357    /// pull of the same resume token.
358    Deferred,
359    Abort(String, String),
360    Transport(TransportError),
361}
362
363#[derive(Debug)]
364pub(crate) struct RequestMeta {
365    pushed_ids: Vec<String>,
366    /// Hex of the registration digest this pull carried; the server stores
367    /// that list once the round completes (§4.1).
368    registration_digest: String,
369    pub(crate) accept: u8,
370    /// §6.1 splitBatch: outbox commits held back from THIS request because
371    /// the running operation count reached the push cap — the next round
372    /// pushes them (`sync_needed` stays set while any remain).
373    deferred_commits: usize,
374    /// §5.6 applied-segment record per unit: segment id → rows applied at
375    /// the unit's current cursor and token.
376    pub(crate) applied: BTreeMap<String, BTreeMap<String, i64>>,
377    /// §5.3 client memory: (unit, segment id) of the first image whose body
378    /// the round did not fetch because the batch already holds two images.
379    /// That unit's section stops there without persisting `SUB_END`.
380    pub(crate) deferred: BTreeSet<(String, String)>,
381}
382
383#[derive(Default)]
384struct ChangeAccumulator {
385    /// None means table-wide; Some is the exact scoped domain.
386    tables: BTreeMap<String, Option<BTreeSet<String>>>,
387    windows: BTreeMap<String, BTreeSet<String>>,
388    status: bool,
389    conflicts: bool,
390    rejections: bool,
391    outcomes: bool,
392}
393
394impl ChangeAccumulator {
395    fn table(&mut self, table: &str) {
396        self.tables.insert(table.to_owned(), None);
397    }
398
399    fn scope(&mut self, table: &str, key: String) {
400        match self.tables.get_mut(table) {
401            Some(None) => {}
402            Some(Some(keys)) => {
403                keys.insert(key);
404            }
405            None => {
406                self.tables
407                    .insert(table.to_owned(), Some(BTreeSet::from([key])));
408            }
409        }
410    }
411
412    fn window(&mut self, base_key: &str, value: &str) {
413        self.windows
414            .entry(base_key.to_owned())
415            .or_default()
416            .insert(value.to_owned());
417    }
418
419    fn touched(&self) -> bool {
420        !self.tables.is_empty()
421            || !self.windows.is_empty()
422            || self.status
423            || self.conflicts
424            || self.rejections
425            || self.outcomes
426    }
427}
428
429/// §6.1: the server caps total operations per request (reference default
430/// 500) and rejects the whole batch with `sync.too_many_operations`; the
431/// client "splits and retries". Splitting happens at build time: commits are
432/// included IN ORDER until the operation budget is spent, the rest wait for
433/// the next round.
434const PUSH_OPS_PER_REQUEST: usize = 500;
435const MAX_LOCAL_PURGE_TARGETS: usize = 64;
436const MAX_LOCAL_PURGE_SELECTORS: usize = 8;
437const MAX_LOCAL_PURGE_VALUES: usize = 128;
438const MAX_LOCAL_PURGE_VALUE_LENGTH: usize = 256;
439pub const SECURITY_PREFLIGHT_REQUIRED_CODE: &str = "client.security_preflight_required";
440
441/// Dirty visible tables; `None` means a whole-replica recovery/reset.
442struct OverlayDirty(RefCell<Option<BTreeSet<String>>>);
443
444impl OverlayDirty {
445    fn clean() -> Self {
446        Self(RefCell::new(Some(BTreeSet::new())))
447    }
448    fn get(&self) -> bool {
449        self.0
450            .borrow()
451            .as_ref()
452            .is_none_or(|tables| !tables.is_empty())
453    }
454    fn set(&self, dirty: bool) {
455        *self.0.borrow_mut() = if dirty { None } else { Some(BTreeSet::new()) };
456    }
457    fn table(&self, table: &str) {
458        if let Some(tables) = self.0.borrow_mut().as_mut() {
459            tables.insert(table.to_owned());
460        }
461    }
462    fn snapshot(&self) -> Option<BTreeSet<String>> {
463        self.0.borrow().clone()
464    }
465    fn restore(&self, tables: Option<BTreeSet<String>>) {
466        *self.0.borrow_mut() = tables;
467    }
468}
469
470/// A copy-on-write in-memory mirror: a [`Rollback`] shares it, and the first
471/// write after a capture copies it once.
472#[derive(Debug, Clone, Default)]
473struct Shared<T>(Arc<T>);
474
475impl<T> std::ops::Deref for Shared<T> {
476    type Target = T;
477    fn deref(&self) -> &T {
478        &self.0
479    }
480}
481
482impl<T: Clone> std::ops::DerefMut for Shared<T> {
483    fn deref_mut(&mut self) -> &mut T {
484        Arc::make_mut(&mut self.0)
485    }
486}
487
488impl<T> From<T> for Shared<T> {
489    fn from(value: T) -> Self {
490        Self(Arc::new(value))
491    }
492}
493
494/// The in-memory mirrors of durable state, captured before a local
495/// transaction and restored when it rolls back. Queues only grow inside a
496/// transaction, so their lengths suffice.
497#[derive(Debug, Clone)]
498struct Rollback {
499    outbox: Shared<Vec<Arc<OutboxCommit>>>,
500    failed_commits: Shared<Vec<FailedCommit>>,
501    conflicts: Shared<Vec<ConflictRecord>>,
502    rejections: Shared<Vec<RejectionRecord>>,
503    schema_floor: Option<SchemaFloor>,
504    lease_state: Option<LeaseState>,
505    stopped: bool,
506    upgrading: bool,
507    sync_needed: bool,
508    active_round: Option<uuid::Uuid>,
509    overlay_dirty: Option<BTreeSet<String>>,
510    change_queue: usize,
511    sync_intent_queue: usize,
512    last_change: Option<DiagnosticLastChange>,
513}
514
515pub struct SyncClient {
516    #[cfg(feature = "bench-internals")]
517    benchmark_phases: Recorder,
518    conn: Connection,
519    schema: ClientSchema,
520    client_id: String,
521    limits: ClientLimits,
522    outbox: Shared<Vec<Arc<OutboxCommit>>>,
523    failed_commits: Shared<Vec<FailedCommit>>,
524    retain_failed_commits: bool,
525    conflicts: Shared<Vec<ConflictRecord>>,
526    rejections: Shared<Vec<RejectionRecord>>,
527    schema_floor: Option<SchemaFloor>,
528    /// §7.3.5: the opaque auth-lease state (from LEASE frames + lease errors).
529    lease_state: Option<LeaseState>,
530    /// §1.6: the schema-floor response stops syncing until an upgrade.
531    stopped: bool,
532    /// §7.4.5: true while a schema-bump reset + first re-bootstrap is in flight.
533    upgrading: bool,
534    /// §8.4 coalesced sync-needed signal.
535    sync_needed: bool,
536    /// §8.8: how sync rounds treat the realtime binding, and its exact
537    /// availability state with the evidence behind a lost or refused one.
538    realtime_policy: RealtimePolicy,
539    realtime_state: RealtimeState,
540    realtime_reason_code: Option<String>,
541    realtime_retry_delay_ms: Option<u64>,
542    /// §8.6 presence: scopeKey → (`actorId clientId` peer key → peer).
543    presence: HashMap<String, HashMap<String, PresencePeer>>,
544    /// Client clock (epoch ms) for the §5.4 `urlExpiresAtMs` check and the
545    /// §7.8 drive deadline; the host may pin it and move the pinned value
546    /// (conformance runs on a virtual clock).
547    now_ms: Option<std::sync::Arc<std::sync::atomic::AtomicI64>>,
548    /// §5.11 client-side encryption keys (`keyId → key bytes`). Empty ⇒ E2EE
549    /// off. The encrypt/decrypt seam (`values.rs`) is compiled only under the
550    /// `e2ee` feature; without it, a schema with encrypted columns fails loud.
551    encryption: crate::values::EncryptionConfig,
552    /// Fail-closed host bootstrap gate. While set, command hosts permit only
553    /// status/lifecycle inspection and an exact authorized local purge.
554    security_preflight: bool,
555    security_epoch: u64,
556    transport_enabled: bool,
557    transport_epoch: u64,
558    /// Raised by every registration change; a current-base proof spanning
559    /// one reports missing coverage.
560    registration_epoch: u64,
561    /// RFC 0005 D2/D8: the previous-version capture config. `None` (default)
562    /// means the feature is off; the descriptor write and the container orphan
563    /// sweep still happen.
564    previous_version: Option<PreviousVersionContextConfig>,
565    /// Per-table primary-key upsert SQL, built once per (full table name) —
566    /// the row write path runs per row during bootstrap (§5.6), so the SQL
567    /// string (and, via `prepare_cached`, its compiled statement) is reused
568    /// instead of being rebuilt and re-prepared per row. Cleared on a §7.4.3
569    /// schema reset (the column lists may have changed).
570    insert_sql: RefCell<HashMap<String, String>>,
571    /// §7.1: reconcile only tables whose base/outbox changed. Whole-replica
572    /// reset and restart explicitly invalidate every table.
573    overlay_dirty: OverlayDirty,
574    // First SQLite failure of the current operation, before savepoint cleanup.
575    storage_failure: RefCell<Option<QueryReadFailure>>,
576    /// Test-only structural performance signal: response processing must not
577    /// turn a batch of acknowledgements into one full overlay rebuild each.
578    #[cfg(test)]
579    overlay_rebuild_count: Cell<usize>,
580    #[cfg(test)]
581    acknowledged_replay_count: Cell<usize>,
582    /// Test-only structural performance signal: outcome retention is enforced
583    /// once per response rather than once per acknowledgement.
584    #[cfg(test)]
585    outcome_prune_count: Cell<usize>,
586    /// Test-only pinned duration of every §5.3 image chunk, in ms.
587    #[cfg(test)]
588    image_chunk_ms: Cell<Option<f64>>,
589    /// Exact observer-transaction output drained by command/FFI hosts.
590    progress: ProgressObserver,
591    change_queue: VecDeque<ClientChangeBatch>,
592    sync_intent_queue: VecDeque<SyncIntent>,
593    /// §8.7 `scopes` events received since the last drain.
594    scopes_changed: usize,
595    /// Explicit exponential retry policy for transient transport failures.
596    retry_delay_ms: u64,
597    /// §7.7: the background retry the current round scheduled, if any.
598    round_retry_delay_ms: Option<u64>,
599    /// §5.3: a table's next image chunk size, carried from one range image
600    /// of the table to the next.
601    image_chunk_rows: HashMap<String, u32>,
602    /// A round whose response never completed (§1.5): the client keeps its
603    /// code and reports it on every following request until a round receives a
604    /// complete, decodable response. The server already recorded the round, so
605    /// nothing else can tell it the round failed on this side.
606    round_failure: Option<String>,
607    active_round: Option<uuid::Uuid>,
608    last_round: Option<DiagnosticLastRound>,
609    last_change: Option<DiagnosticLastChange>,
610    /// §7.6: latest failed owned snapshot read per owner id, oldest first.
611    query_failures: Vec<DiagnosticQueryFailure>,
612    /// SPEC §1.4 rule 4: the open coalesced apply transaction of a response.
613    coalesced: Option<CoalescedApply>,
614    /// A coalesced apply transaction of the current response failed; every
615    /// later observation of that response fails until it ends.
616    coalesced_failure: Option<String>,
617}
618
619/// SPEC §1.4 rule 4: the frames and blocks of one response share one local
620/// write transaction of at most `COALESCED_APPLY_ROWS` rows. The in-memory
621/// state captured at its `BEGIN` is restored when its `COMMIT` fails.
622struct CoalescedApply {
623    rows: usize,
624    rollback: Rollback,
625}
626
627const COALESCED_APPLY_ROWS: usize = 1024;
628
629pub(crate) fn quote_ident(name: &str) -> String {
630    format!("\"{}\"", name.replace('"', "\"\""))
631}
632/// §7.4.1 `_syncular_meta` helpers. Free functions so RFC 0005's
633/// previous-version module reads and writes the SAME records as the client
634/// instead of growing a parallel meta layer.
635pub(crate) fn meta_get(conn: &Connection, key: &str) -> rusqlite::Result<Option<String>> {
636    conn.query_row(
637        "SELECT value FROM _syncular_meta WHERE key = ?1",
638        rusqlite::params![key],
639        |row| row.get::<_, String>(0),
640    )
641    .optional()
642}
643
644pub(crate) fn meta_set(conn: &Connection, key: &str, value: &str) -> rusqlite::Result<()> {
645    conn.execute(
646        "INSERT OR REPLACE INTO _syncular_meta (key, value) VALUES (?1, ?2)",
647        rusqlite::params![key, value],
648    )
649    .map(|_| ())
650}
651
652pub(crate) fn meta_delete(conn: &Connection, key: &str) -> rusqlite::Result<()> {
653    conn.execute(
654        "DELETE FROM _syncular_meta WHERE key = ?1",
655        rusqlite::params![key],
656    )
657    .map(|_| ())
658}
659
660/// Seek by the existing key while retaining the exact legacy text match.
661fn row_id_predicate(table: &TableSchema) -> String {
662    let key = quote_ident(&table.primary_key);
663    let text_match = format!("CAST({key} AS TEXT) = ?1");
664    match table.columns[table.pk_index].ty {
665        ColumnType::String | ColumnType::Json => {
666            format!("{key} = ?1 AND {text_match}")
667        }
668        // Native columns have no declared SQLite affinity. Match the stored
669        // integer bind type explicitly. Unary + removes expression affinity so
670        // SQLite can seek the typeless key; the text check preserves exact IDs.
671        ColumnType::Integer | ColumnType::Boolean => {
672            format!("{key} = +CAST(?1 AS INTEGER) AND {text_match}")
673        }
674        // REAL-to-text conversion can round distinct values to the same text.
675        // Narrowing those matches by numeric equality would change behavior.
676        _ => text_match,
677    }
678}
679
680/// §3.3/§4.8/§5.6 row ownership: rows whose local scope columns match
681/// `effective` and that no entry of `owners` matches. `owners` holds the
682/// pair scopes of every owner of the table (§3.3 row ownership); a scoped delete never removes a row one of them still covers.
683/// Scope values bind by the column's declared type, as `row_id_predicate`
684/// does. `Ok(None)` matches nothing (an empty map or value list); `Err(())`
685/// is a missing local scope-column mapping (the fail-closed case).
686fn local_scope_predicate(
687    table: &TableSchema,
688    effective: &[(String, Vec<String>)],
689    owners: &[Vec<(String, Vec<String>)>],
690) -> Result<Option<(String, Vec<SqlValue>)>, ()> {
691    if effective.is_empty() || effective.iter().any(|(_, values)| values.is_empty()) {
692        return Ok(None);
693    }
694    let column = |variable: &str| {
695        let name = table.scope_column(variable).ok_or(())?;
696        let column = table
697            .columns
698            .iter()
699            .find(|column| column.name == name)
700            .ok_or(())?;
701        Ok((quote_ident(name), column.ty == ColumnType::Integer))
702    };
703    let mut clauses = Vec::new();
704    let mut params = Vec::new();
705    for (variable, values) in effective {
706        let (key, integer) = column(variable)?;
707        let holes = vec!["?"; values.len()].join(", ");
708        clauses.push(if integer {
709            params.extend(values.iter().cloned().map(SqlValue::Text));
710            let seek = vec!["+CAST(? AS INTEGER)"; values.len()].join(", ");
711            format!("{key} IN ({seek}) AND CAST({key} AS TEXT) IN ({holes})")
712        } else {
713            format!("{key} IN ({holes})")
714        });
715        params.extend(values.iter().cloned().map(SqlValue::Text));
716    }
717    // Owners bind as one JSON parameter per variable set, so the statement
718    // keeps its size at any owner count; owners of one variable bind the
719    // union of their values. An owner holding other values of a variable
720    // `effective` names owns none of its rows and drops out. Values compare
721    // as text, the exact semantics of the effective clauses.
722    let mut groups: BTreeMap<Vec<&str>, Vec<Vec<&[String]>>> = BTreeMap::new();
723    for owner in owners {
724        if owner.is_empty()
725            || owner.iter().any(|(variable, values)| {
726                values.is_empty()
727                    || effective.iter().any(|(name, wanted)| {
728                        name == variable && wanted.iter().all(|value| !values.contains(value))
729                    })
730            })
731        {
732            continue;
733        }
734        let mut owner: Vec<_> = owner.iter().collect();
735        owner.sort_by(|a, b| a.0.cmp(&b.0));
736        groups
737            .entry(
738                owner
739                    .iter()
740                    .map(|(variable, _)| variable.as_str())
741                    .collect(),
742            )
743            .or_default()
744            .push(owner.iter().map(|(_, values)| values.as_slice()).collect());
745    }
746    let within = |variable: &str, list: &str| {
747        let (key, integer) = column(variable)?;
748        let key = if integer {
749            format!("CAST({key} AS TEXT)")
750        } else {
751            key
752        };
753        Ok(format!("{key} IN (SELECT value FROM json_each({list}))"))
754    };
755    for (variables, tuples) in groups {
756        if let [variable] = variables[..] {
757            let values: BTreeSet<&String> = tuples.iter().flatten().copied().flatten().collect();
758            params.push(SqlValue::Text(
759                Value::from_iter(values.into_iter().cloned()).to_string(),
760            ));
761            clauses.push(format!("NOT {}", within(variable, "?")?));
762        } else {
763            params.push(SqlValue::Text(
764                serde_json::to_string(&tuples).expect("owner tuples serialize"),
765            ));
766            let matches = variables
767                .iter()
768                .enumerate()
769                .map(|(index, variable)| within(variable, &format!("o.value,'$[{index}]'")))
770                .collect::<Result<Vec<_>, ()>>()?;
771            clauses.push(format!(
772                "NOT EXISTS (SELECT 1 FROM json_each(?) o WHERE {})",
773                matches.join(" AND ")
774            ));
775        }
776    }
777    Ok(Some((clauses.join(" AND "), params)))
778}
779
780/// [`local_scope_predicate`] over decoded row values (retained rows).
781fn owns_local_row(
782    table: &TableSchema,
783    effective: &[(String, Vec<String>)],
784    owners: &[Vec<(String, Vec<String>)>],
785    values: &Map<String, Value>,
786) -> bool {
787    let matches = |scopes: &[(String, Vec<String>)]| {
788        !scopes.is_empty()
789            && scopes.iter().all(|(variable, allowed)| {
790                table
791                    .scope_column(variable)
792                    .and_then(|column| values.get(column))
793                    .filter(|value| !value.is_null())
794                    .is_some_and(|value| {
795                        allowed.contains(
796                            &value
797                                .as_str()
798                                .map_or_else(|| value.to_string(), str::to_owned),
799                        )
800                    })
801            })
802    };
803    matches(effective) && !owners.iter().any(|owner| matches(owner))
804}
805
806fn is_local_operation_code_like(value: &str) -> bool {
807    let bytes = value.as_bytes();
808    bytes.first().is_some_and(u8::is_ascii_alphanumeric)
809        && bytes
810            .iter()
811            .all(|byte| byte.is_ascii_alphanumeric() || matches!(*byte, b'.' | b'_' | b':' | b'-'))
812}
813
814fn base_table(name: &str) -> String {
815    quote_ident(&format!("_syncular_base_{name}"))
816}
817
818/// §4.8 deferred eviction record: (unit id, table, effective scope map).
819type PendingEvict = (String, String, Vec<(String, Vec<String>)>);
820
821/// §4.8: a stable key for a window base: variable + canonical fixed scopes.
822fn window_base_key(base: &WindowBase) -> String {
823    format!(
824        "{}\0{}",
825        base.variable,
826        ssp2::registration::canonical_unit_json(&scope_pairs(&base.fixed_scopes))
827    )
828}
829
830/// The unit of one window value: the fixed scopes plus the value.
831fn window_unit_scopes(base: &WindowBase, value: &str) -> UnitScopes {
832    let mut scopes = base.fixed_scopes.clone();
833    scopes.insert(base.variable.clone(), value.to_owned());
834    scopes
835}
836
837/// A unit's scope map as the codec and the digests take it.
838fn scope_pairs(scopes: &UnitScopes) -> Vec<(String, String)> {
839    let mut pairs: Vec<(String, String)> = scopes
840        .iter()
841        .map(|(variable, value)| (variable.clone(), value.clone()))
842        .collect();
843    pairs.sort_by(|a, b| a.0.encode_utf16().cmp(b.0.encode_utf16()));
844    pairs
845}
846
847/// §4.1: the unit covers every table declaring all of its variables.
848fn covers(table: &TableSchema, scopes: &UnitScopes) -> bool {
849    scopes
850        .keys()
851        .all(|variable| table.scope_column(variable).is_some())
852}
853
854/// §4.4: the effective scopes of the (unit, table) pair under `echo`: the
855/// scope map as one-element lists plus the fenced lists. `None` when the
856/// fence revoked the pair.
857fn pair_scopes(
858    scopes: &UnitScopes,
859    table: &str,
860    echo: Option<&[EchoRecord]>,
861) -> Option<Vec<(String, Vec<String>)>> {
862    let fenced = echo.and_then(|echo| echo.iter().find(|record| record.table == table));
863    if fenced.is_some_and(|record| record.status == "revoked") {
864        return None;
865    }
866    let mut effective: BTreeMap<String, Vec<String>> = fenced
867        .map(|record| record.scopes.clone())
868        .unwrap_or_default();
869    for (variable, value) in scopes {
870        effective.insert(variable.clone(), vec![value.clone()]);
871    }
872    Some(effective.into_iter().collect())
873}
874
875/// §3.3 row ownership: the pair scopes of every owner of `table` except
876/// `except_id`. An owner is active with a persisted echo and live or
877/// post-bootstrap, or bootstrapping with a segment of the table in its
878/// current bootstrap.
879fn unit_owners(
880    units: &[UnitRecord],
881    table: &TableSchema,
882    except_id: &str,
883) -> Vec<Vec<(String, Vec<String>)>> {
884    units
885        .iter()
886        .filter(|unit| {
887            unit.id != except_id
888                && unit.status == UnitStatus::Active
889                && unit.echo.is_some()
890                && (!unit.bootstrapping() || unit.seeded.contains(&table.name))
891                && covers(table, &unit.scopes)
892        })
893        .filter_map(|unit| pair_scopes(&unit.scopes, &table.name, unit.echo.as_deref()))
894        .collect()
895}
896
897/// §5.5 / §5.6: the unit's scope map, each value a one-element list.
898fn unit_scope_lists(scopes: &UnitScopes) -> Vec<(String, Vec<String>)> {
899    scopes
900        .iter()
901        .map(|(variable, value)| (variable.clone(), vec![value.clone()]))
902        .collect()
903}
904
905/// §7.1 acknowledged intent of the rows `?1` names (`[[table, id], ...]`).
906/// It joins through the (tbl, id) index: a scan per acknowledgement would
907/// make draining a batch quadratic.
908const ACKED_ROWS_BY_ROW: &str = "SELECT a.op_json,a.intent_json FROM json_each(?1) k JOIN _syncular_acked_rows a ON a.tbl=json_extract(k.value,'$[0]') AND a.id=json_extract(k.value,'$[1]') WHERE ?2 IS NULL OR a.tbl IN(SELECT value FROM json_each(?2)) ORDER BY a.commit_seq,a.idx";
909
910const UNIT_COLUMNS: &str =
911    "id, scopes, explicit, status, reason_code, cursor, bootstrap_state, seeded, echo";
912
913/// A unit row; a field that does not parse is a conversion failure, which
914/// both read paths classify as `sync.local_corrupt`.
915fn unit_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result<UnitRecord> {
916    fn json<T: serde::de::DeserializeOwned>(index: usize, raw: &str) -> rusqlite::Result<T> {
917        serde_json::from_str(raw).map_err(|error| {
918            rusqlite::Error::FromSqlConversionFailure(
919                index,
920                rusqlite::types::Type::Text,
921                Box::new(error),
922            )
923        })
924    }
925    Ok(UnitRecord {
926        id: row.get(0)?,
927        scopes: json(1, &row.get::<_, String>(1)?)?,
928        explicit: row.get::<_, i64>(2)? == 1,
929        status: json(
930            3,
931            &serde_json::to_string(&row.get::<_, String>(3)?).expect("text"),
932        )?,
933        reason_code: row.get(4)?,
934        cursor: row.get(5)?,
935        bootstrap_state: row.get(6)?,
936        seeded: json(7, &row.get::<_, String>(7)?)?,
937        echo: row
938            .get::<_, Option<String>>(8)?
939            .map(|echo| json::<Vec<EchoRecord>>(8, &echo))
940            .transpose()?
941            .map(|echo| {
942                if echo
943                    .iter()
944                    .all(|record| record.status == "active" || record.status == "revoked")
945                {
946                    Ok(echo)
947                } else {
948                    Err(rusqlite::Error::FromSqlConversionFailure(
949                        8,
950                        rusqlite::types::Type::Text,
951                        "echo status is neither active nor revoked".into(),
952                    ))
953                }
954            })
955            .transpose()?,
956    })
957}
958
959/// Every unit in registration order (§4.1).
960fn load_units(conn: &Connection) -> rusqlite::Result<Vec<UnitRecord>> {
961    conn.prepare_cached(&format!(
962        "SELECT {UNIT_COLUMNS} FROM _syncular_units ORDER BY rowid"
963    ))?
964    .query_map([], unit_from_row)?
965    .collect()
966}
967
968fn get_unit(conn: &Connection, id: &str) -> rusqlite::Result<Option<UnitRecord>> {
969    conn.query_row(
970        &format!("SELECT {UNIT_COLUMNS} FROM _syncular_units WHERE id = ?1"),
971        [id],
972        unit_from_row,
973    )
974    .optional()
975}
976
977/// Insert or update in place: an update keeps the registration order. The
978/// §5.6 applied-segment record ends when the cursor or the token changes.
979fn save_unit(conn: &Connection, unit: &UnitRecord) -> rusqlite::Result<()> {
980    conn.prepare_cached(&format!(
981        "INSERT INTO _syncular_units({UNIT_COLUMNS}) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9)
982         ON CONFLICT(id) DO UPDATE SET explicit=excluded.explicit, status=excluded.status,
983           reason_code=excluded.reason_code, cursor=excluded.cursor,
984           bootstrap_state=excluded.bootstrap_state, seeded=excluded.seeded, echo=excluded.echo,
985           applied=CASE WHEN cursor IS excluded.cursor
986             AND bootstrap_state IS excluded.bootstrap_state THEN applied END"
987    ))?
988    .execute(rusqlite::params![
989        unit.id,
990        serde_json::to_string(&unit.scopes).expect("scope map serializes"),
991        i64::from(unit.explicit),
992        unit.status.name(),
993        unit.reason_code,
994        unit.cursor,
995        unit.bootstrap_state,
996        serde_json::to_string(&unit.seeded).expect("table list serializes"),
997        unit.echo
998            .as_ref()
999            .map(|echo| serde_json::to_string(echo).expect("echo serializes")),
1000    ])
1001    .map(|_| ())
1002}
1003
1004/// §5.6 applied-segment record of unit `id`: segment id → rows applied at
1005/// its current cursor and token.
1006fn applied_rows(conn: &Connection, id: &str) -> rusqlite::Result<BTreeMap<String, i64>> {
1007    conn.query_row(
1008        "SELECT applied FROM _syncular_units WHERE id = ?1",
1009        [id],
1010        |row| row.get::<_, Option<String>>(0),
1011    )
1012    .optional()?
1013    .flatten()
1014    .map_or(Ok(BTreeMap::new()), |raw| {
1015        serde_json::from_str(&raw).map_err(|error| {
1016            rusqlite::Error::FromSqlConversionFailure(
1017                0,
1018                rusqlite::types::Type::Text,
1019                Box::new(error),
1020            )
1021        })
1022    })
1023}
1024
1025/// §5.6: `rows` rows of `segment_id` applied at unit `id`'s position. Runs
1026/// in the transaction that commits them.
1027fn record_applied_rows(
1028    conn: &Connection,
1029    id: &str,
1030    segment_id: &str,
1031    rows: i64,
1032) -> rusqlite::Result<()> {
1033    conn.prepare_cached(
1034        "UPDATE _syncular_units SET applied=json_patch(coalesce(applied,'{}'),?1) WHERE id=?2",
1035    )?
1036    .execute(rusqlite::params![
1037        serde_json::json!({ segment_id: rows }).to_string(),
1038        id
1039    ])
1040    .map(|_| ())
1041}
1042
1043/// §4.1 references: every window registry entry `(base, value)` holding a unit.
1044fn window_refs(conn: &Connection, unit_id: &str) -> rusqlite::Result<Vec<(String, String)>> {
1045    conn.prepare_cached("SELECT base, unit FROM _syncular_windows WHERE sub_id = ?1")?
1046        .query_map([unit_id], |row| Ok((row.get(0)?, row.get(1)?)))?
1047        .collect()
1048}
1049
1050/// §7.4.3 reset, §2.1 log reset and local rebootstrap: keep the unit
1051/// registrations and discard the synced state of every unit outside `keep`,
1052/// so the next round sends the list and fresh-bootstraps those units. The
1053/// replica cursor survives only while a kept unit is live. The caller owns
1054/// the transaction.
1055fn reset_units(conn: &Connection, keep: &BTreeSet<String>) -> rusqlite::Result<()> {
1056    conn.execute(
1057        "UPDATE _syncular_units SET status='active', reason_code=NULL, cursor=-1,
1058           bootstrap_state=NULL, seeded='[]', echo=NULL, applied=NULL
1059         WHERE id NOT IN (SELECT value FROM json_each(?1))",
1060        [serde_json::to_string(keep).expect("unit ids serialize")],
1061    )?;
1062    conn.execute_batch(
1063        "DELETE FROM _syncular_meta WHERE key IN ('echoDigest','registrationDigest');
1064         DELETE FROM _syncular_meta WHERE key='replicaCursor'
1065           AND NOT EXISTS (SELECT 1 FROM _syncular_units WHERE cursor IS NULL);",
1066    )
1067}
1068
1069/// §4.5: the replica cursor, `-1` while no unit is live.
1070fn replica_cursor(conn: &Connection) -> rusqlite::Result<i64> {
1071    meta_get(conn, REPLICA_CURSOR_KEY)?.map_or(Ok(-1), |raw| {
1072        raw.parse().map_err(|error| {
1073            rusqlite::Error::FromSqlConversionFailure(
1074                0,
1075                rusqlite::types::Type::Text,
1076                Box::new(error),
1077            )
1078        })
1079    })
1080}
1081
1082fn visible_table(name: &str) -> String {
1083    quote_ident(name)
1084}
1085
1086const FTS_SOURCE_ID_COLUMN: &str = "_syncular_source_id";
1087
1088/// §7.4.3: is a `sqlite_master` table name a synced table (visible or base),
1089/// i.e. NOT one of the durable bookkeeping tables the reset preserves?
1090fn is_synced_table_name(name: &str) -> bool {
1091    if name.starts_with("sqlite_") {
1092        return false;
1093    }
1094    if name.starts_with("_syncular_base_") {
1095        return true; // the base half of a synced table pair
1096    }
1097    // Bookkeeping: outbox, units, meta, blob cache/uploads.
1098    !name.starts_with("_syncular_")
1099}
1100
1101/// `"sha256:" + hex` of the bytes — the content address (§5.9.1).
1102fn blob_id_for(bytes: &[u8]) -> String {
1103    let digest = Sha256::digest(bytes);
1104    format!("sha256:{}", bytes_to_hex(&digest))
1105}
1106
1107/// One [`SyncClient::write_row`] bind parameter, borrowing the row-codec
1108/// value it wraps — strings/JSON/bytes bind as borrowed TEXT/BLOB (no copy
1109/// per row on the §5.6 bootstrap path), scalars bind owned.
1110enum RowParam<'a> {
1111    Cell(&'a Option<ColumnValue>),
1112    Version(i64),
1113    Text(Option<&'a str>),
1114}
1115
1116impl rusqlite::ToSql for RowParam<'_> {
1117    fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
1118        Ok(match self {
1119            RowParam::Version(v) => ToSqlOutput::Owned(SqlValue::Integer(*v)),
1120            RowParam::Text(text) => match text {
1121                Some(text) => ToSqlOutput::Borrowed(ValueRef::Text(text.as_bytes())),
1122                None => ToSqlOutput::Owned(SqlValue::Null),
1123            },
1124            RowParam::Cell(cell) => match cell {
1125                None => ToSqlOutput::Owned(SqlValue::Null),
1126                Some(ColumnValue::String(s)) => ToSqlOutput::Borrowed(ValueRef::Text(s.as_bytes())),
1127                Some(ColumnValue::Integer(i)) => ToSqlOutput::Owned(SqlValue::Integer(*i)),
1128                Some(ColumnValue::Float(f)) => ToSqlOutput::Owned(SqlValue::Real(*f)),
1129                Some(ColumnValue::Boolean(b)) => {
1130                    ToSqlOutput::Owned(SqlValue::Integer(i64::from(*b)))
1131                }
1132                Some(ColumnValue::Json(raw)) | Some(ColumnValue::BlobRef(raw)) => {
1133                    ToSqlOutput::Borrowed(ValueRef::Text(raw.0.as_bytes()))
1134                }
1135                // §5.10: crdt bytes store as BLOB, like bytes.
1136                Some(ColumnValue::Bytes(b)) | Some(ColumnValue::Crdt(b)) => {
1137                    ToSqlOutput::Borrowed(ValueRef::Blob(b))
1138                }
1139            },
1140        })
1141    }
1142}
1143
1144// Retain a typed primary key across the current import chunk without copying row payloads.
1145fn owned_sql_value(param: impl rusqlite::ToSql) -> Result<SqlValue, String> {
1146    match param.to_sql().map_err(|error| error.to_string())? {
1147        ToSqlOutput::Owned(value) => Ok(value),
1148        ToSqlOutput::Borrowed(value) => {
1149            SqlValue::try_from(value).map_err(|error| error.to_string())
1150        }
1151        _ => Err("sync.invalid_request: unsupported SQLite key parameter".into()),
1152    }
1153}
1154
1155/// §5.3 image cell → bind parameter, strict per the declared column type
1156/// (`boolean` from INTEGER 0/1, `json` from its raw TEXT, NULL only when
1157/// nullable). Mismatches are image-producer violations. Returns the cell
1158/// borrowed when the stored representation already matches what the row
1159/// codec would write, or the normalized scalar (boolean → 0/1, float from
1160/// INTEGER → REAL) otherwise — the local write is byte-identical to the
1161/// old convert-then-insert path without allocating per cell.
1162fn image_cell_param<'a>(column: &Column, value: ValueRef<'a>) -> Result<ToSqlOutput<'a>, String> {
1163    use ssp2::segment::ColumnType;
1164    let mismatch = || {
1165        Err(format!(
1166            "image column {:?} holds a value of the wrong type",
1167            column.name
1168        ))
1169    };
1170    match value {
1171        ValueRef::Null => {
1172            if !column.nullable {
1173                return Err(format!(
1174                    "image column {:?} is NULL but not nullable",
1175                    column.name
1176                ));
1177            }
1178            Ok(ToSqlOutput::Owned(SqlValue::Null))
1179        }
1180        ValueRef::Integer(i) => match column.ty {
1181            ColumnType::Integer => Ok(ToSqlOutput::Borrowed(value)),
1182            ColumnType::Boolean => Ok(ToSqlOutput::Owned(SqlValue::Integer(i64::from(i != 0)))),
1183            ColumnType::Float => Ok(ToSqlOutput::Owned(SqlValue::Real(i as f64))),
1184            _ => mismatch(),
1185        },
1186        ValueRef::Real(_) => match column.ty {
1187            ColumnType::Float => Ok(ToSqlOutput::Borrowed(value)),
1188            _ => mismatch(),
1189        },
1190        ValueRef::Text(t) => {
1191            std::str::from_utf8(t)
1192                .map_err(|_| format!("image column {:?} is not UTF-8", column.name))?;
1193            match column.ty {
1194                ColumnType::String | ColumnType::Json | ColumnType::BlobRef => {
1195                    Ok(ToSqlOutput::Borrowed(value))
1196                }
1197                _ => mismatch(),
1198            }
1199        }
1200        ValueRef::Blob(_) => match column.ty {
1201            // §5.10: a crdt column stores its opaque bytes as BLOB, like bytes.
1202            ColumnType::Bytes | ColumnType::Crdt => Ok(ToSqlOutput::Borrowed(value)),
1203            _ => mismatch(),
1204        },
1205    }
1206}
1207
1208pub(crate) fn sql_ref_to_json(column: &Column, value: rusqlite::types::ValueRef<'_>) -> Value {
1209    use rusqlite::types::ValueRef;
1210    match value {
1211        ValueRef::Null => Value::Null,
1212        ValueRef::Integer(i) => match column.ty {
1213            ssp2::segment::ColumnType::Boolean => Value::Bool(i != 0),
1214            ssp2::segment::ColumnType::Float => {
1215                serde_json::Number::from_f64(i as f64).map_or(Value::Null, Value::Number)
1216            }
1217            _ => Value::from(i),
1218        },
1219        ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
1220        ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
1221        ValueRef::Blob(b) => {
1222            let mut map = Map::new();
1223            map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
1224            Value::Object(map)
1225        }
1226    }
1227}
1228
1229/// Bind a driver JSON value form as a rusqlite parameter for [`SyncClient::query`].
1230/// Objects are accepted in lossless `{"$bytes": hex}` and
1231/// `{"$bigint": decimal}` envelope forms.
1232fn json_param_to_sql(value: &Value) -> Result<SqlValue, String> {
1233    Ok(match value {
1234        Value::Null => SqlValue::Null,
1235        Value::Bool(b) => SqlValue::Integer(i64::from(*b)),
1236        Value::Number(n) => {
1237            if let Some(i) = n.as_i64() {
1238                SqlValue::Integer(i)
1239            } else if let Some(f) = n.as_f64() {
1240                SqlValue::Real(f)
1241            } else {
1242                return Err(format!("query param number {n} is out of range"));
1243            }
1244        }
1245        Value::String(s) => SqlValue::Text(s.clone()),
1246        Value::Object(_) => {
1247            if let Some(hex) = value.get("$bytes").and_then(Value::as_str) {
1248                SqlValue::Blob(crate::values::hex_to_bytes(hex)?)
1249            } else if let Some(decimal) = value.get("$bigint").and_then(Value::as_str) {
1250                SqlValue::Integer(decimal.parse::<i64>().map_err(|_| {
1251                    format!("query bigint param {decimal:?} is outside SQLite's i64 range")
1252                })?)
1253            } else {
1254                return Err(
1255                    "query object param must be a {$bytes: hex} or {$bigint: decimal} value"
1256                        .to_owned(),
1257                );
1258            }
1259        }
1260        Value::Array(_) => return Err("query array params are not supported".to_owned()),
1261    })
1262}
1263
1264/// Map a rusqlite value with no schema column to consult (arbitrary query
1265/// output): integers/reals/text pass through by stored affinity, blobs ride
1266/// as `{"$bytes": hex}`. Distinct from [`sql_ref_to_json`], which uses the
1267/// schema column type to recover booleans/floats/json.
1268fn sql_ref_to_json_dynamic(value: rusqlite::types::ValueRef<'_>) -> Value {
1269    use rusqlite::types::ValueRef;
1270    match value {
1271        ValueRef::Null => Value::Null,
1272        ValueRef::Integer(i) => {
1273            // JSON/Tauri IPC cannot represent every SQLite i64 exactly. Keep
1274            // ordinary UI-sized integers ergonomic and envelope only values
1275            // beyond JavaScript's safe range.
1276            const MAX_SAFE_INTEGER: i64 = 9_007_199_254_740_991;
1277            if (-MAX_SAFE_INTEGER..=MAX_SAFE_INTEGER).contains(&i) {
1278                Value::from(i)
1279            } else {
1280                let mut map = Map::new();
1281                map.insert("$bigint".to_owned(), Value::from(i.to_string()));
1282                Value::Object(map)
1283            }
1284        }
1285        ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
1286        ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
1287        ValueRef::Blob(b) => {
1288            let mut map = Map::new();
1289            map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
1290            Value::Object(map)
1291        }
1292    }
1293}
1294
1295/// §7.5: SQLite failures classify by result code into [`QueryReadFailure`].
1296fn query_connection(
1297    conn: &Connection,
1298    sql: &str,
1299    params: &[QueryValue],
1300) -> Result<Vec<QueryRow>, QueryReadFailure> {
1301    let lowered_sql = crate::query_guard::lower_public_query_sql(sql);
1302    let mut stmt = crate::query_guard::prepare_read_only(conn, &lowered_sql)?;
1303    let bound: Vec<SqlValue> = params
1304        .iter()
1305        .map(json_param_to_sql)
1306        .collect::<Result<_, _>>()?;
1307    // §2.4: an unaliased physical version column stays engine-internal, so a
1308    // `SELECT *` row feeds straight back into mutation values. An explicit
1309    // alias (`_sync_version AS v`) returns it as ordinary query data. The
1310    // §5.11 sealed columns follow the same rule.
1311    let columns: Vec<(usize, String)> = stmt
1312        .column_names()
1313        .enumerate()
1314        .filter(|(_, name)| {
1315            !matches!(
1316                *name,
1317                "_syncular_version" | "_sync_sealed_key" | "_sync_sealed"
1318            )
1319        })
1320        .map(|(i, name)| (i, name.to_owned()))
1321        .collect();
1322    let bound_refs: Vec<&dyn rusqlite::ToSql> =
1323        bound.iter().map(|v| v as &dyn rusqlite::ToSql).collect();
1324    let mut sql_rows = stmt.query(bound_refs.as_slice())?;
1325    let mut out = Vec::new();
1326    while let Some(row) = sql_rows.next()? {
1327        let mut record = Map::new();
1328        for (i, name) in &columns {
1329            let value = row.get_ref(*i)?;
1330            record.insert(name.clone(), sql_ref_to_json_dynamic(value));
1331        }
1332        out.push(record);
1333    }
1334    Ok(out)
1335}
1336
1337fn persisted_window_state(
1338    conn: &Connection,
1339    base: &WindowBase,
1340) -> Result<WindowState, QueryReadFailure> {
1341    let mut stmt = conn.prepare(
1342        "SELECT windows.unit, units.status, units.cursor, units.bootstrap_state
1343           FROM _syncular_windows AS windows
1344           LEFT JOIN _syncular_units AS units ON units.id = windows.sub_id
1345          WHERE windows.base = ?1
1346          ORDER BY windows.unit ASC",
1347    )?;
1348    let rows = stmt.query_map(rusqlite::params![window_base_key(base)], |row| {
1349        Ok((
1350            row.get::<_, String>(0)?,
1351            row.get::<_, Option<String>>(1)?,
1352            row.get::<_, Option<i64>>(2)?,
1353            row.get::<_, Option<String>>(3)?,
1354        ))
1355    })?;
1356    let mut units = Vec::new();
1357    let mut pending = Vec::new();
1358    for row in rows {
1359        let (unit, status, cursor, bootstrap_state) = row?;
1360        // §4.8: complete iff registered, active and not bootstrapping.
1361        if status.as_deref() != Some("active")
1362            || cursor.is_some_and(|cursor| cursor < 0 || bootstrap_state.is_some())
1363        {
1364            pending.push(unit.clone());
1365        }
1366        units.push(unit);
1367    }
1368    Ok(WindowState { units, pending })
1369}
1370
1371/// One read-only statement in a §7.5 snapshot read.
1372#[derive(Debug, Clone, Copy)]
1373pub struct SnapshotStatement<'a> {
1374    pub sql: &'a str,
1375    pub params: &'a [Value],
1376}
1377
1378/// A §7.5 atomic read request: read-only statements, window coverage, unit
1379/// catch-up states, and commit deliveries, all resolved from one SQLite read
1380/// transaction and one revision.
1381#[derive(Debug, Clone, Default)]
1382pub struct SnapshotReadRequest<'a> {
1383    pub statements: Vec<SnapshotStatement<'a>>,
1384    pub coverage: &'a [WindowCoverage],
1385    pub units: Vec<String>,
1386    pub commit_ids: Vec<String>,
1387}
1388
1389#[cfg(test)]
1390thread_local! {
1391    /// Test-only rendezvous that runs between the statements of one snapshot
1392    /// read. A test uses it to commit on another connection mid-read and prove
1393    /// the read transaction's snapshot and revision stay stable.
1394    static SNAPSHOT_READ_HOOK: std::cell::RefCell<Option<Box<dyn FnOnce()>>> =
1395        std::cell::RefCell::new(None);
1396}
1397
1398#[cfg(test)]
1399fn run_snapshot_read_hook() {
1400    SNAPSHOT_READ_HOOK.with(|slot| {
1401        if let Some(hook) = slot.borrow_mut().take() {
1402            hook();
1403        }
1404    });
1405}
1406
1407/// §7.5: resolve a request over one SQLite read transaction. The revision,
1408/// every statement's rows, coverage, catch-up states, and deliveries are
1409/// captured under one `SAVEPOINT`; any statement's failure rolls the whole
1410/// read back and releases the connection.
1411fn snapshot_read_connection(
1412    conn: &Connection,
1413    request: &SnapshotReadRequest<'_>,
1414) -> Result<SnapshotRead, QueryReadFailure> {
1415    conn.execute_batch("SAVEPOINT syncular_snapshot_read")?;
1416    let corrupt = |message: &str| QueryReadFailure {
1417        code: Some("sync.local_corrupt"),
1418        sqlite_code: None,
1419        sqlite_message: None,
1420        rollback_failure: None,
1421        message: message.to_owned(),
1422    };
1423    let stored_failure = |error: rusqlite::Error, message: &str| match error {
1424        rusqlite::Error::Utf8Error(..)
1425        | rusqlite::Error::InvalidColumnType(..)
1426        | rusqlite::Error::FromSqlConversionFailure(..)
1427        | rusqlite::Error::IntegralValueOutOfRange(..) => corrupt(message),
1428        _ => QueryReadFailure::from(error),
1429    };
1430    let result: Result<SnapshotRead, QueryReadFailure> = (|| {
1431        // The durable revision as a canonical decimal `u64`; a missing marker
1432        // is the legacy zero, and any other non-canonical value is corrupt.
1433        let revision: Option<String> = conn
1434            .query_row(
1435                "SELECT value FROM _syncular_meta WHERE key = ?1",
1436                rusqlite::params![LOCAL_REVISION_KEY],
1437                |row| row.get(0),
1438            )
1439            .optional()
1440            .map_err(|error| stored_failure(error, "persisted local revision is invalid"))?;
1441        let revision = match revision {
1442            None => "0".to_owned(),
1443            Some(value)
1444                if value == "0"
1445                    || (matches!(value.as_bytes().first(), Some(b'1'..=b'9'))
1446                        && value.bytes().all(|b| b.is_ascii_digit())
1447                        && value.parse::<u64>().is_ok()) =>
1448            {
1449                value
1450            }
1451            Some(_) => return Err(corrupt("persisted local revision is invalid")),
1452        };
1453        let mut queries = Vec::with_capacity(request.statements.len());
1454        for statement in &request.statements {
1455            queries.push(query_connection(conn, statement.sql, statement.params)?);
1456            #[cfg(test)]
1457            run_snapshot_read_hook();
1458        }
1459        let mut pending = Vec::new();
1460        let mut missing = Vec::new();
1461        for requested in request.coverage {
1462            let base_key = window_base_key(&requested.base);
1463            let state = persisted_window_state(conn, &requested.base)?;
1464            for unit in BTreeSet::from_iter(requested.units.iter().cloned()) {
1465                let reference = WindowUnitRef {
1466                    base_key: base_key.clone(),
1467                    unit: unit.clone(),
1468                };
1469                if !state.units.iter().any(|held| held == &unit) {
1470                    missing.push(reference);
1471                } else if state.pending.iter().any(|held| held == &unit) {
1472                    pending.push(reference);
1473                }
1474            }
1475        }
1476        let mut units = Vec::with_capacity(request.units.len());
1477        for id in &request.units {
1478            let exists: bool = conn.query_row(
1479                "SELECT EXISTS(SELECT 1 FROM _syncular_units WHERE id = ?1)",
1480                [id],
1481                |row| row.get(0),
1482            )?;
1483            if !exists {
1484                units.push(UnitCatchup::Unknown { id: id.clone() });
1485                continue;
1486            }
1487            // Every persisted field is required to be well typed; a corrupt
1488            // record fails typed instead of inventing a state.
1489            let unit = get_unit(conn, id)
1490                .map_err(|error| stored_failure(error, "persisted unit state is invalid"))?
1491                .ok_or_else(|| corrupt("persisted unit state is invalid"))?;
1492            let cursor = match unit.cursor {
1493                Some(cursor) => cursor,
1494                None => replica_cursor(conn)
1495                    .map_err(|error| stored_failure(error, "persisted unit state is invalid"))?,
1496            };
1497            if cursor.unsigned_abs() > MAX_JS_SAFE_INTEGER {
1498                return Err(corrupt("persisted unit state is invalid"));
1499            }
1500            let active = unit.status == UnitStatus::Active;
1501            let pending = unit.bootstrapping();
1502            units.push(UnitCatchup::Known {
1503                id: id.clone(),
1504                status: unit.status,
1505                live: unit.cursor.is_none(),
1506                cursor,
1507                has_resume_token: unit.bootstrap_state.is_some(),
1508                bootstrap_complete: active && !pending,
1509                known_pending_pages: active && pending,
1510                reason_code: unit.reason_code,
1511            });
1512        }
1513        let mut deliveries = Vec::with_capacity(request.commit_ids.len());
1514        for id in &request.commit_ids {
1515            let pending: bool = conn.query_row(
1516                "SELECT EXISTS(SELECT 1 FROM _syncular_outbox WHERE commit_id = ?1)",
1517                rusqlite::params![id],
1518                |row| row.get(0),
1519            )?;
1520            if pending {
1521                deliveries.push(CommitDelivery::Pending {
1522                    client_commit_id: id.clone(),
1523                });
1524                continue;
1525            }
1526            let row = conn
1527                .query_row(
1528                    "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
1529                            resolution, resolved_at_ms, replacement_client_commit_id
1530                       FROM _syncular_commit_outcomes WHERE client_commit_id = ?1",
1531                    rusqlite::params![id],
1532                    StoredCommitOutcomeRow::from_row,
1533                )
1534                .optional()
1535                .map_err(|error| stored_failure(error, "persisted commit outcome is invalid"))?;
1536            deliveries.push(match row {
1537                Some(row) => CommitDelivery::Known {
1538                    client_commit_id: id.clone(),
1539                    outcome: SyncClient::outcome_from_row(row)
1540                        .map_err(|_| corrupt("persisted commit outcome is invalid"))?,
1541                },
1542                None => CommitDelivery::Unknown {
1543                    client_commit_id: id.clone(),
1544                },
1545            });
1546        }
1547        Ok(SnapshotRead {
1548            revision,
1549            queries,
1550            coverage: CoverageSnapshot {
1551                complete: pending.is_empty() && missing.is_empty(),
1552                pending,
1553                missing,
1554            },
1555            units,
1556            deliveries,
1557        })
1558    })();
1559    match result {
1560        Ok(read) => {
1561            conn.execute_batch("RELEASE syncular_snapshot_read")?;
1562            Ok(read)
1563        }
1564        Err(mut error) => {
1565            if let Err(rollback) = conn
1566                .execute_batch("ROLLBACK TO syncular_snapshot_read; RELEASE syncular_snapshot_read")
1567            {
1568                error.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
1569            }
1570            Err(error)
1571        }
1572    }
1573}
1574
1575fn snapshot_connection(
1576    conn: &Connection,
1577    sql: &str,
1578    params: &[Value],
1579    coverage: &[WindowCoverage],
1580) -> Result<QuerySnapshot, QueryReadFailure> {
1581    let read = snapshot_read_connection(
1582        conn,
1583        &SnapshotReadRequest {
1584            statements: vec![SnapshotStatement { sql, params }],
1585            coverage,
1586            units: Vec::new(),
1587            commit_ids: Vec::new(),
1588        },
1589    )?;
1590    Ok(QuerySnapshot {
1591        revision: read.revision,
1592        rows: read
1593            .queries
1594            .into_iter()
1595            .next()
1596            .expect("single-query snapshot returns one result"),
1597        coverage: read.coverage,
1598    })
1599}
1600
1601/// A long-lived read-only SQLite sidecar for latency-critical native views.
1602/// Network rounds stay serialized on the mutable core owner, while atomic
1603/// query snapshots use this independent connection and therefore never queue
1604/// behind HTTP/WebSocket latency.
1605pub struct FileQuerySnapshotReader {
1606    path: String,
1607    conn: Option<Connection>,
1608}
1609
1610impl FileQuerySnapshotReader {
1611    #[must_use]
1612    pub fn new(path: impl Into<String>) -> Self {
1613        Self {
1614            path: path.into(),
1615            conn: None,
1616        }
1617    }
1618
1619    fn connection(&mut self) -> Result<&Connection, QueryReadFailure> {
1620        if self.conn.is_none() {
1621            let conn = Connection::open_with_flags(
1622                &self.path,
1623                OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
1624            )
1625            .map_err(QueryReadFailure::from)?;
1626            conn.busy_timeout(std::time::Duration::from_millis(250))
1627                .map_err(QueryReadFailure::from)?;
1628            self.conn = Some(conn);
1629        }
1630        self.conn
1631            .as_ref()
1632            .ok_or_else(|| QueryReadFailure::from("read sidecar connection missing".to_owned()))
1633    }
1634
1635    /// One atomic read over the shared primitive: every statement, coverage,
1636    /// catch-up, and delivery under one `SAVEPOINT` and one revision.
1637    pub fn snapshot_read(
1638        &mut self,
1639        request: &SnapshotReadRequest<'_>,
1640    ) -> Result<SnapshotRead, QueryReadFailure> {
1641        snapshot_read_connection(self.connection()?, request)
1642    }
1643
1644    /// The current durable local revision. A host can skip a read when it is
1645    /// unchanged; a corrupt or unreadable marker fails typed.
1646    pub fn local_revision(&mut self) -> Result<String, QueryReadFailure> {
1647        Ok(self
1648            .snapshot_read(&SnapshotReadRequest::default())?
1649            .revision)
1650    }
1651
1652    /// One unit's persisted catch-up state (see the §7.5 contract).
1653    pub fn unit_catchup(&mut self, id: &str) -> Result<UnitCatchup, QueryReadFailure> {
1654        let read = self.snapshot_read(&SnapshotReadRequest {
1655            units: vec![id.to_owned()],
1656            ..Default::default()
1657        })?;
1658        Ok(read
1659            .units
1660            .into_iter()
1661            .next()
1662            .expect("one requested catch-up"))
1663    }
1664
1665    /// One client commit's delivery status: pending, its retained outcome, or
1666    /// an explicit unknown (never held, or pruned by retention).
1667    pub fn commit_delivery(&mut self, id: &str) -> Result<CommitDelivery, QueryReadFailure> {
1668        let read = self.snapshot_read(&SnapshotReadRequest {
1669            commit_ids: vec![id.to_owned()],
1670            ..Default::default()
1671        })?;
1672        Ok(read
1673            .deliveries
1674            .into_iter()
1675            .next()
1676            .expect("one requested delivery"))
1677    }
1678
1679    /// Hosts report an owned failure, and the first success after it, to the
1680    /// owning core through [`SyncClient::record_query_read`] (§7.6).
1681    pub fn query_snapshot(
1682        &mut self,
1683        sql: &str,
1684        params: &[Value],
1685        coverage: &[WindowCoverage],
1686    ) -> Result<QuerySnapshot, QueryReadFailure> {
1687        snapshot_connection(self.connection()?, sql, params, coverage)
1688    }
1689}
1690
1691fn validate_authority_reads(
1692    schema: &ClientSchema,
1693    declarations: &[crate::api::AuthorityReadDeclaration],
1694) -> Result<(), String> {
1695    let mut seen = BTreeSet::new();
1696    for read in declarations {
1697        let table = schema
1698            .table(&read.table)
1699            .ok_or("client.authority_read_forbidden: authority table is not declared")?;
1700        let plain = |name: &str| {
1701            table.columns.iter().enumerate().any(|(i, c)| {
1702                c.name == name
1703                    && name != "authority_version"
1704                    && !name.starts_with("_sync")
1705                    && !table.encrypted_columns.iter().any(|e| e.index == i)
1706                    && matches!(
1707                        c.ty,
1708                        ssp2::segment::ColumnType::String
1709                            | ssp2::segment::ColumnType::Integer
1710                            | ssp2::segment::ColumnType::Float
1711                            | ssp2::segment::ColumnType::Boolean
1712                            | ssp2::segment::ColumnType::Json
1713                    )
1714            })
1715        };
1716        if !seen.insert(&read.table)
1717            || read.columns.is_empty()
1718            || read.columns.iter().collect::<BTreeSet<_>>().len() != read.columns.len()
1719            || !read.columns.contains(&table.primary_key)
1720            || !read.columns.iter().all(|c| plain(c))
1721            || read.scopes.is_empty()
1722            || !read.scopes.iter().all(|(v, values)| {
1723                table
1724                    .scope_column(v)
1725                    .is_some_and(|c| read.columns.iter().any(|column| column == c) && plain(c))
1726                    && !values.is_empty()
1727                    && values.iter().all(|v| !v.is_empty() && v != "*")
1728            })
1729        {
1730            return Err("client.authority_read_forbidden: authority reads require plain columns, a primary key and nonempty plain scope selectors".into());
1731        }
1732    }
1733    Ok(())
1734}
1735
1736// §7.4.2: every open and recreation checks before any persistent writes.
1737/// A marker read that hit a recognized SQLite storage failure keeps that
1738/// classification (BUSY/LOCKED stay retryable); only an unclassified read
1739/// failure is the invalid-marker `sync.local_corrupt`.
1740fn schema_marker_read_failure(error: rusqlite::Error) -> String {
1741    let failure = QueryReadFailure::from(error);
1742    if failure.code.is_some() {
1743        failure.to_string()
1744    } else {
1745        "sync.local_corrupt: persisted local schema marker is unreadable or invalid".to_owned()
1746    }
1747}
1748
1749/// §7.1: reject an invalid configured client limit before any storage is
1750/// opened or written. Zero would wedge the push queue and an out-of-range
1751/// value would differ per target; the TS core enforces the same bounds.
1752fn validate_client_limits(limits: &crate::api::ClientLimits) -> Result<(), String> {
1753    if limits.outcome_retention_max_entries == Some(0) {
1754        return Err("sync.invalid_request: outcomeRetentionMaxEntries must be positive".to_owned());
1755    }
1756    if [
1757        limits.max_push_commits_per_request,
1758        limits.max_push_operations_per_request,
1759        limits.max_push_request_bytes,
1760    ]
1761    .into_iter()
1762    .flatten()
1763    .any(|value| value == 0 || value > u32::MAX as usize)
1764    {
1765        return Err(
1766            "sync.invalid_request: push limit must be an integer in 1..=4294967295".to_owned(),
1767        );
1768    }
1769    Ok(())
1770}
1771
1772fn read_local_schema_version(conn: &Connection, requested: i32) -> Result<Option<i32>, String> {
1773    let corrupt =
1774        || "sync.local_corrupt: persisted local schema marker is unreadable or invalid".to_owned();
1775    let table_type: Option<String> = conn
1776        .query_row(
1777            "SELECT type FROM sqlite_master WHERE name = '_syncular_meta'",
1778            [],
1779            |row| row.get(0),
1780        )
1781        .optional()
1782        .map_err(schema_marker_read_failure)?;
1783    let Some(table_type) = table_type else {
1784        return Ok(None);
1785    };
1786    if table_type != "table" {
1787        return Err(corrupt());
1788    }
1789    // §7.4.2: a duplicated marker cannot be resolved. `query_row` would take
1790    // whichever row SQLite returns first, and an older one would drive a
1791    // destructive reset over a newer replica.
1792    let mut statement = conn
1793        .prepare("SELECT value FROM _syncular_meta WHERE key = ?1")
1794        .map_err(schema_marker_read_failure)?;
1795    let mut rows = statement
1796        .query([LOCAL_SCHEMA_VERSION_KEY])
1797        .map_err(schema_marker_read_failure)?;
1798    let value: Option<String> = rows
1799        .next()
1800        .map_err(schema_marker_read_failure)?
1801        .map(|row| row.get(0))
1802        .transpose()
1803        .map_err(schema_marker_read_failure)?;
1804    if rows.next().map_err(schema_marker_read_failure)?.is_some() {
1805        return Err(corrupt());
1806    }
1807    let Some(value) = value else {
1808        let has_descriptor = conn
1809            .query_row(
1810                "SELECT 1 FROM _syncular_meta WHERE key = ?1 LIMIT 1",
1811                [crate::previous_version::LOCAL_SCHEMA_DESCRIPTOR_KEY],
1812                |_| Ok(()),
1813            )
1814            .optional()
1815            .map_err(schema_marker_read_failure)?
1816            .is_some();
1817        if has_descriptor {
1818            return Err(corrupt());
1819        }
1820        return Ok(None);
1821    };
1822    if !matches!(value.as_bytes().first(), Some(b'1'..=b'9'))
1823        || !value.bytes().all(|b| b.is_ascii_digit())
1824    {
1825        return Err(corrupt());
1826    }
1827    let version = value.parse::<i32>().map_err(|_| corrupt())?;
1828    if version > requested {
1829        return Err(
1830            "client.schema_downgrade: persisted local schema is newer than the requested schema"
1831                .to_owned(),
1832        );
1833    }
1834    Ok(Some(version))
1835}
1836
1837/// Open a replica file. On a browser worker the file must recover a hot
1838/// journal before any statement reads it: `wasm_vfs` replaces the installed
1839/// OPFS VFS's reserved lock callback before the connection opens its file. A
1840/// new browser replica uses 64 KiB pages (§5.3), because every OPFS call costs
1841/// the same whatever its size; a database with tables keeps its page size. Its
1842/// page cache holds 64 MiB, because SQLite's 2,000 KiB default is 31 such pages
1843/// and every cache miss is an OPFS call.
1844fn open_replica(path: &str) -> Result<Connection, String> {
1845    let browser = cfg!(all(target_arch = "wasm32", target_os = "unknown"))
1846        && !path.is_empty()
1847        && path != ":memory:";
1848    #[cfg(all(target_arch = "wasm32", target_os = "unknown"))]
1849    if browser {
1850        crate::wasm_vfs::disable_foreign_lock_check()?;
1851    }
1852    let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
1853    if browser {
1854        conn.execute_batch("PRAGMA page_size = 65536; PRAGMA cache_size = -65536")
1855            .map_err(|e| format!("open db {path:?}: {e}"))?;
1856    }
1857    Ok(conn)
1858}
1859
1860impl SyncClient {
1861    pub fn new_with_identity(
1862        client_id: Option<String>,
1863        schema_json: &Value,
1864        limits: ClientLimits,
1865    ) -> Result<Self, String> {
1866        let resolved = client_id.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
1867        let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
1868        Self::with_connection(resolved, schema_json, limits, conn)
1869    }
1870
1871    pub fn new(
1872        client_id: String,
1873        schema_json: &Value,
1874        limits: ClientLimits,
1875    ) -> Result<Self, String> {
1876        let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
1877        Self::with_connection(client_id, schema_json, limits, conn)
1878    }
1879
1880    /// Build a client backed by an on-disk SQLite database at `path` — the
1881    /// seam a native host (Tauri plugin, FFI file-DB variant) uses to persist
1882    /// across process restarts. `create_tables` runs `IF NOT EXISTS`, so
1883    /// re-opening the same file reuses the persisted rows. Keeps rusqlite out
1884    /// of the command router's dependency set (the router only holds a path).
1885    pub fn open_path(
1886        client_id: String,
1887        schema_json: &Value,
1888        limits: ClientLimits,
1889        path: &str,
1890    ) -> Result<Self, String> {
1891        validate_client_limits(&limits)?;
1892        parse_schema_json(schema_json)?;
1893        let conn = open_replica(path)?;
1894        Self::with_connection(client_id, schema_json, limits, conn)
1895    }
1896
1897    /// Open a file replica and resolve its client id. Native targets require
1898    /// WAL for the independent snapshot reader; `wasm32-unknown-unknown`
1899    /// accepts the sqlite-wasm worker journal instead.
1900    pub fn open_path_with_identity(
1901        client_id: Option<String>,
1902        schema_json: &Value,
1903        limits: ClientLimits,
1904        path: &str,
1905    ) -> Result<Self, String> {
1906        Self::open_path_with_identity_journal(
1907            client_id,
1908            schema_json,
1909            limits,
1910            path,
1911            !cfg!(all(target_arch = "wasm32", target_os = "unknown")),
1912        )
1913    }
1914
1915    fn open_path_with_identity_journal(
1916        client_id: Option<String>,
1917        schema_json: &Value,
1918        limits: ClientLimits,
1919        path: &str,
1920        require_wal: bool,
1921    ) -> Result<Self, String> {
1922        validate_client_limits(&limits)?;
1923        // §7.4.2: an invalid requested version is refused before the replica is
1924        // even created, so a rejected constructor leaves no file behind.
1925        parse_schema_json(schema_json)?;
1926        let conn = open_replica(path)?;
1927        // A file-backed native client uses an independent read connection for
1928        // latency-critical snapshots. WAL is SQLite's intended reader/writer
1929        // concurrency mode: a view read never holds a rollback-journal lock
1930        // that delays the client's next commit. The busy timeout must precede
1931        // this replica's own startup transaction; WAL is applied only after
1932        // startup succeeded, because a refused open must leave the replica's
1933        // journal mode and contents untouched (§7.4.2).
1934        conn.busy_timeout(std::time::Duration::from_millis(250))
1935            .map_err(|error| format!("configure db {path:?} busy timeout: {error}"))?;
1936        // Durability is the caller's contract here, not the VFS default: a
1937        // rollback-journal commit needs FULL before it writes the database
1938        // pages, or a killed worker can leave a half-written image behind.
1939        // WAL below inherits the same setting.
1940        conn.pragma_update(None, "synchronous", "FULL")
1941            .map_err(|error| format!("configure db {path:?} synchronous: {error}"))?;
1942        let client = Self::with_connection_identity(client_id, schema_json, limits, conn)?;
1943        // A browser worker's sqlite-wasm VFS has no WAL mode, and the wasm
1944        // host has no independent snapshot reader. The WAL requirement exists
1945        // for native file replicas, where it is the reader/writer boundary.
1946        if require_wal {
1947            let journal_mode: String = client
1948                .conn
1949                .pragma_update_and_check(None, "journal_mode", "WAL", |row| row.get(0))
1950                .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
1951            if journal_mode != "wal" {
1952                return Err(
1953                    "sync.invalid_request: file-backed client requires WAL journal mode".to_owned(),
1954                );
1955            }
1956        }
1957        Ok(client)
1958    }
1959
1960    /// Delete a file replica at `path`, including its `-journal`, `-wal`, and
1961    /// `-shm` sidecars, through the VFS that owns it. Use this after
1962    /// `client.storage_corrupt`: an OPFS SAH pool stores those files under
1963    /// opaque names, so deleting the pool directory behind SQLite bypasses the
1964    /// pool's slot accounting.
1965    ///
1966    /// Close the client and every [`FileQuerySnapshotReader`] on `path` first;
1967    /// a VFS cannot delete a file another handle still owns. A file that does
1968    /// not exist counts as deleted, so a cleanly closed replica without
1969    /// sidecars resets. The next open
1970    /// creates a fresh replica (new client id) and re-bootstraps from the
1971    /// server. A memory replica has nothing to delete and fails with
1972    /// `sync.invalid_request`.
1973    pub fn reset_replica(path: &str) -> Result<(), String> {
1974        if path.is_empty() || path == ":memory:" {
1975            return Err("sync.invalid_request: a memory replica has nothing to reset".to_owned());
1976        }
1977        // SAFETY: a null VFS name asks SQLite for its process-wide default VFS,
1978        // the same one `Connection::open(path)` above selects. `xDelete` is
1979        // that VFS's own function pointer, and `c_name` outlives the call.
1980        let vfs = unsafe { rusqlite::ffi::sqlite3_vfs_find(std::ptr::null()) };
1981        if vfs.is_null() {
1982            return Err("client.storage_io: no SQLite VFS is registered".to_owned());
1983        }
1984        let delete = unsafe { (*vfs).xDelete }
1985            .ok_or_else(|| "client.storage_io: the SQLite VFS cannot delete files".to_owned())?;
1986        for suffix in ["-wal", "-shm", "-journal", ""] {
1987            let file_name = format!("{path}{suffix}");
1988            let c_name = std::ffi::CString::new(file_name.as_str()).map_err(|_| {
1989                format!("sync.invalid_request: replica path {file_name:?} contains a NUL byte")
1990            })?;
1991            // SAFETY: `vfs` and its `xDelete` pointer come from the live
1992            // SQLite library; `c_name` is a valid NUL-terminated path.
1993            let code = unsafe { delete(vfs, c_name.as_ptr(), 1) };
1994            // A cleanly closed WAL replica has no sidecars; a missing file is
1995            // already deleted.
1996            if code != rusqlite::ffi::SQLITE_OK && code != rusqlite::ffi::SQLITE_IOERR_DELETE_NOENT
1997            {
1998                return Err(format!(
1999                    "client.storage_io: deleting replica file {file_name:?} failed with SQLite code {code}"
2000                ));
2001            }
2002        }
2003        Ok(())
2004    }
2005
2006    #[must_use]
2007    pub fn client_id(&self) -> &str {
2008        &self.client_id
2009    }
2010
2011    /// Rotate the caller's client id (§1.5) and return the new one. Use it
2012    /// after a `sync.invalid_client_id` refusal (`recommendedAction:
2013    /// resetClientId`): the next round carries the new id. A live realtime
2014    /// socket is closed so the next connection registers with the new id.
2015    /// The replica and its outbox stay in place; wipe them yourself when the
2016    /// stored data belongs to the account that signed out.
2017    pub fn reset_client_id(&mut self, transport: &mut dyn Transport) -> Result<String, String> {
2018        self.disconnect_realtime(transport);
2019        let client_id = uuid::Uuid::new_v4().to_string();
2020        self.set_meta(CLIENT_ID_KEY, &client_id)?;
2021        self.client_id.clone_from(&client_id);
2022        self.round_failure = None;
2023        Ok(client_id)
2024    }
2025
2026    /// Build a client over a caller-supplied rusqlite connection — the seam a
2027    /// native host (Tauri plugin, FFI file-DB variant) uses to back the core
2028    /// with an on-disk database (`Connection::open(path)`) rather than the
2029    /// default `:memory:`. The connection MUST be fresh (no pre-existing
2030    /// syncular tables); `create_tables` runs `IF NOT EXISTS`, so re-opening
2031    /// the same file across process restarts reuses the persisted rows.
2032    pub fn with_connection(
2033        client_id: String,
2034        schema_json: &Value,
2035        limits: ClientLimits,
2036        conn: Connection,
2037    ) -> Result<Self, String> {
2038        Self::with_connection_identity(Some(client_id), schema_json, limits, conn)
2039    }
2040
2041    /// [`Self::with_connection`] for `open_path_with_identity`: no requested id
2042    /// means a durable replica's persisted `clientId` wins over a fresh one.
2043    fn with_connection_identity(
2044        client_id: Option<String>,
2045        schema_json: &Value,
2046        limits: ClientLimits,
2047        conn: Connection,
2048    ) -> Result<Self, String> {
2049        validate_client_limits(&limits)?;
2050        let schema = parse_schema_json(schema_json)?;
2051        validate_authority_reads(&schema, &limits.authority_reads)?;
2052        // RFC 0005 D8: resolve the previous-version config before the opening
2053        // reset runs — the capture happens inside it. Bad bounds fail loud.
2054        let previous_version = match limits.previous_version_context {
2055            Some(config) => {
2056                config.validate()?;
2057                Some(config)
2058            }
2059            None => None,
2060        };
2061        let mut client = SyncClient {
2062            #[cfg(feature = "bench-internals")]
2063            benchmark_phases: Recorder::default(),
2064            conn,
2065            schema,
2066            client_id: client_id
2067                .clone()
2068                .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()),
2069            limits,
2070            outbox: Shared::default(),
2071            failed_commits: Shared::default(),
2072            retain_failed_commits: false,
2073            conflicts: Shared::default(),
2074            rejections: Shared::default(),
2075            schema_floor: None,
2076            lease_state: None,
2077            stopped: false,
2078            upgrading: false,
2079            sync_needed: false,
2080            realtime_policy: RealtimePolicy::Optional,
2081            realtime_state: RealtimeState::Disconnected,
2082            realtime_reason_code: None,
2083            realtime_retry_delay_ms: None,
2084            presence: HashMap::new(),
2085            now_ms: None,
2086            encryption: crate::values::EncryptionConfig::default(),
2087            security_preflight: false,
2088            security_epoch: 0,
2089            transport_enabled: true,
2090            transport_epoch: 0,
2091            registration_epoch: 0,
2092            previous_version,
2093            insert_sql: RefCell::new(HashMap::new()),
2094            overlay_dirty: OverlayDirty::clean(),
2095            storage_failure: RefCell::new(None),
2096            #[cfg(test)]
2097            overlay_rebuild_count: Cell::new(0),
2098            #[cfg(test)]
2099            acknowledged_replay_count: Cell::new(0),
2100            #[cfg(test)]
2101            outcome_prune_count: Cell::new(0),
2102            #[cfg(test)]
2103            image_chunk_ms: Cell::new(None),
2104            progress: ProgressObserver::default(),
2105            change_queue: VecDeque::new(),
2106            sync_intent_queue: VecDeque::new(),
2107            scopes_changed: 0,
2108            retry_delay_ms: 250,
2109            round_retry_delay_ms: None,
2110            image_chunk_rows: HashMap::new(),
2111            round_failure: None,
2112            active_round: None,
2113            last_round: None,
2114            last_change: None,
2115            query_failures: Vec::new(),
2116            coalesced: None,
2117            coalesced_failure: None,
2118        };
2119        // The row write path leans on the prepared-statement cache (two
2120        // insert statements per synced table, plus the bookkeeping
2121        // statements); size it so a multi-table schema never thrashes.
2122        client
2123            .conn
2124            .set_prepared_statement_cache_capacity(64.max(client.schema.tables.len() * 4));
2125        // §7.4.2: one write transaction (SQLite `BEGIN IMMEDIATE`, a reserved
2126        // writer) covers the marker read, the identity check, and every
2127        // bookkeeping/schema write, so a concurrent open that upgraded the
2128        // replica cannot interleave between the guard and the writes, and a
2129        // refusal leaves the replica unchanged.
2130        client
2131            .conn
2132            .execute_batch("BEGIN IMMEDIATE")
2133            .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
2134        let startup = (|| -> Result<(), String> {
2135            // The marker guard runs inside the write transaction, so the version
2136            // it validates is the version every write below commits under.
2137            let marker = read_local_schema_version(&client.conn, client.schema.version)?;
2138            // New app indexes may reference columns that only exist after a
2139            // version-bump reset.
2140            client.create_bookkeeping_tables()?;
2141            match client.get_meta(CLIENT_ID_KEY)? {
2142                Some(existing) => match client_id {
2143                    Some(requested) if existing != requested => {
2144                        return Err(format!(
2145                            "client.identity_mismatch: this database belongs to {existing:?}; refusing to rebind it to {requested:?}"
2146                        ))
2147                    }
2148                    _ => client.client_id = existing,
2149                },
2150                None => client.set_meta(CLIENT_ID_KEY, &client.client_id)?,
2151            }
2152            client.restore_persisted_state()?;
2153            client.failed_commits = client.load_failed_commits()?.into();
2154            match marker {
2155                None => {
2156                    client.create_synced_tables()?;
2157                    client.persist_schema_version()?;
2158                    client.save_unit_scope_schema()?;
2159                }
2160                Some(version) if version == client.schema.version => {
2161                    client.create_synced_tables()?;
2162                    // RFC 0005 D1: the same-version open backfills the descriptor
2163                    // for a database first opened by an unaware binary, so the
2164                    // NEXT bump can capture without needing a schema bump first.
2165                    set_local_schema_descriptor(&client.conn, &client.schema)
2166                        .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
2167                    client.save_unit_scope_schema()?;
2168                }
2169                Some(_) => client.run_schema_reset()?,
2170            }
2171            // RFC 0005 D9: the reset sweep cannot see a container whose capture
2172            // committed in its OWN file while the replica's savepoint never
2173            // released — a crash at that seam leaves a container beside a matching
2174            // OLD marker, and the next open at that version runs no reset. Discard
2175            // it before anything can read it. No-op when no container exists.
2176            client.reconcile_previous_version_at_boot()?;
2177            client.clear_satisfied_persisted_schema_floor()?;
2178            client.prune_unknown_units(false)?;
2179            // §7.1/§7.4.4: recover intrinsically codec-invalid persisted values at
2180            // the full startup/reset reconciliation boundary, inside this open
2181            // transaction and after the schema and identity guards, so a commit
2182            // an earlier version accepted is dropped with durable rejection
2183            // evidence before any replay. A table the current schema removed
2184            // keeps the §7.4.4 send-time classification, and an unresolved
2185            // encryption key stays with the send seam because hosts configure
2186            // keys after open.
2187            client.drop_unencodable_outbox(false, None)?;
2188            if marker == Some(client.schema.version)
2189                && (!client.outbox.is_empty()
2190                    || !client.failed_commits.is_empty()
2191                    || client.has_acknowledged_rows()?)
2192            {
2193                // Reconstruct the visible optimistic overlay from the durable
2194                // base plus outbox instead of trusting a process-interrupted
2195                // mirror.
2196                client.overlay_dirty.set(true);
2197                client.rebuild_overlay()?;
2198            }
2199            // Every persisted active unit needs one catch-up pull on
2200            // open: realtime only covers changes after connection, while an
2201            // idempotent setWindow correctly creates no fresh command effect.
2202            // Pending outbox work has the same restart requirement. The core
2203            // owns this intent so native hosts never poll or require an
2204            // application-issued sync().
2205            client.enqueue_startup_sync_if_needed()?;
2206            Ok(())
2207        })();
2208        match startup {
2209            Ok(()) => match client.conn.execute_batch("COMMIT") {
2210                // §4.8: an eviction a crash interrupted finishes before any
2211                // read, with or without a transport; a replica reopened in
2212                // preflight finishes it at activation.
2213                Ok(()) => {
2214                    if !client.security_preflight {
2215                        client.drain_pending_evictions()?;
2216                    }
2217                    Ok(client)
2218                }
2219                Err(error) => {
2220                    let failure = Self::sqlite_failure(&client.storage_failure, error);
2221                    if let Err(rollback) = client.conn.execute_batch("ROLLBACK") {
2222                        // Keep a failed rollback's typed storage classification.
2223                        let _ = Self::sqlite_failure(&client.storage_failure, rollback);
2224                    }
2225                    Err(failure)
2226                }
2227            },
2228            Err(error) => {
2229                if let Err(rollback) = client.conn.execute_batch("ROLLBACK") {
2230                    // Keep a failed rollback's typed storage classification.
2231                    let _ = Self::sqlite_failure(&client.storage_failure, rollback);
2232                }
2233                Err(error)
2234            }
2235        }
2236    }
2237
2238    /// Pin the client clock (epoch ms) — the §5.4 expiry check runs
2239    /// against this instead of system time (conformance virtual clock).
2240    pub fn set_now_ms(&mut self, now_ms: i64) {
2241        self.now_ms = Some(std::sync::Arc::new(now_ms.into()));
2242    }
2243
2244    /// The pinned clock, shared: a host that stores into it moves the client
2245    /// clock while a call runs (a virtual clock that advances per request).
2246    pub fn pinned_clock(&self) -> Option<std::sync::Arc<std::sync::atomic::AtomicI64>> {
2247        self.now_ms.clone()
2248    }
2249
2250    /// §5.11: install the client-side encryption keys (`keyId → key bytes`).
2251    /// The command router parses these from the `create` command's
2252    /// `encryption` config (keys as `{$bytes: hex}`).
2253    pub fn set_encryption(&mut self, encryption: crate::values::EncryptionConfig) {
2254        self.encryption = encryption;
2255    }
2256
2257    #[must_use]
2258    pub fn security_lifecycle(&self) -> &'static str {
2259        if self.security_preflight {
2260            "preflight"
2261        } else {
2262            "active"
2263        }
2264    }
2265
2266    #[must_use]
2267    pub fn security_preflight(&self) -> bool {
2268        self.security_preflight
2269    }
2270
2271    /// Host-owned, non-persisted network gate. Local operations remain available.
2272    pub fn set_transport_enabled(&mut self, transport: &mut dyn Transport, enabled: bool) {
2273        if self.transport_enabled != enabled {
2274            self.transport_enabled = enabled;
2275            self.transport_epoch = self.transport_epoch.wrapping_add(1);
2276            self.sync_intent_queue.clear();
2277            self.sync_intent_queue
2278                .push_back(if enabled && !self.security_preflight {
2279                    SyncIntent::Interactive
2280                } else {
2281                    SyncIntent::None
2282                });
2283        }
2284        // A captured reply keeps its normal apply/revocation semantics.
2285        if !enabled && self.active_round.is_none() {
2286            self.disconnect_realtime(transport);
2287        }
2288    }
2289
2290    pub fn transport_enabled(&self) -> bool {
2291        self.transport_enabled
2292    }
2293
2294    /// Quarantine this replica: enter the fail-closed gate, release all
2295    /// core-owned key material, and record the gate in the database so
2296    /// reopening re-enters preflight until `activate_security` clears it.
2297    ///
2298    /// Use [`SyncClient::seal_security_on_teardown`] for the shutdown barrier,
2299    /// which must NOT leave that durable mark.
2300    pub fn begin_security_preflight(&mut self) -> Result<(), String> {
2301        let entering = !self.security_preflight;
2302        self.seal_security_on_teardown();
2303        // Persist the quarantine so it survives handle teardown and restart:
2304        // reopening this replica re-enters preflight until activation clears it.
2305        self.set_meta(SECURITY_PREFLIGHT_PENDING_KEY, "1")?;
2306        if entering {
2307            self.emit_status_batch()?;
2308        }
2309        Ok(())
2310    }
2311
2312    /// One change batch carrying the status snapshot (§5.11 lifecycle).
2313    fn emit_status_batch(&mut self) -> Result<(), String> {
2314        self.begin_observation("syncular_status")?;
2315        let batch = ChangeAccumulator {
2316            status: true,
2317            ..ChangeAccumulator::default()
2318        };
2319        self.finish_observation("syncular_status", batch)
2320            .inspect_err(|_| self.rollback_observation("syncular_status"))
2321    }
2322
2323    /// Release core-owned key material as a host tears the client down,
2324    /// WITHOUT recording a quarantine.
2325    ///
2326    /// Shutting an activated client down is not a quarantine event. Persisting
2327    /// the gate here marks every cleanly closed replica as pending, and the
2328    /// next plain `create` is then refused permanently: the reopen path
2329    /// restores the flag from the marker and the create guard rejects it. The
2330    /// in-memory flag still closes the gate for anything still holding this
2331    /// instance, which is all a teardown barrier needs.
2332    pub fn seal_security_on_teardown(&mut self) {
2333        self.cancel_sync_round();
2334        self.security_preflight = true;
2335        self.security_epoch = self.security_epoch.wrapping_add(1);
2336        self.encryption = crate::values::EncryptionConfig::default();
2337        self.sync_intent_queue.clear();
2338    }
2339
2340    /// Install the post-authentication keyring and release the host loop.
2341    pub fn activate_security(
2342        &mut self,
2343        encryption: crate::values::EncryptionConfig,
2344    ) -> Result<(), String> {
2345        if !self.security_preflight {
2346            return Err(
2347                "sync.invalid_request: activateSecurity requires security preflight".to_owned(),
2348            );
2349        }
2350        // §4.8: an interrupted eviction finishes before protected reads open.
2351        self.drain_pending_evictions()?;
2352        self.enqueue_startup_sync_if_needed()?;
2353        self.delete_meta(SECURITY_PREFLIGHT_PENDING_KEY)?;
2354        self.encryption = encryption;
2355        self.security_preflight = false;
2356        // §5.11: rows sealed under a key this keyring now holds open at once.
2357        self.reveal_sealed_rows()
2358            .map_err(|error| format!("{}: {}", error.code, error.message))?;
2359        self.emit_status_batch()
2360    }
2361
2362    fn clock_now_ms(&self) -> i64 {
2363        self.now_ms
2364            .as_ref()
2365            .map_or_else(crate::wall_clock_ms, |clock| {
2366                clock.load(std::sync::atomic::Ordering::Relaxed)
2367            })
2368    }
2369
2370    fn create_bookkeeping_tables(&self) -> Result<(), String> {
2371        let exists = |name: &str| {
2372            self.conn
2373                .query_row(
2374                    "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?1)",
2375                    [name],
2376                    |row| row.get::<_, bool>(0),
2377                )
2378                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2379        };
2380        let outbox_existed = exists("_syncular_outbox")?;
2381        // Durable client bookkeeping (outbox + units + meta).
2382        self.conn
2383            .execute_batch(
2384                "CREATE TABLE IF NOT EXISTS _syncular_outbox (
2385                   seq INTEGER PRIMARY KEY AUTOINCREMENT,
2386                   commit_id TEXT NOT NULL UNIQUE, ops_json TEXT NOT NULL);
2387                 CREATE TABLE IF NOT EXISTS _syncular_row_deliveries (
2388                   tbl TEXT NOT NULL, id TEXT NOT NULL, commit_seq INTEGER NOT NULL, PRIMARY KEY(tbl,id));
2389                 CREATE TABLE IF NOT EXISTS _syncular_acked_rows (
2390                   commit_id TEXT NOT NULL, idx INTEGER NOT NULL, tbl TEXT NOT NULL, id TEXT NOT NULL,
2391                   commit_seq INTEGER NOT NULL, op_json TEXT NOT NULL, intent_json TEXT,
2392                   PRIMARY KEY(commit_id, idx));
2393                 CREATE INDEX IF NOT EXISTS _syncular_acked_rows_row
2394                   ON _syncular_acked_rows(tbl, id);
2395                 CREATE TABLE IF NOT EXISTS _syncular_failed_commits (
2396                   seq INTEGER PRIMARY KEY AUTOINCREMENT, client_commit_id TEXT NOT NULL UNIQUE,
2397                   operations_json TEXT NOT NULL, initial_json TEXT NOT NULL);
2398                 CREATE TABLE IF NOT EXISTS _syncular_commit_outcomes (
2399                   seq INTEGER PRIMARY KEY AUTOINCREMENT,
2400                   client_commit_id TEXT NOT NULL UNIQUE,
2401                   status TEXT NOT NULL CHECK(status IN ('applied', 'cached', 'conflict', 'rejected')),
2402                   recorded_at_ms INTEGER NOT NULL,
2403                   results_json TEXT NOT NULL,
2404                   operations_json TEXT,
2405                   resolution TEXT NOT NULL DEFAULT 'active'
2406                     CHECK(resolution IN ('active', 'resolved_keep_server', 'superseded', 'dismissed')),
2407                   resolved_at_ms INTEGER,
2408                   replacement_client_commit_id TEXT);
2409                 CREATE INDEX IF NOT EXISTS _syncular_commit_outcomes_resolution_seq
2410                   ON _syncular_commit_outcomes(resolution, seq);
2411                 CREATE TABLE IF NOT EXISTS _syncular_units (
2412                   id TEXT PRIMARY KEY, scopes TEXT NOT NULL,
2413                   explicit INTEGER NOT NULL DEFAULT 0,
2414                   status TEXT NOT NULL DEFAULT 'active', reason_code TEXT,
2415                   cursor INTEGER DEFAULT -1, bootstrap_state TEXT,
2416                   seeded TEXT NOT NULL DEFAULT '[]', echo TEXT, applied TEXT);
2417                 CREATE TABLE IF NOT EXISTS _syncular_meta (
2418                   key TEXT PRIMARY KEY, value TEXT NOT NULL);
2419                 CREATE TABLE IF NOT EXISTS _syncular_windows (
2420                   base TEXT NOT NULL, unit TEXT NOT NULL, sub_id TEXT NOT NULL,
2421                   PRIMARY KEY (base, unit));
2422                 CREATE INDEX IF NOT EXISTS _syncular_windows_unit ON _syncular_windows(sub_id);
2423                 CREATE TABLE IF NOT EXISTS _syncular_pending_evictions (
2424                   unit_id TEXT NOT NULL, tbl TEXT NOT NULL,
2425                   effective_scopes TEXT NOT NULL, PRIMARY KEY (unit_id, tbl));
2426                 CREATE TABLE IF NOT EXISTS _syncular_blob_commit_refs(
2427                   commit_id TEXT NOT NULL, blob_id TEXT NOT NULL,
2428                   PRIMARY KEY(commit_id, blob_id));
2429                 CREATE INDEX IF NOT EXISTS _syncular_blob_commit_refs_body
2430                   ON _syncular_blob_commit_refs(blob_id);",
2431            )
2432            .map_err(|e| e.to_string())?;
2433        // §5.6 applied-segment record, added after the units table shipped.
2434        if !self
2435            .conn
2436            .prepare("SELECT 1 FROM pragma_table_info('_syncular_units') WHERE name = 'applied'")
2437            .and_then(|mut stmt| stmt.exists([]))
2438            .map_err(|e| e.to_string())?
2439        {
2440            self.conn
2441                .execute_batch("ALTER TABLE _syncular_units ADD COLUMN applied TEXT")
2442                .map_err(|e| e.to_string())?;
2443        }
2444        // Migrate an outcome journal created before failed aggregate
2445        // envelopes were retained. Historical rows intentionally stay NULL.
2446        let _ = self
2447            .conn
2448            .execute_batch("ALTER TABLE _syncular_commit_outcomes ADD COLUMN operations_json TEXT");
2449        if self.get_meta(LOCAL_REVISION_KEY)?.is_none() {
2450            self.set_meta(LOCAL_REVISION_KEY, "0")?;
2451        }
2452        // §7.1: `outboxId` names the outbox sequence. A new outbox table starts
2453        // a new sequence, so it replaces the id in the same transaction.
2454        if !outbox_existed || self.get_meta(OUTBOX_ID_KEY)?.is_none() {
2455            self.set_meta(OUTBOX_ID_KEY, &uuid::Uuid::new_v4().to_string())?;
2456        }
2457        // §9: a replica written under wire version 3 keeps its outbox, journal,
2458        // rows and pending evictions and registers its units fresh. Its
2459        // subscriptions, cursors, echoes, resume tokens and window entries go;
2460        // units left from an earlier version 4 life restart.
2461        if exists("_syncular_window_pending_evict")? {
2462            let invalid: bool = self
2463                .conn
2464                .query_row(
2465                    "SELECT EXISTS(SELECT 1 FROM _syncular_window_pending_evict
2466                       WHERE NOT json_valid(effective_scopes)
2467                          OR json_type(effective_scopes) <> 'object'
2468                          OR EXISTS(SELECT 1 FROM json_each(effective_scopes)
2469                               WHERE type <> 'array'
2470                                  OR EXISTS(SELECT 1 FROM json_each(value) AS v
2471                                       WHERE v.type <> 'text')))",
2472                    [],
2473                    |row| row.get(0),
2474                )
2475                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
2476            if invalid {
2477                return Err(
2478                    "sync.local_corrupt: a version 3 pending eviction cannot be upgraded"
2479                        .to_owned(),
2480                );
2481            }
2482            self.conn
2483                .execute_batch(
2484                    "INSERT OR REPLACE INTO _syncular_pending_evictions
2485                       SELECT sub_id, tbl, effective_scopes FROM _syncular_window_pending_evict;
2486                     DROP TABLE _syncular_window_pending_evict;
2487                     DROP TABLE IF EXISTS _syncular_subscriptions;
2488                     DELETE FROM _syncular_windows;",
2489                )
2490                .and_then(|()| reset_units(&self.conn, &BTreeSet::new()))
2491                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
2492        }
2493        // §5.9.7 blob cache. Mutable upload state stays off the body row so
2494        // completing a large upload does not rewrite its BLOB record.
2495        if self.schema_has_blobs() {
2496            if exists("_syncular_blobs")? {
2497                let mut statement = self
2498                    .conn
2499                    .prepare("PRAGMA table_info(_syncular_blobs)")
2500                    .map_err(|e| e.to_string())?;
2501                let mut columns = statement
2502                    .query_map([], |row| row.get::<_, String>(1))
2503                    .map_err(|e| e.to_string())?
2504                    .collect::<Result<Vec<_>, _>>()
2505                    .map_err(|e| e.to_string())?;
2506                columns.sort();
2507                let mut expected = vec![
2508                    "blob_id",
2509                    "byte_length",
2510                    "bytes",
2511                    "created_at_ms",
2512                    "media_type",
2513                ];
2514                expected.sort_unstable();
2515                if columns != expected {
2516                    return Err(
2517                        "sync.schema_mismatch: Local blob schema is incompatible".to_owned()
2518                    );
2519                }
2520            } else {
2521                self.conn
2522                    .execute_batch(
2523                        "CREATE TABLE _syncular_blobs(
2524                           blob_id TEXT PRIMARY KEY,
2525                           bytes BLOB NOT NULL,
2526                           byte_length INTEGER NOT NULL,
2527                           media_type TEXT,
2528                           created_at_ms INTEGER NOT NULL);",
2529                    )
2530                    .map_err(|e| e.to_string())?;
2531            }
2532            self.conn
2533                .execute_batch(
2534                    "CREATE TABLE IF NOT EXISTS _syncular_blob_uploads(
2535                       blob_id TEXT PRIMARY KEY,
2536                       media_type TEXT,
2537                       created_at_ms INTEGER NOT NULL);",
2538                )
2539                .map_err(|e| e.to_string())?;
2540        }
2541        Ok(())
2542    }
2543
2544    /// True iff any synced table declares a `blob_ref` column (§5.9).
2545    fn schema_has_blobs(&self) -> bool {
2546        self.schema
2547            .tables
2548            .iter()
2549            .any(|t| t.columns.iter().any(|c| c.ty == ColumnType::BlobRef))
2550    }
2551
2552    // -- meta (§7.4.1 marker, bookkeeping) ------------------------------------
2553
2554    fn get_meta(&self, key: &str) -> Result<Option<String>, String> {
2555        meta_get(&self.conn, key)
2556            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2557    }
2558
2559    // The caller holds the startup/reset transaction. Either both schema
2560    // records persist or the enclosing operation fails and rolls back.
2561    fn persist_schema_version(&self) -> Result<(), String> {
2562        self.conn
2563            .execute(
2564                "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
2565                rusqlite::params![LOCAL_SCHEMA_VERSION_KEY, self.schema.version.to_string()],
2566            )
2567            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
2568        set_local_schema_descriptor(&self.conn, &self.schema)
2569            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2570    }
2571
2572    pub(crate) fn set_meta(&self, key: &str, value: &str) -> Result<(), String> {
2573        meta_set(&self.conn, key, value)
2574            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2575    }
2576
2577    fn delete_meta(&self, key: &str) -> Result<(), String> {
2578        meta_delete(&self.conn, key)
2579            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2580    }
2581
2582    /// §7.1: the durable outbox in FIFO order, in schema-agnostic local form.
2583    fn load_outbox(&self) -> Result<Vec<Arc<OutboxCommit>>, String> {
2584        let mut stmt = self
2585            .conn
2586            .prepare("SELECT commit_id, ops_json FROM _syncular_outbox ORDER BY seq ASC")
2587            .map_err(|error| error.to_string())?;
2588        let rows = stmt
2589            .query_map([], |row| {
2590                Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
2591            })
2592            .map_err(|error| error.to_string())?;
2593        let mut commits = Vec::new();
2594        for row in rows {
2595            let (client_commit_id, raw) = row.map_err(|error| error.to_string())?;
2596            let entries: Vec<Value> = serde_json::from_str(&raw).map_err(|error| {
2597                format!("invalid persisted outbox {client_commit_id:?}: {error}")
2598            })?;
2599            let mut ops = Vec::with_capacity(entries.len());
2600            for entry in entries {
2601                let op = entry.get("op").and_then(Value::as_str).unwrap_or("delete");
2602                ops.push(OutboxOp {
2603                    upsert: op == "upsert",
2604                    table: entry
2605                        .get("table")
2606                        .and_then(Value::as_str)
2607                        .ok_or_else(|| "persisted outbox operation missing table".to_owned())?
2608                        .to_owned(),
2609                    row_id: entry
2610                        .get("rowId")
2611                        .and_then(Value::as_str)
2612                        .ok_or_else(|| "persisted outbox operation missing rowId".to_owned())?
2613                        .to_owned(),
2614                    base_version: entry.get("baseVersion").and_then(Value::as_i64),
2615                    // A legacy entry's `changedFields` key is ignored:
2616                    // the values map is the presence set (§6.1).
2617                    values: entry.get("values").and_then(Value::as_object).cloned(),
2618                    insert: entry.get("insert") == Some(&Value::Bool(true)),
2619                });
2620            }
2621            commits.push(Arc::new(OutboxCommit {
2622                client_commit_id,
2623                ops,
2624            }));
2625        }
2626        Ok(commits)
2627    }
2628
2629    fn restore_persisted_state(&mut self) -> Result<(), String> {
2630        self.outbox = self.load_outbox()?.into();
2631
2632        self.prune_commit_outcomes()?;
2633        let active = self.commit_outcomes(CommitOutcomeQuery {
2634            active_only: true,
2635            ..CommitOutcomeQuery::default()
2636        })?;
2637        self.conflicts = active
2638            .iter()
2639            .flat_map(|outcome| outcome.results.iter())
2640            .filter_map(|result| match result {
2641                CommitOperationOutcome::Conflict { conflict } => Some(conflict.clone()),
2642                _ => None,
2643            })
2644            .collect::<Vec<_>>()
2645            .into();
2646        self.rejections = active
2647            .iter()
2648            .flat_map(|outcome| outcome.results.iter())
2649            .filter_map(|result| match result {
2650                CommitOperationOutcome::Error { rejection } => Some(rejection.clone()),
2651                _ => None,
2652            })
2653            .collect::<Vec<_>>()
2654            .into();
2655
2656        self.lease_state = self
2657            .get_meta(LEASE_STATE_KEY)?
2658            .map(|raw| serde_json::from_str(&raw))
2659            .transpose()
2660            .map_err(|error| format!("invalid persisted lease state: {error}"))?;
2661        self.schema_floor = self
2662            .get_meta(SCHEMA_FLOOR_KEY)?
2663            .map(|raw| serde_json::from_str(&raw))
2664            .transpose()
2665            .map_err(|error| format!("invalid persisted schema floor: {error}"))?;
2666        self.stopped = self.schema_floor.is_some();
2667        // A replica left in unactivated preflight reopens gated: the quarantine
2668        // decision persists with the data across handle teardown and restart.
2669        if self.get_meta(SECURITY_PREFLIGHT_PENDING_KEY)?.as_deref() == Some("1") {
2670            self.security_preflight = true;
2671        }
2672        Ok(())
2673    }
2674
2675    fn save_unit_scope_schema(&self) -> Result<(), String> {
2676        let declarations: BTreeMap<_, BTreeMap<_, _>> = self
2677            .schema
2678            .tables
2679            .iter()
2680            .map(|table| {
2681                (
2682                    table.name.clone(),
2683                    table
2684                        .scope_variables
2685                        .iter()
2686                        .map(|scope| {
2687                            (
2688                                scope.variable.clone(),
2689                                [scope.prefix.clone(), scope.column.clone()],
2690                            )
2691                        })
2692                        .collect(),
2693                )
2694            })
2695            .collect();
2696        self.set_meta(
2697            "subscriptionScopeSchema",
2698            &serde_json::to_string(&declarations).expect("scope schema serialization"),
2699        )?;
2700        let generations: BTreeMap<_, _> = self
2701            .schema
2702            .tables
2703            .iter()
2704            .filter_map(|table| Some((&table.name, table.generation.as_ref()?)))
2705            .collect();
2706        self.set_meta(
2707            TABLE_GENERATIONS_KEY,
2708            &serde_json::to_string(&generations).expect("generations serialize"),
2709        )
2710    }
2711
2712    /// Drop units that cover no table of the running schema, with their window
2713    /// registry rows and pending evictions; never translate scopes. At a bump
2714    /// (§7.4.3) a unit also needs every variable unchanged in prefix and column
2715    /// on every table it covered before.
2716    fn prune_unknown_units(&mut self, bump: bool) -> Result<(), String> {
2717        let previous: Option<BTreeMap<String, BTreeMap<String, [String; 2]>>> = if bump {
2718            self.get_meta("subscriptionScopeSchema")?
2719                .map(|raw| serde_json::from_str(&raw))
2720                .transpose()
2721                .map_err(|_| "sync.local_corrupt: invalid persisted scope schema".to_owned())?
2722        } else {
2723            None
2724        };
2725        // A table keeps the variables' meaning when each has the same prefix
2726        // and column as before. An old binary stored no declaration evidence;
2727        // matching names cannot prove unchanged meaning, so the app registers
2728        // again.
2729        let unchanged = |name: &str, variables: &[&String]| {
2730            let (Some(old), Some(table)) = (
2731                previous.as_ref().and_then(|previous| previous.get(name)),
2732                self.schema.table(name),
2733            ) else {
2734                return false;
2735            };
2736            variables.iter().all(|variable| {
2737                table
2738                    .scope_variables
2739                    .iter()
2740                    .find(|scope| &scope.variable == *variable)
2741                    .is_some_and(|scope| {
2742                        old.get(*variable) == Some(&[scope.prefix.clone(), scope.column.clone()])
2743                    })
2744            })
2745        };
2746        let compatible = |scopes: &UnitScopes| {
2747            if !self.schema.tables.iter().any(|table| covers(table, scopes)) {
2748                return false;
2749            }
2750            if !bump {
2751                return true;
2752            }
2753            let Some(previous) = &previous else {
2754                return false;
2755            };
2756            previous.iter().all(|(name, old)| {
2757                !scopes.keys().all(|variable| old.contains_key(variable))
2758                    || unchanged(name, &scopes.keys().collect::<Vec<_>>())
2759            })
2760        };
2761        let mut stale: BTreeSet<String> = self
2762            .stored(load_units(&self.conn))?
2763            .into_iter()
2764            .filter(|unit| !compatible(&unit.scopes))
2765            .map(|unit| unit.id)
2766            .collect();
2767        // A shrunk window's deferred eviction outlives its unit row and keeps
2768        // its per-table scopes. A table the schema no longer has cannot be
2769        // evicted, and at a bump a table whose scope declarations changed must
2770        // not be evicted by the outdated rule.
2771        for (id, table, effective) in self.load_pending_evictions()? {
2772            if self.schema.table(&table).is_none()
2773                || (bump
2774                    && !unchanged(
2775                        &table,
2776                        &effective
2777                            .iter()
2778                            .map(|(variable, _)| variable)
2779                            .collect::<Vec<_>>(),
2780                    ))
2781            {
2782                stale.insert(id);
2783            }
2784        }
2785        for id in &stale {
2786            for sql in [
2787                "DELETE FROM _syncular_windows WHERE sub_id = ?1",
2788                "DELETE FROM _syncular_pending_evictions WHERE unit_id = ?1",
2789                "DELETE FROM _syncular_units WHERE id = ?1",
2790            ] {
2791                self.conn
2792                    .execute(sql, rusqlite::params![id])
2793                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
2794            }
2795        }
2796        Ok(())
2797    }
2798
2799    pub fn local_revision(&self) -> Result<u64, String> {
2800        Ok(self
2801            .get_meta(LOCAL_REVISION_KEY)?
2802            .and_then(|value| value.parse().ok())
2803            .unwrap_or(0))
2804    }
2805
2806    /// §7.5: resolve a snapshot read on the mutable owner connection through
2807    /// the same primitive the read-only sidecar uses, so the command surface
2808    /// and the file sidecar return one revision, rows, coverage, catch-up, and
2809    /// delivery shape. The structured [`QueryReadFailure`] is preserved; an
2810    /// `owner` records its failure (and its next success) in diagnostics.
2811    pub fn snapshot_read(
2812        &mut self,
2813        request: &SnapshotReadRequest<'_>,
2814        owner: Option<QueryOwner<'_>>,
2815    ) -> Result<SnapshotRead, QueryReadFailure> {
2816        let result = snapshot_read_connection(&self.conn, request);
2817        if let Some(owner) = owner {
2818            self.record_query_read(owner, result.as_ref().err());
2819        }
2820        result
2821    }
2822
2823    #[must_use]
2824    pub fn status_snapshot(&self) -> SyncStatusSnapshot {
2825        SyncStatusSnapshot {
2826            current_schema_version: self.schema.version,
2827            outbox: self.outbox.len(),
2828            upgrading: self.upgrading,
2829            lease_state: self.lease_state.clone(),
2830            schema_floor: self.schema_floor.clone(),
2831            sync_needed: self.sync_needed,
2832            previous_version_context: self.previous_version_status(),
2833            security_lifecycle: self.security_lifecycle(),
2834        }
2835    }
2836
2837    /// RFC 0005 A2: container presence for a host that must act on it (Diego's
2838    /// rollback precondition), read without creating the file.
2839    fn previous_version_status(&self) -> PreviousVersionStatus {
2840        let Some(replica_path) = self.previous_version_replica_path() else {
2841            return PreviousVersionStatus {
2842                present: false,
2843                created_at_ms: None,
2844            };
2845        };
2846        match crate::previous_version::read_previous_version_record(&replica_path) {
2847            Ok(Some(record)) => PreviousVersionStatus {
2848                present: true,
2849                created_at_ms: Some(record.created_at_ms),
2850            },
2851            _ => PreviousVersionStatus {
2852                present: std::path::Path::new(
2853                    &crate::previous_version::previous_version_container_path(&replica_path),
2854                )
2855                .exists(),
2856                created_at_ms: None,
2857            },
2858        }
2859    }
2860
2861    /// RFC 0005 D7: read captured rows from the previous local schema. `state`
2862    /// is always `previousVersion`; the container never makes
2863    /// `query_snapshot()` coverage complete. Lease stop/expiry, scope
2864    /// revocation, coverage completion and the TTL DISCARD the container and
2865    /// close the read with their own reason.
2866    pub fn previous_version_snapshot(
2867        &self,
2868        spec: &PreviousVersionReadSpec,
2869    ) -> Result<PreviousVersionSnapshot, String> {
2870        let now_ms = self.clock_now_ms();
2871        crate::previous_version::previous_version_snapshot(
2872            &self.conn,
2873            self.previous_version_replica_path().as_deref(),
2874            self.previous_version.as_ref(),
2875            PreviousVersionLifecycle {
2876                lease_inactive: self.previous_version_lease_inactive(now_ms),
2877                scope_revoked: self
2878                    .stored(load_units(&self.conn))?
2879                    .iter()
2880                    .any(|unit| unit.status == UnitStatus::Revoked),
2881                coverage_complete: self.previous_version_coverage_complete()?,
2882            },
2883            now_ms,
2884            self.schema.version,
2885            spec,
2886        )
2887    }
2888
2889    /// RFC 0005 D6: the advisory pre-reset compatibility audit.
2890    pub fn previous_version_audit(
2891        &self,
2892    ) -> Result<Option<crate::previous_version::PreviousVersionAudit>, String> {
2893        crate::previous_version::stored_previous_version_audit(&self.conn)
2894            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2895    }
2896
2897    /// RFC 0005 A2/D9: the executable downgrade step. Drop the container file
2898    /// and both metadata records, and report whether anything was present.
2899    /// Idempotent: absence is a successful no-op.
2900    pub fn previous_version_discard(
2901        &self,
2902    ) -> Result<crate::previous_version::PreviousVersionDiscardOutcome, String> {
2903        self.drop_previous_version()
2904    }
2905
2906    fn drop_previous_version(
2907        &self,
2908    ) -> Result<crate::previous_version::PreviousVersionDiscardOutcome, String> {
2909        let Some(replica_path) = self.previous_version_replica_path() else {
2910            return Ok(crate::previous_version::PreviousVersionDiscardOutcome {
2911                present: false,
2912                discarded: false,
2913            });
2914        };
2915        crate::previous_version::discard_previous_version(&self.conn, &replica_path)
2916    }
2917
2918    /// RFC 0005 D7: `leaseState.errorCode` or an expired lease.
2919    fn previous_version_lease_inactive(&self, now_ms: i64) -> bool {
2920        let Some(lease) = &self.lease_state else {
2921            return false;
2922        };
2923        if lease.error_code.is_some() {
2924            return true;
2925        }
2926        lease.expires_at_ms.is_some_and(|expires| expires <= now_ms)
2927    }
2928
2929    /// RFC 0005 D7: §7.4.5 replacement coverage — no active unit is
2930    /// bootstrapping. An empty active set is not completion (mirrors the TS
2931    /// predicate exactly).
2932    fn previous_version_coverage_complete(&self) -> Result<bool, String> {
2933        let units = self.stored(load_units(&self.conn))?;
2934        let mut active = units
2935            .iter()
2936            .filter(|unit| unit.status == UnitStatus::Active)
2937            .peekable();
2938        Ok(active.peek().is_some() && !active.any(UnitRecord::bootstrapping))
2939    }
2940
2941    /// RFC 0005 D7: lifetime triggers evaluated once per sync round, after the
2942    /// apply. A container that no longer has a reason to exist is discarded so
2943    /// the rows do not outlive the window they were kept for.
2944    fn previous_version_lifetime_check(&self) {
2945        let Some(config) = self.previous_version.filter(|config| config.enabled) else {
2946            return;
2947        };
2948        let now_ms = self.clock_now_ms();
2949        if self.previous_version_lease_inactive(now_ms) {
2950            let _ = self.drop_previous_version();
2951            return;
2952        }
2953        let Some(replica_path) = self.previous_version_replica_path() else {
2954            return;
2955        };
2956        let Ok(Some(record)) = crate::previous_version::read_previous_version_record(&replica_path)
2957        else {
2958            return;
2959        };
2960        let Ok(units) = load_units(&self.conn) else {
2961            return;
2962        };
2963        if units.iter().any(|unit| unit.status == UnitStatus::Revoked) {
2964            let _ = self.drop_previous_version();
2965            return;
2966        }
2967        if now_ms - record.created_at_ms > config.max_age_ms {
2968            let _ = self.drop_previous_version();
2969            return;
2970        }
2971        if self.previous_version_coverage_complete() == Ok(true) {
2972            let _ = self.drop_previous_version();
2973        }
2974    }
2975
2976    pub fn diagnostics_snapshot(&self) -> Result<ClientDiagnosticsSnapshot, String> {
2977        let registered = self.stored(load_units(&self.conn))?;
2978        // Both meta values in one statement: the FFI host recomputes
2979        // diagnostics after every command.
2980        let (local_revision, replica_cursor): (Option<String>, Option<String>) = self.stored(
2981            self.conn
2982                .prepare_cached(
2983                    "SELECT (SELECT value FROM _syncular_meta WHERE key = ?1),
2984                            (SELECT value FROM _syncular_meta WHERE key = ?2)",
2985                )
2986                .and_then(|mut statement| {
2987                    statement.query_row([LOCAL_REVISION_KEY, REPLICA_CURSOR_KEY], |row| {
2988                        Ok((row.get(0)?, row.get(1)?))
2989                    })
2990                }),
2991        )?;
2992        let mut units = DiagnosticUnits::default();
2993        for unit in &registered {
2994            *match unit.status {
2995                UnitStatus::Revoked => &mut units.revoked,
2996                UnitStatus::Failed => &mut units.failed,
2997                UnitStatus::Active if !unit.bootstrapping() => &mut units.complete,
2998                UnitStatus::Active
2999                    if unit.reason_code.as_deref() == Some("sync.cursor_expired") =>
3000                {
3001                    &mut units.reset
3002                }
3003                UnitStatus::Active => &mut units.bootstrapping,
3004            } += 1;
3005        }
3006        units.reason_codes = registered
3007            .iter()
3008            .filter_map(|unit| unit.reason_code.as_deref().map(Self::diagnostic_code))
3009            .collect::<BTreeSet<_>>()
3010            .into_iter()
3011            .collect();
3012        let captured_at_ms = self.clock_now_ms();
3013        let lease = if let Some(error_code) = self
3014            .lease_state
3015            .as_ref()
3016            .and_then(|state| state.error_code.clone())
3017        {
3018            ClientDiagnosticsLease {
3019                state: "stopped".to_owned(),
3020                expires_at_ms: self
3021                    .lease_state
3022                    .as_ref()
3023                    .and_then(|state| state.expires_at_ms),
3024                error_code: Some(Self::diagnostic_code(&error_code)),
3025            }
3026        } else if let Some(expires_at_ms) = self
3027            .lease_state
3028            .as_ref()
3029            .and_then(|state| state.expires_at_ms)
3030        {
3031            ClientDiagnosticsLease {
3032                state: if expires_at_ms <= captured_at_ms {
3033                    "expired".to_owned()
3034                } else {
3035                    "active".to_owned()
3036                },
3037                expires_at_ms: Some(expires_at_ms),
3038                error_code: None,
3039            }
3040        } else {
3041            ClientDiagnosticsLease {
3042                state: "none".to_owned(),
3043                expires_at_ms: None,
3044                error_code: None,
3045            }
3046        };
3047        let connectivity = match self.last_round.as_ref() {
3048            _ if !self.transport_enabled && self.active_round.is_none() => "offline",
3049            Some(round) if round.status == "succeeded" => "online",
3050            Some(round)
3051                if round.status == "failed"
3052                    && matches!(
3053                        round.cause_kind,
3054                        Some(
3055                            crate::TransportCauseKind::Network | crate::TransportCauseKind::Connect
3056                        )
3057                    ) =>
3058            {
3059                "offline"
3060            }
3061            Some(round) if round.status == "failed" && round.http_status.is_some() => "online",
3062            _ => "unknown",
3063        };
3064        Ok(ClientDiagnosticsSnapshot {
3065            version: CLIENT_DIAGNOSTICS_VERSION,
3066            captured_at_ms,
3067            host: ClientDiagnosticsHost {
3068                kind: "direct".to_owned(),
3069                role: "single".to_owned(),
3070                connectivity: connectivity.to_owned(),
3071                realtime: self.realtime_state,
3072                realtime_policy: self.realtime_policy,
3073                realtime_reason_code: self.realtime_reason_code.clone(),
3074                realtime_retry_delay_ms: self.realtime_retry_delay_ms,
3075            },
3076            security_lifecycle: self.security_lifecycle().to_owned(),
3077            schema: ClientDiagnosticsSchema {
3078                current_version: self.schema.version,
3079                upgrading: self.upgrading,
3080                required_version: self
3081                    .schema_floor
3082                    .as_ref()
3083                    .and_then(|floor| floor.required_schema_version),
3084                latest_version: self
3085                    .schema_floor
3086                    .as_ref()
3087                    .and_then(|floor| floor.latest_schema_version),
3088            },
3089            replica: ClientDiagnosticsReplica {
3090                local_revision: local_revision
3091                    .and_then(|value| value.parse::<u64>().ok())
3092                    .unwrap_or(0)
3093                    .to_string(),
3094                sync_needed: self.sync_needed,
3095                pending_outbox: self.outbox.len(),
3096            },
3097            lease,
3098            active_units: registered
3099                .iter()
3100                .filter(|unit| unit.status == UnitStatus::Active)
3101                .count(),
3102            units,
3103            replica_cursor: replica_cursor.map_or(Ok(-1), |raw| {
3104                raw.parse()
3105                    .map_err(|_| "sync.local_corrupt: persisted unit state is invalid".to_owned())
3106            })?,
3107            last_round: self.last_round.clone(),
3108            last_change: self.last_change.clone(),
3109            storage: self.diagnostics_storage(),
3110            query_failures: self.query_failures.clone(),
3111        })
3112    }
3113
3114    fn diagnostics_storage(&self) -> ClientDiagnosticsStorage {
3115        let read = || -> Result<ClientDiagnosticsStorage, rusqlite::Error> {
3116            let page_count: i64 = self
3117                .conn
3118                .query_row("PRAGMA page_count", [], |row| row.get(0))?;
3119            let page_size: i64 = self
3120                .conn
3121                .query_row("PRAGMA page_size", [], |row| row.get(0))?;
3122            // One cached aggregate, since the FFI host recomputes diagnostics
3123            // after every command; every invocation still reads current storage.
3124            let (outbox_bytes, outcome_entries, outcome_bytes): (i64, i64, i64) = self
3125                .conn
3126                .prepare_cached(
3127                    "SELECT (SELECT COALESCE(SUM(LENGTH(ops_json)), 0) FROM _syncular_outbox),
3128                            COUNT(*), COALESCE(SUM(LENGTH(results_json) + COALESCE(LENGTH(operations_json), 0)), 0)
3129                       FROM _syncular_commit_outcomes",
3130                )?
3131                .query_row([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))?;
3132            let blob_bytes = if self.schema_has_blobs() {
3133                self.conn
3134                    .prepare_cached("SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs")?
3135                    .query_row([], |row| row.get(0))?
3136            } else {
3137                0
3138            };
3139            let pressure = self
3140                .limits
3141                .blob_cache_max_bytes
3142                .is_some_and(|limit| blob_bytes > limit);
3143            Ok(ClientDiagnosticsStorage {
3144                status: if pressure { "pressure" } else { "healthy" }.to_owned(),
3145                database_bytes_approx: Some(page_count.saturating_mul(page_size).max(0)),
3146                pending_outbox_bytes_approx: Some(outbox_bytes.max(0)),
3147                retained_outcome_bytes_approx: Some(outcome_bytes.max(0)),
3148                retained_outcome_entries: Some(outcome_entries.max(0)),
3149                blob_cache_bytes_approx: Some(blob_bytes.max(0)),
3150                pressure_reason_code: pressure.then(|| "client.blob_cache_over_limit".to_owned()),
3151            })
3152        };
3153        read().unwrap_or_else(|_| ClientDiagnosticsStorage {
3154            status: "unreadable".to_owned(),
3155            database_bytes_approx: None,
3156            pending_outbox_bytes_approx: None,
3157            retained_outcome_bytes_approx: None,
3158            retained_outcome_entries: None,
3159            blob_cache_bytes_approx: None,
3160            pressure_reason_code: None,
3161        })
3162    }
3163
3164    pub fn drain_change_batches(&mut self) -> Vec<ClientChangeBatch> {
3165        self.change_queue.drain(..).collect()
3166    }
3167
3168    pub fn drain_sync_intents(&mut self) -> Vec<SyncIntent> {
3169        let intents = self.sync_intent_queue.drain(..).collect();
3170        if self.transport_enabled {
3171            intents
3172        } else {
3173            vec![SyncIntent::None]
3174        }
3175    }
3176
3177    /// §8.7 host announcement: the number of `scopes` events received since
3178    /// the last drain. The event changes no sync state; the host surfaces each
3179    /// one to the app, whose discovery registers the newly readable units.
3180    pub fn drain_scopes_changed(&mut self) -> usize {
3181        std::mem::take(&mut self.scopes_changed)
3182    }
3183
3184    fn schedule_background_retry(&mut self) -> u64 {
3185        let delay_ms = self.retry_delay_ms;
3186        self.sync_intent_queue
3187            .push_back(SyncIntent::Background { delay_ms });
3188        self.retry_delay_ms = (self.retry_delay_ms * 2).min(30_000);
3189        self.round_retry_delay_ms = Some(delay_ms);
3190        delay_ms
3191    }
3192
3193    fn reset_background_retry(&mut self) {
3194        self.retry_delay_ms = 250;
3195    }
3196
3197    fn retryable_transport_code(code: &str) -> bool {
3198        code == "transport.failed"
3199            || code == "transport.timeout"
3200            || code == "transport.unavailable"
3201            || code == "sync.transport_failed"
3202    }
3203
3204    /// §1.8: the transport reported a refused or unreachable endpoint. The
3205    /// native transport says `connect` (DNS failure, connection refused,
3206    /// reset, or aborted); a browser fetch transport says `network`. Repeating
3207    /// the request on a backoff repairs neither, so no retry is scheduled.
3208    fn unreachable_transport(details: &Option<serde_json::Value>) -> bool {
3209        matches!(
3210            details
3211                .as_ref()
3212                .and_then(|details| details.get("causeKind"))
3213                .and_then(serde_json::Value::as_str),
3214            Some("connect" | "network")
3215        )
3216    }
3217
3218    /// §1.5: a segment fetch that fails inside an otherwise complete round is
3219    /// invisible to the server (the round response arrived whole), so its code
3220    /// rides the following requests like a socket-round failure. A code the
3221    /// frame grammar rejects is not reportable and the previous code stays.
3222    fn segment_fetch_failed(&mut self, error: TransportError) -> SectionError {
3223        if ssp2::model::valid_round_failure_code(&error.code) {
3224            self.round_failure = Some(error.code.clone());
3225        }
3226        SectionError::Transport(error)
3227    }
3228
3229    /// A failed round worth a background retry: a transport failure, or a
3230    /// §10.2 code the catalog marks retryable (clients may hardcode the
3231    /// catalog metadata, §10.1). Matches the TypeScript core, which retries
3232    /// on the server's `retryable` flag.
3233    fn retryable_failure_code(code: &str) -> bool {
3234        Self::retryable_transport_code(code)
3235            || matches!(
3236                code,
3237                "sync.auth_required"
3238                    | "sync.auth_lease_required"
3239                    | "sync.auth_lease_revoked"
3240                    | "sync.internal_error"
3241                    | "sync.idempotency_cache_miss"
3242                    | "sync.schema_not_ready"
3243                    | "sync.segment_expired"
3244                    | "sync.rate_limited"
3245                    | "sync.websocket_connection_limit"
3246                    | "sync.idle_timeout"
3247            )
3248    }
3249
3250    fn diagnostic_code(code: &str) -> String {
3251        let valid = !code.is_empty()
3252            && code.len() <= 96
3253            && code.contains('.')
3254            && code
3255                .bytes()
3256                .next()
3257                .is_some_and(|byte| byte.is_ascii_lowercase())
3258            && code.bytes().all(|byte| {
3259                byte.is_ascii_lowercase()
3260                    || byte.is_ascii_digit()
3261                    || matches!(byte, b'.' | b'_' | b'-')
3262            });
3263        if valid {
3264            code.to_owned()
3265        } else {
3266            "client.unknown_failure".to_owned()
3267        }
3268    }
3269
3270    fn set_sync_needed(&mut self, value: bool, interactive: bool) {
3271        if self.sync_needed != value {
3272            if self.begin_observation("syncular_status").is_ok() {
3273                self.sync_needed = value;
3274                let batch = ChangeAccumulator {
3275                    status: true,
3276                    ..ChangeAccumulator::default()
3277                };
3278                if self.finish_observation("syncular_status", batch).is_err() {
3279                    self.rollback_observation("syncular_status");
3280                }
3281            } else {
3282                self.sync_needed = value;
3283            }
3284        }
3285        if value && interactive {
3286            self.sync_intent_queue.push_back(SyncIntent::Interactive);
3287        }
3288    }
3289
3290    /// A unit-store result: a row that does not parse is `sync.local_corrupt`;
3291    /// every other SQLite error keeps its storage classification.
3292    fn stored<T>(&self, result: rusqlite::Result<T>) -> Result<T, String> {
3293        result.map_err(|error| match error {
3294            rusqlite::Error::FromSqlConversionFailure(..) => {
3295                "sync.local_corrupt: persisted unit state is invalid".to_owned()
3296            }
3297            error => Self::sqlite_failure(&self.storage_failure, error),
3298        })
3299    }
3300
3301    fn sqlite_failure(
3302        storage_failure: &RefCell<Option<QueryReadFailure>>,
3303        error: rusqlite::Error,
3304    ) -> String {
3305        let message = error.to_string();
3306        let failure = QueryReadFailure::from(error);
3307        let code = failure.code;
3308        if code.is_some() && storage_failure.borrow().is_none() {
3309            *storage_failure.borrow_mut() = Some(failure);
3310        }
3311        match code {
3312            Some(code) => format!("{code}: {message}"),
3313            None => message,
3314        }
3315    }
3316
3317    fn begin_observation(&self, name: &str) -> Result<(), String> {
3318        if let Some(failure) = &self.coalesced_failure {
3319            return Err(failure.clone());
3320        }
3321        // SQLite rolls a whole transaction back on some I/O and capacity
3322        // errors; later frames must not commit on their own after that.
3323        if self.coalesced.is_some() && self.conn.is_autocommit() {
3324            return Err("the coalesced apply transaction was rolled back".to_owned());
3325        }
3326        self.conn
3327            .execute_batch(&format!("SAVEPOINT {name}"))
3328            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
3329    }
3330
3331    /// SPEC §1.4 rule 4: open a coalesced apply transaction unless a caller
3332    /// already owns one.
3333    fn begin_coalesced_apply(&mut self) -> Result<(), String> {
3334        if self.coalesced.is_some() || !self.conn.is_autocommit() {
3335            return Ok(());
3336        }
3337        self.conn
3338            .execute_batch("BEGIN")
3339            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3340        self.coalesced = Some(CoalescedApply {
3341            rows: 0,
3342            rollback: self.capture(),
3343        });
3344        Ok(())
3345    }
3346
3347    fn capture(&self) -> Rollback {
3348        Rollback {
3349            outbox: self.outbox.clone(),
3350            failed_commits: self.failed_commits.clone(),
3351            conflicts: self.conflicts.clone(),
3352            rejections: self.rejections.clone(),
3353            schema_floor: self.schema_floor.clone(),
3354            lease_state: self.lease_state.clone(),
3355            stopped: self.stopped,
3356            upgrading: self.upgrading,
3357            sync_needed: self.sync_needed,
3358            active_round: self.active_round,
3359            overlay_dirty: self.overlay_dirty.snapshot(),
3360            change_queue: self.change_queue.len(),
3361            sync_intent_queue: self.sync_intent_queue.len(),
3362            last_change: self.last_change.clone(),
3363        }
3364    }
3365
3366    fn restore(&mut self, rollback: Rollback) {
3367        self.outbox = rollback.outbox;
3368        self.failed_commits = rollback.failed_commits;
3369        self.conflicts = rollback.conflicts;
3370        self.rejections = rollback.rejections;
3371        self.schema_floor = rollback.schema_floor;
3372        self.lease_state = rollback.lease_state;
3373        self.stopped = rollback.stopped;
3374        self.upgrading = rollback.upgrading;
3375        self.sync_needed = rollback.sync_needed;
3376        self.active_round = rollback.active_round;
3377        self.overlay_dirty.restore(rollback.overlay_dirty);
3378        self.change_queue.truncate(rollback.change_queue);
3379        self.sync_intent_queue.truncate(rollback.sync_intent_queue);
3380        self.last_change = rollback.last_change;
3381    }
3382
3383    /// Account the rows of the next frame or block. A unit that would take the
3384    /// coalesced transaction past the bound commits the earlier units first.
3385    fn coalesce_unit(&mut self, rows: usize) -> Result<(), String> {
3386        let Some(open) = &self.coalesced else {
3387            return Ok(());
3388        };
3389        if open.rows > 0 && open.rows + rows > COALESCED_APPLY_ROWS {
3390            self.commit_coalesced_apply()?;
3391            self.begin_coalesced_apply()?;
3392        }
3393        if let Some(open) = &mut self.coalesced {
3394            open.rows += rows;
3395        }
3396        Ok(())
3397    }
3398
3399    /// COMMIT the coalesced transaction. A failed COMMIT rolls every coalesced
3400    /// frame back, restores the in-memory state of its `BEGIN`, drops the
3401    /// change batches and sync intents queued since, and fails the rest of the
3402    /// response.
3403    fn commit_coalesced_apply(&mut self) -> Result<(), String> {
3404        let Some(open) = self.coalesced.take() else {
3405            return Ok(());
3406        };
3407        let committed = if self.conn.is_autocommit() {
3408            Err("the coalesced apply transaction was rolled back".to_owned())
3409        } else {
3410            self.conn
3411                .execute_batch("COMMIT")
3412                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
3413        };
3414        let Err(error) = committed else {
3415            return Ok(());
3416        };
3417        if !self.conn.is_autocommit() {
3418            if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
3419                let _ = Self::sqlite_failure(&self.storage_failure, rollback);
3420            }
3421        }
3422        self.restore(open.rollback);
3423        self.coalesced_failure = Some(error.clone());
3424        Err(error)
3425    }
3426
3427    /// End the response's coalescing: COMMIT what is open and clear a failure.
3428    fn finish_coalesced_apply(&mut self) -> Result<(), String> {
3429        let committed = self.commit_coalesced_apply();
3430        match self.coalesced_failure.take() {
3431            Some(failure) => Err(failure),
3432            None => committed,
3433        }
3434    }
3435
3436    fn rollback_observation(&self, name: &str) {
3437        if let Err(error) = self
3438            .conn
3439            .execute_batch(&format!("ROLLBACK TO {name}; RELEASE {name}"))
3440        {
3441            if let Some(failure) = self.storage_failure.borrow_mut().as_mut() {
3442                if failure.rollback_failure.is_none() {
3443                    failure.rollback_failure = Some(Box::new(QueryReadFailure::from(error)));
3444                }
3445            }
3446        }
3447    }
3448
3449    fn finish_observation(
3450        &mut self,
3451        name: &str,
3452        mut batch: ChangeAccumulator,
3453    ) -> Result<(), String> {
3454        #[cfg(feature = "bench-internals")]
3455        let _phase = self.benchmark_phases.start(Phase::ObservationCommit);
3456        if self.failed_commits.iter().any(|(commit, _)| {
3457            commit
3458                .ops
3459                .iter()
3460                .any(|operation| batch.tables.contains_key(&operation.table))
3461        }) {
3462            batch.outcomes = true;
3463        }
3464        if !batch.touched() {
3465            self.conn
3466                .execute_batch(&format!("RELEASE {name}"))
3467                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3468            return Ok(());
3469        }
3470        let revision = self
3471            .local_revision()?
3472            .checked_add(1)
3473            .ok_or_else(|| "local revision exhausted u64".to_owned())?;
3474        self.conn
3475            .execute(
3476                "INSERT OR REPLACE INTO _syncular_meta(key, value) VALUES (?1, ?2)",
3477                rusqlite::params![LOCAL_REVISION_KEY, revision.to_string()],
3478            )
3479            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3480        let status = batch.status.then(|| self.status_snapshot());
3481        let event = ClientChangeBatch {
3482            revision: revision.to_string(),
3483            tables: batch
3484                .tables
3485                .into_iter()
3486                .map(|(table, scope_keys)| TableChange {
3487                    table,
3488                    scope_keys: scope_keys.map(|keys| keys.into_iter().collect()),
3489                })
3490                .collect(),
3491            windows: batch
3492                .windows
3493                .into_iter()
3494                .map(|(base_key, units)| WindowChange {
3495                    base_key,
3496                    units: units.into_iter().collect(),
3497                })
3498                .collect(),
3499            status,
3500            conflicts_changed: batch.conflicts,
3501            rejections_changed: batch.rejections,
3502            outcomes_changed: batch.outcomes,
3503            applied_commits: Vec::new(),
3504        };
3505        self.conn
3506            .execute_batch(&format!("RELEASE {name}"))
3507            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3508        let mut diagnostic_tables = event
3509            .tables
3510            .iter()
3511            .map(|entry| entry.table.clone())
3512            .collect::<BTreeSet<_>>()
3513            .into_iter()
3514            .collect::<Vec<_>>();
3515        // §7.6: the selected variables of the changed window bases.
3516        let mut diagnostic_windows = event
3517            .windows
3518            .iter()
3519            .map(|entry| {
3520                entry
3521                    .base_key
3522                    .split('\0')
3523                    .next()
3524                    .unwrap_or_default()
3525                    .to_owned()
3526            })
3527            .collect::<BTreeSet<_>>()
3528            .into_iter()
3529            .collect::<Vec<_>>();
3530        let domains_truncated = diagnostic_tables.len() > MAX_DIAGNOSTIC_DOMAINS
3531            || diagnostic_windows.len() > MAX_DIAGNOSTIC_DOMAINS;
3532        diagnostic_tables.truncate(MAX_DIAGNOSTIC_DOMAINS);
3533        diagnostic_windows.truncate(MAX_DIAGNOSTIC_DOMAINS);
3534        self.last_change = Some(DiagnosticLastChange {
3535            revision: event.revision.clone(),
3536            recorded_at_ms: self.clock_now_ms(),
3537            tables: diagnostic_tables,
3538            windows: diagnostic_windows,
3539            domains_truncated,
3540            status_changed: event.status.is_some(),
3541            conflicts_changed: event.conflicts_changed,
3542            rejections_changed: event.rejections_changed,
3543            outcomes_changed: event.outcomes_changed,
3544        });
3545        self.change_queue.push_back(event);
3546        Ok(())
3547    }
3548
3549    fn record_scope_map(
3550        &self,
3551        batch: &mut ChangeAccumulator,
3552        table_name: &str,
3553        scopes: &[(String, Vec<String>)],
3554    ) {
3555        let Some(table) = self.schema.table(table_name) else {
3556            return;
3557        };
3558        // A bulk scoped delete whose map leaves a declared variable unnamed
3559        // cannot list the deleted rows' keys for it: the change is table-wide.
3560        if table.scope_variables.iter().any(|scope| {
3561            !scopes
3562                .iter()
3563                .any(|(variable, _)| *variable == scope.variable)
3564        }) {
3565            batch.table(table_name);
3566            return;
3567        }
3568        for (variable, values) in scopes {
3569            let Some(scope) = table
3570                .scope_variables
3571                .iter()
3572                .find(|scope| &scope.variable == variable)
3573            else {
3574                continue;
3575            };
3576            for value in values {
3577                batch.scope(table_name, format!("{}:{value}", scope.prefix));
3578            }
3579        }
3580    }
3581
3582    /// Record a row's current scope keys from the base or visible table.
3583    fn record_row_scopes(
3584        &self,
3585        batch: &mut ChangeAccumulator,
3586        table_name: &str,
3587        row_id: &str,
3588        base: bool,
3589    ) -> bool {
3590        let Some(table) = self.schema.table(table_name) else {
3591            return false;
3592        };
3593        if table.scope_variables.is_empty() {
3594            return false;
3595        }
3596        let columns = table
3597            .scope_variables
3598            .iter()
3599            .map(|scope| quote_ident(&scope.column))
3600            .collect::<Vec<_>>()
3601            .join(", ");
3602        let full_table = if base {
3603            base_table(table_name)
3604        } else {
3605            visible_table(table_name)
3606        };
3607        let sql = format!(
3608            "SELECT {columns} FROM {full_table} WHERE {} LIMIT 1",
3609            row_id_predicate(table)
3610        );
3611        let Ok(mut stmt) = self.conn.prepare_cached(&sql) else {
3612            return false;
3613        };
3614        let values = stmt.query_row(rusqlite::params![row_id], |row| {
3615            let mut values = Vec::with_capacity(table.scope_variables.len());
3616            for index in 0..table.scope_variables.len() {
3617                values.push(row.get::<_, Option<String>>(index)?);
3618            }
3619            Ok(values)
3620        });
3621        let Ok(values) = values else {
3622            return false;
3623        };
3624        let mut recorded = false;
3625        for (scope, value) in table.scope_variables.iter().zip(values) {
3626            if let Some(value) = value {
3627                batch.scope(table_name, format!("{}:{value}", scope.prefix));
3628                recorded = true;
3629            }
3630        }
3631        recorded
3632    }
3633
3634    fn record_commit_changes(
3635        &self,
3636        batch: &mut ChangeAccumulator,
3637        tables: &[String],
3638        changes: &[ssp2::model::Change],
3639    ) {
3640        #[cfg(feature = "bench-internals")]
3641        let _phase = self.benchmark_phases.start(Phase::ObservationPrepare);
3642        for change in changes {
3643            let Some(table_name) = tables.get(change.table_index as usize) else {
3644                continue;
3645            };
3646            let mut precise = self.record_row_scopes(batch, table_name, &change.row_id, true);
3647            if let Some(table) = self.schema.table(table_name) {
3648                for (variable, value) in &change.scopes {
3649                    if let Some(scope) = table
3650                        .scope_variables
3651                        .iter()
3652                        .find(|scope| &scope.variable == variable)
3653                    {
3654                        batch.scope(table_name, format!("{}:{value}", scope.prefix));
3655                        precise = true;
3656                    }
3657                }
3658            }
3659            if !precise {
3660                batch.table(table_name);
3661            }
3662        }
3663    }
3664
3665    /// Whether a base or visible row matches the scope predicate; `false`
3666    /// without a local scope-column mapping. A SQLite failure is an error.
3667    fn scoped_rows_exist(
3668        &self,
3669        table_name: &str,
3670        effective: &[(String, Vec<String>)],
3671        owners: &[Vec<(String, Vec<String>)>],
3672    ) -> Result<bool, String> {
3673        let Some(table) = self.schema.table(table_name) else {
3674            return Ok(false);
3675        };
3676        let Ok(Some((predicate, params))) = local_scope_predicate(table, effective, owners) else {
3677            return Ok(false);
3678        };
3679        for source in [base_table(table_name), visible_table(table_name)] {
3680            if self
3681                .conn
3682                .query_row(
3683                    &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {predicate})"),
3684                    rusqlite::params_from_iter(&params),
3685                    |row| row.get::<_, bool>(0),
3686                )
3687                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
3688            {
3689                return Ok(true);
3690            }
3691        }
3692        Ok(false)
3693    }
3694
3695    /// §5.6 first-page clear of `unit_id`'s pair: delete its rows except
3696    /// those another owner covers (§3.3), failing closed without a local
3697    /// scope-column mapping. Owners are read only when a row matches, inside
3698    /// the caller's transaction. Returns whether rows matched.
3699    fn clear_scope(
3700        &mut self,
3701        table: &TableSchema,
3702        effective: &[(String, Vec<String>)],
3703        unit_id: &str,
3704    ) -> Result<bool, SectionError> {
3705        let storage = |message| SectionError::Abort("storage.failed".to_owned(), message);
3706        let matched = self
3707            .scoped_rows_exist(&table.name, effective, &[])
3708            .map_err(storage)?;
3709        let owners = if matched {
3710            unit_owners(
3711                &self.stored(load_units(&self.conn)).map_err(storage)?,
3712                table,
3713                unit_id,
3714            )
3715        } else {
3716            Vec::new()
3717        };
3718        let changed = matched
3719            && (owners.is_empty()
3720                || self
3721                    .scoped_rows_exist(&table.name, effective, &owners)
3722                    .map_err(storage)?);
3723        if !self
3724            .purge_scope_rows(&table.name, effective, &owners)
3725            .map_err(storage)?
3726        {
3727            return Err(SectionError::FailClosed);
3728        }
3729        Ok(changed)
3730    }
3731
3732    /// §7.4.5: true while a schema-bump reset + first re-bootstrap runs.
3733    pub fn upgrading(&self) -> bool {
3734        self.upgrading
3735    }
3736
3737    /// §7.4.2 "app ships new code": swap to a NEW generated schema while
3738    /// keeping this client's local database (identity, outbox, tables). The
3739    /// §7.4.1 marker check then fires the wipe/re-bootstrap flow when the
3740    /// version increases. Recreation runs the same marker guard as opening
3741    /// a durable replica, holds the guard and every write it authorizes under
3742    /// one write transaction, and preserves the current client (compiled
3743    /// schema, outbox, tables) on refusal.
3744    pub fn recreate_with_schema(&mut self, schema_json: &Value) -> Result<(), String> {
3745        let new_schema = parse_schema_json(schema_json)?;
3746        validate_authority_reads(&new_schema, &self.limits.authority_reads)?;
3747        self.conn
3748            .execute_batch("BEGIN IMMEDIATE")
3749            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3750        let prior_schema = std::mem::replace(&mut self.schema, new_schema);
3751        // §7.4.3/§7.4.4: a reset inside this transaction mutates in-memory
3752        // state (dropped outbox commits, rejections, upgrade and floor flags,
3753        // overlay dirtiness) and appends change/intent batches. A COMMIT that
3754        // fails after the reset succeeded rolls the database back, so every
3755        // one of those fields is captured here and restored with it.
3756        let rollback = self.capture();
3757        let guarded = (|| -> Result<(), String> {
3758            // §7.4.2: the marker is re-read under the write lock the caller
3759            // holds, so a replica another process upgraded meanwhile refuses
3760            // this recreation instead of having its marker and tables
3761            // downgraded.
3762            let marker = read_local_schema_version(&self.conn, self.schema.version)?;
3763            self.create_bookkeeping_tables()?;
3764            match marker {
3765                Some(version) if version < self.schema.version => self.run_schema_reset()?,
3766                None => {
3767                    self.create_synced_tables()?;
3768                    self.persist_schema_version()?;
3769                }
3770                _ => {}
3771            }
3772            // §7.4.2: recreation IS the boot for the in-memory core, so reload the
3773            // durable outbox the same-version path is about to replay. Only the
3774            // outbox is reloaded: the recovery below reads it, and the caller's
3775            // rollback snapshot already covers this field.
3776            self.outbox = self.load_outbox()?.into();
3777            // §7.1/§7.4.4: recover intrinsically codec-invalid persisted values at
3778            // this guarded boundary once the schema and marker agree. A bump went
3779            // through `run_schema_reset`, which performs the same recovery after
3780            // it creates the new physical tables and before its final replay.
3781            self.drop_unencodable_outbox(false, None)?;
3782            if marker == Some(self.schema.version)
3783                && (!self.outbox.is_empty()
3784                    || !self.failed_commits.is_empty()
3785                    || self.has_acknowledged_rows()?)
3786            {
3787                self.overlay_dirty.set(true);
3788                self.rebuild_overlay()?;
3789            }
3790            self.prune_unknown_units(false)?;
3791            self.save_unit_scope_schema()?;
3792            // The conformance recreate is the in-memory equivalent of reopening
3793            // a durable client. Apply the same startup catch-up contract even
3794            // when the schema itself did not change.
3795            self.enqueue_startup_sync_if_needed()?;
3796            Ok(())
3797        })();
3798        let outcome = match guarded {
3799            Ok(()) => self
3800                .conn
3801                .execute_batch("COMMIT")
3802                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error)),
3803            Err(error) => Err(error),
3804        };
3805        if let Err(error) = outcome {
3806            if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
3807                // Keep a failed rollback's typed storage classification.
3808                let _ = Self::sqlite_failure(&self.storage_failure, rollback);
3809            }
3810            self.schema = prior_schema;
3811            self.restore(rollback);
3812            // Prepared insert statements were derived from the reverted schema.
3813            self.insert_sql.borrow_mut().clear();
3814            return Err(error);
3815        }
3816        if !self.security_preflight {
3817            self.drain_pending_evictions()?;
3818        }
3819        Ok(())
3820    }
3821
3822    fn enqueue_startup_sync_if_needed(&mut self) -> Result<(), String> {
3823        let startup_work = !self.stopped
3824            && (!self.outbox.is_empty()
3825                || self.has_acknowledged_rows()?
3826                || !self.load_pending_evictions()?.is_empty()
3827                || self
3828                    .stored(load_units(&self.conn))?
3829                    .iter()
3830                    .any(|unit| unit.status == UnitStatus::Active));
3831        if startup_work {
3832            self.sync_needed = true;
3833            self.sync_intent_queue.push_back(SyncIntent::Interactive);
3834        }
3835        Ok(())
3836    }
3837
3838    /// A native client persists schema-floor stops across process restarts.
3839    /// Once the running app already satisfies that floor, the persisted stop
3840    /// is only stale evidence from an older server round (for example, the
3841    /// server was restarted after catching up to an app that was ahead).
3842    /// Clear it and let the normal startup pull re-negotiate. If the server is
3843    /// still incompatible it will return the floor again in that first round.
3844    fn clear_satisfied_persisted_schema_floor(&mut self) -> Result<(), String> {
3845        let satisfied = self
3846            .schema_floor
3847            .as_ref()
3848            .and_then(|floor| floor.required_schema_version)
3849            .is_some_and(|required| self.schema.version >= required);
3850        if !satisfied {
3851            return Ok(());
3852        }
3853        self.delete_meta(SCHEMA_FLOOR_KEY)?;
3854        self.schema_floor = None;
3855        self.stopped = false;
3856        Ok(())
3857    }
3858
3859    /// RFC 0005 D1: discard an orphan or stale container at boot. The reset
3860    /// path already swept it; this covers a same-version or fresh-install open,
3861    /// which runs no reset and therefore no sweep — plus the D7 TTL, which only
3862    /// an aware binary can apply.
3863    fn reconcile_previous_version_at_boot(&self) -> Result<(), String> {
3864        let Some(replica_path) = self.previous_version_replica_path() else {
3865            return Ok(());
3866        };
3867        let max_age_ms = self
3868            .previous_version
3869            .filter(|config| config.enabled)
3870            .map(|config| config.max_age_ms);
3871        reconcile_previous_version_at_boot(
3872            &self.conn,
3873            &replica_path,
3874            self.schema.version,
3875            max_age_ms,
3876            self.clock_now_ms(),
3877        )
3878    }
3879
3880    /// RFC 0005 D3: the sibling container path, or `None` when the replica has
3881    /// no file — an in-memory replica returns an empty path, and a `file:` URI
3882    /// has no plain sibling path — so the feature resolves as off there.
3883    fn previous_version_replica_path(&self) -> Option<String> {
3884        let path = self.conn.path()?;
3885        if path.is_empty() || path.starts_with("file:") {
3886            return None;
3887        }
3888        Some(path.to_owned())
3889    }
3890
3891    /// §7.4.3 reset: whole-database local reset EXCEPT the outbox, clientId,
3892    /// and leaseState. Drops/recreates every synced table from the new
3893    /// schema, resets unit sync state (keeping registrations), clears
3894    /// the schema-floor stop state, rewrites the marker, drops outbox commits
3895    /// that cannot re-encode (§7.4.4), and replays the survivors on top.
3896    fn run_schema_reset(&mut self) -> Result<(), String> {
3897        self.begin_observation("syncular_schema_reset")?;
3898        let rollback = self.capture();
3899        let mut batch = ChangeAccumulator::default();
3900        let result = self
3901            .run_schema_reset_observed(&mut batch, true)
3902            .and_then(|()| self.finish_observation("syncular_schema_reset", batch));
3903        if let Err(error) = result {
3904            self.rollback_observation("syncular_schema_reset");
3905            self.restore(rollback);
3906            return Err(error);
3907        }
3908        Ok(())
3909    }
3910
3911    fn run_log_epoch_reset(&mut self, log_epoch: &str) -> Result<Vec<String>, String> {
3912        if self.get_meta(LOG_EPOCH_KEY)?.is_none() {
3913            self.begin_observation("syncular_epoch_acquisition")?;
3914            let rollback = self.capture();
3915            if let Err(error) = self.conn.execute(
3916                "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
3917                rusqlite::params![LOG_EPOCH_KEY, log_epoch],
3918            ) {
3919                self.rollback_observation("syncular_epoch_acquisition");
3920                return Err(Self::sqlite_failure(&self.storage_failure, error));
3921            }
3922            self.sync_needed = true;
3923            let batch = ChangeAccumulator {
3924                status: true,
3925                ..ChangeAccumulator::default()
3926            };
3927            if let Err(error) = self.finish_observation("syncular_epoch_acquisition", batch) {
3928                self.rollback_observation("syncular_epoch_acquisition");
3929                self.restore(rollback);
3930                return Err(error);
3931            }
3932            self.sync_intent_queue.push_back(SyncIntent::Interactive);
3933            return Ok(Vec::new());
3934        }
3935        let resets = self
3936            .stored(load_units(&self.conn))?
3937            .into_iter()
3938            .map(|unit| unit.id)
3939            .collect();
3940        self.begin_observation("syncular_log_epoch_reset")?;
3941        let rollback = self.capture();
3942        let mut batch = ChangeAccumulator::default();
3943        let result = self
3944            .run_schema_reset_observed(&mut batch, false)
3945            .and_then(|()| {
3946                self.conn
3947                    .execute(
3948                        "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
3949                        rusqlite::params![LOG_EPOCH_KEY, log_epoch],
3950                    )
3951                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3952                self.sync_needed = true;
3953                batch.status = true;
3954                self.finish_observation("syncular_log_epoch_reset", batch)
3955            });
3956        if let Err(error) = result {
3957            self.rollback_observation("syncular_log_epoch_reset");
3958            self.restore(rollback);
3959            return Err(error);
3960        }
3961        self.sync_intent_queue.push_back(SyncIntent::Interactive);
3962        Ok(resets)
3963    }
3964
3965    fn run_schema_reset_observed(
3966        &mut self,
3967        batch: &mut ChangeAccumulator,
3968        capture: bool,
3969    ) -> Result<(), String> {
3970        // §7.4.2: the reset is the only path that wipes local tables for a
3971        // version bump, and it rewrites the marker last. Re-read the marker
3972        // under this reset's transaction: a replica another process upgraded
3973        // meanwhile refuses here instead of being reset to this client's
3974        // older schema.
3975        read_local_schema_version(&self.conn, self.schema.version)?;
3976        // §7.4.3: a schema bump keeps every table whose generation equals the
3977        // persisted one; the log-epoch reset keeps nothing. The previous
3978        // declaration evidence decides which units cover only kept tables.
3979        let mut kept = BTreeSet::new();
3980        let mut previous_declarations: Option<BTreeMap<String, BTreeMap<String, Value>>> = None;
3981        if capture {
3982            let persisted: BTreeMap<String, String> = self
3983                .get_meta(TABLE_GENERATIONS_KEY)?
3984                .map(|raw| serde_json::from_str(&raw))
3985                .transpose()
3986                .map_err(|_| "sync.local_corrupt: invalid persisted table generations".to_owned())?
3987                .unwrap_or_default();
3988            kept = self
3989                .schema
3990                .tables
3991                .iter()
3992                .filter(|table| {
3993                    table.generation.is_some()
3994                        && table.generation.as_ref() == persisted.get(&table.name)
3995                })
3996                .map(|table| table.name.clone())
3997                .collect();
3998            previous_declarations = self
3999                .get_meta("subscriptionScopeSchema")?
4000                .map(|raw| serde_json::from_str(&raw))
4001                .transpose()
4002                .map_err(|_| "sync.local_corrupt: invalid persisted scope schema".to_owned())?;
4003        }
4004        // Kept tables keep both halves, their FTS projections with the FTS5
4005        // shadow tables and id mapping, and their row-level bookkeeping.
4006        let mut kept_objects = BTreeSet::new();
4007        for table in self
4008            .schema
4009            .tables
4010            .iter()
4011            .filter(|table| kept.contains(&table.name))
4012        {
4013            kept_objects.insert(table.name.clone());
4014            kept_objects.insert(format!("_syncular_base_{}", table.name));
4015            for index in &table.fts_indexes {
4016                kept_objects.insert(index.name.clone());
4017                for shadow in ["data", "idx", "content", "docsize", "config"] {
4018                    kept_objects.insert(format!("{}_{shadow}", index.name));
4019                }
4020            }
4021        }
4022        let kept_json = serde_json::to_string(&kept).expect("table names serialize");
4023        for sql in [
4024            "DELETE FROM _syncular_acked_rows WHERE tbl NOT IN (SELECT value FROM json_each(?1))",
4025            "DELETE FROM _syncular_row_deliveries WHERE tbl NOT IN (SELECT value FROM json_each(?1))",
4026        ] {
4027            self.conn
4028                .execute(sql, [&kept_json])
4029                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4030        }
4031        self.cancel_sync_round();
4032        self.upgrading = true;
4033        batch.status = true;
4034        // RFC 0005 D6: classify the pending outbox against the NEW compiled
4035        // schema BEFORE the wipe, and record the advisory audit. This pass
4036        // drops nothing, so the audit sees the original pending intent; the
4037        // §7.4.4 send-time drop and the intrinsic value recovery later in this
4038        // reset are the paths that remove a commit. The classification is
4039        // shared with the send-time drop so the two cannot disagree about what
4040        // is incompatible.
4041        if self.previous_version.is_some_and(|config| config.enabled) {
4042            let previous_version = self
4043                .get_meta(LOCAL_SCHEMA_VERSION_KEY)?
4044                .and_then(|value| value.parse::<i32>().ok())
4045                .unwrap_or(self.schema.version);
4046            let pending = self
4047                .outbox
4048                .iter()
4049                .map(|commit| PendingCommitAudit {
4050                    commit_id: &commit.client_commit_id,
4051                    operations: commit_audit_operations(commit),
4052                })
4053                .collect::<Vec<_>>();
4054            crate::previous_version::write_previous_version_audit(
4055                &self.conn,
4056                &crate::previous_version::build_previous_version_audit(
4057                    &self.schema,
4058                    &pending,
4059                    previous_version,
4060                    self.schema.version,
4061                    self.clock_now_ms(),
4062                ),
4063            )?;
4064        }
4065        // RFC 0005 D5: the container lives in its own FILE, so the reset can
4066        // never reach it. Sweep first (unconditional — also on the log-epoch
4067        // reset, which captures nothing), then capture BEFORE the wipe drops
4068        // any row. A refused capture records its reason and leaves no file.
4069        if let Some(replica_path) = self.previous_version_replica_path() {
4070            sweep_previous_version_container(&self.conn, &replica_path)?;
4071            if capture {
4072                if let Some(config) = self.previous_version.filter(|config| config.enabled) {
4073                    let previous_version = self
4074                        .get_meta(LOCAL_SCHEMA_VERSION_KEY)?
4075                        .and_then(|value| value.parse::<i32>().ok())
4076                        .unwrap_or(self.schema.version);
4077                    capture_previous_version_from_replica(
4078                        &self.conn,
4079                        &replica_path,
4080                        previous_version,
4081                        self.schema.version,
4082                        &config,
4083                        self.clock_now_ms(),
4084                    )?;
4085                }
4086            }
4087        }
4088        for table in self
4089            .schema
4090            .tables
4091            .iter()
4092            .filter(|table| !kept.contains(&table.name))
4093        {
4094            batch.table(&table.name);
4095        }
4096        for (base_key, value) in self.load_window_registry()? {
4097            batch.window(&base_key, &value);
4098        }
4099        if capture {
4100            self.prune_unknown_units(true)?;
4101        }
4102        // The per-table insert SQL is derived from the OLD column lists.
4103        self.insert_sql.borrow_mut().clear();
4104        self.overlay_dirty.set(true);
4105        // Drop every synced table (base + visible) that currently exists —
4106        // discovered from sqlite_master so a bump that adds/removes tables is
4107        // handled. Bookkeeping tables (`_syncular_outbox/_units/_meta`
4108        // and the blob cache) are preserved; base tables are `_syncular_base_*`
4109        // so they are matched explicitly, not by the bookkeeping filter.
4110        // Drop virtual tables first so SQLite removes every FTS shadow table
4111        // atomically instead of the generic discovery tearing it apart.
4112        let virtual_tables: Vec<String> = {
4113            let mut stmt = self
4114                .conn
4115                .prepare(
4116                    "SELECT name FROM sqlite_master WHERE type = 'table' AND sql LIKE 'CREATE VIRTUAL TABLE%'",
4117                )
4118                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4119            let rows = stmt
4120                .query_map([], |row| row.get::<_, String>(0))
4121                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4122            rows.filter_map(Result::ok)
4123                .filter(|name| is_synced_table_name(name) && !kept_objects.contains(name))
4124                .collect()
4125        };
4126        for name in virtual_tables {
4127            self.conn
4128                .execute_batch(&format!(
4129                    "DROP TABLE IF EXISTS {}",
4130                    quote_ident(&format!("_syncular_fts_{name}"))
4131                ))
4132                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4133            self.conn
4134                .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
4135                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4136        }
4137        let existing: Vec<String> = {
4138            let mut stmt = self
4139                .conn
4140                .prepare("SELECT name FROM sqlite_master WHERE type = 'table'")
4141                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4142            let rows = stmt
4143                .query_map([], |row| row.get::<_, String>(0))
4144                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4145            rows.filter_map(Result::ok)
4146                .filter(|name| is_synced_table_name(name) && !kept_objects.contains(name))
4147                .collect()
4148        };
4149        for name in &existing {
4150            if let Some(table) = name.strip_prefix("_syncular_base_") {
4151                batch.table(table);
4152            } else if !name.starts_with("_syncular_") {
4153                batch.table(name);
4154            }
4155        }
4156        for name in existing {
4157            self.conn
4158                .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
4159                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4160        }
4161        // Recreate the synced tables from the NEW schema.
4162        self.create_synced_tables()?;
4163        // Reset the sync state of every unit that covers a table the reset
4164        // did not keep, under the previous declarations or the new schema,
4165        // keeping the registration.
4166        let keep_units = match &previous_declarations {
4167            Some(previous) if !kept.is_empty() => self
4168                .stored(load_units(&self.conn))?
4169                .into_iter()
4170                .filter(|unit| {
4171                    previous.iter().all(|(name, old)| {
4172                        kept.contains(name) || !unit.scopes.keys().all(|v| old.contains_key(v))
4173                    }) && self
4174                        .schema
4175                        .tables
4176                        .iter()
4177                        .all(|table| kept.contains(&table.name) || !covers(table, &unit.scopes))
4178                })
4179                .map(|unit| unit.id)
4180                .collect(),
4181            _ => BTreeSet::new(),
4182        };
4183        reset_units(&self.conn, &keep_units)
4184            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4185        // The stop state is over: this client now ships a servable schema.
4186        self.stopped = false;
4187        self.schema_floor = None;
4188        self.delete_meta(SCHEMA_FLOOR_KEY)?;
4189        // Rewrite the marker LAST so a crash mid-reset re-runs the reset.
4190        self.persist_schema_version()?;
4191        self.save_unit_scope_schema()?;
4192        // §7.1/§7.4.4: the previous-version capture and audit above saw the
4193        // original pending intent; the synced tables now carry the new schema, so
4194        // recover intrinsically codec-invalid persisted values here, before the
4195        // final replay. A pending upsert that names a column the new schema
4196        // removed stays pending for the send-time `drop_incompatible_outbox`
4197        // prepass. The reset wrapper snapshots and restores the outbox,
4198        // rejections, and change queues, so a failed commit rolls this recovery
4199        // back with the rest of the reset.
4200        // §7.4.4: the enclosing reset observation owns the publication, so the
4201        // recovery contributes its rejection, outcome, and table domains to it.
4202        self.drop_unencodable_outbox(false, Some(batch))?;
4203        // Re-apply the surviving outbox optimistically over the empty tables.
4204        self.rebuild_overlay()?;
4205        Ok(())
4206    }
4207
4208    /// §7.4.4 send-time prepass: a persisted upsert whose values reference a
4209    /// column the current schema lacks (or a removed table) cannot be encoded
4210    /// (`commit_audit_operations` inspects upserts, so a value-free delete stays
4211    /// encodable and is validated by the server), nor can an insert-intent
4212    /// upsert that lacks a non-nullable column (§7.1, the audit's
4213    /// `missing-required-column`). Drop the commit, undo
4214    /// its optimistic projection, and raise a client-local
4215    /// `sync.outbox_incompatible` rejection.
4216    fn drop_incompatible_outbox(&mut self) -> Result<bool, String> {
4217        let incompatible = self
4218            .outbox
4219            .iter()
4220            .filter(|commit| {
4221                first_incompatibility(&self.schema, &commit_audit_operations(commit)).is_some()
4222            })
4223            .cloned()
4224            .collect::<Vec<_>>();
4225        if incompatible.is_empty() {
4226            return Ok(false);
4227        }
4228        self.drop_outbox_commits(
4229            &incompatible,
4230            OUTBOX_INCOMPATIBLE_CODE,
4231            "the persisted commit cannot encode under the current schema",
4232        )?;
4233        Ok(true)
4234    }
4235
4236    /// §7.4.4 incompatible-commit path, also taken by a refused blob upload
4237    /// (§5.9.7) with the server's code: reject every operation of `commits`,
4238    /// remove them from the outbox, and undo their optimistic rows.
4239    fn drop_outbox_commits(
4240        &mut self,
4241        incompatible: &[Arc<OutboxCommit>],
4242        code: &str,
4243        message: &str,
4244    ) -> Result<(), String> {
4245        // §7.4.4: one observation covers the durable outcome writes, the outbox
4246        // removal, the in-memory drop, and the overlay rebuild, so a storage
4247        // fault leaves the replica and the client unchanged and publishes no
4248        // change; a successful drop publishes exactly one batch carrying the
4249        // rejections, outcomes, status, and the tables whose projection changed.
4250        let rollback = self.capture();
4251        let owns_transaction = self.conn.is_autocommit();
4252        self.begin_observation("syncular_outbox_drop")?;
4253        let mut batch = ChangeAccumulator::default();
4254        let dropped = (|| -> Result<(), String> {
4255            let mut rejections = Vec::new();
4256            for commit in incompatible {
4257                let results = commit
4258                    .ops
4259                    .iter()
4260                    .enumerate()
4261                    .map(|(op_index, operation)| {
4262                        let rejection = RejectionRecord {
4263                            client_commit_id: commit.client_commit_id.clone(),
4264                            op_index: op_index as i32,
4265                            code: code.to_owned(),
4266                            message: message.to_owned(),
4267                            retryable: false,
4268                            details: None,
4269                            operation: Some(CommitOperation::from(operation)),
4270                        };
4271                        rejections.push(rejection.clone());
4272                        CommitOperationOutcome::Error { rejection }
4273                    })
4274                    .collect::<Vec<_>>();
4275                self.persist_commit_outcome(
4276                    &commit.client_commit_id,
4277                    CommitOutcomeStatus::Rejected,
4278                    &results,
4279                    Some(&commit.ops),
4280                )?;
4281                self.delete_outbox_persisted(&commit.client_commit_id)?;
4282                // A removed table has no local projection to invalidate; every
4283                // surviving table the dropped commit touched does.
4284                for operation in &commit.ops {
4285                    if self.schema.table(&operation.table).is_some() {
4286                        batch.table(&operation.table);
4287                    }
4288                }
4289            }
4290            self.prune_commit_outcomes()?;
4291            let incompatible_ids = incompatible
4292                .iter()
4293                .map(|commit| commit.client_commit_id.as_str())
4294                .collect::<BTreeSet<_>>();
4295            self.outbox
4296                .retain(|commit| !incompatible_ids.contains(commit.client_commit_id.as_str()));
4297            self.rejections.extend(rejections);
4298            // The dropped commits' purely-optimistic rows are undone: the
4299            // visible projection is re-derived from the surviving base + outbox.
4300            self.overlay_dirty.set(true);
4301            self.rebuild_overlay()?;
4302            batch.status = true;
4303            batch.rejections = true;
4304            batch.outcomes = true;
4305            Ok(())
4306        })();
4307        let result = dropped.and_then(|()| self.finish_observation("syncular_outbox_drop", batch));
4308        if let Err(error) = result {
4309            // An outer RELEASE is the COMMIT. Roll back that transaction
4310            // directly: retrying RELEASE after ROLLBACK TO still needs the
4311            // reader's lock. A nested observation only rolls back its savepoint.
4312            if owns_transaction {
4313                if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
4314                    let mut failure = self.storage_failure.borrow_mut();
4315                    let failure =
4316                        failure.get_or_insert_with(|| QueryReadFailure::from(error.clone()));
4317                    failure.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
4318                }
4319            } else {
4320                self.rollback_observation("syncular_outbox_drop");
4321            }
4322            self.restore(rollback);
4323            return Err(error);
4324        }
4325        Ok(())
4326    }
4327
4328    /// §7.4.3: (re)create the base + visible table pair for every synced
4329    /// table in the CURRENT schema (idempotent — `IF NOT EXISTS`).
4330    fn create_synced_tables(&self) -> Result<(), String> {
4331        for table in &self.schema.tables {
4332            // The base half + the visible half form the synced-table pair. An
4333            // index name is global in SQLite, so the base half's indexes are
4334            // name-prefixed (`_syncular_base_<index>`) to stay distinct.
4335            for (full, index_prefix) in [
4336                (base_table(&table.name), "_syncular_base_"),
4337                (visible_table(&table.name), ""),
4338            ] {
4339                let mut cols: Vec<String> =
4340                    table.columns.iter().map(|c| quote_ident(&c.name)).collect();
4341                cols.push("\"_syncular_version\" INTEGER NOT NULL".to_owned());
4342                cols.extend(
4343                    table
4344                        .sealed_columns()
4345                        .iter()
4346                        .map(|name| format!("{} TEXT", quote_ident(name))),
4347                );
4348                let sql = format!(
4349                    "CREATE TABLE IF NOT EXISTS {full} ({} , PRIMARY KEY ({}))",
4350                    cols.join(", "),
4351                    quote_ident(&table.primary_key)
4352                );
4353                self.conn
4354                    .execute(&sql, [])
4355                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4356                // §5.11: a replica created before sealed apply gains the
4357                // reserved columns in place, on both halves in the same order,
4358                // so data, indexes and the FTS projection stay.
4359                for name in table.sealed_columns() {
4360                    let present: bool = self
4361                        .conn
4362                        .query_row(
4363                            "SELECT EXISTS(SELECT 1 FROM pragma_table_info(?1) WHERE name = ?2)",
4364                            rusqlite::params![format!("{index_prefix}{}", table.name), name],
4365                            |row| row.get(0),
4366                        )
4367                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4368                    if !present {
4369                        self.conn
4370                            .execute(
4371                                &format!(
4372                                    "ALTER TABLE {full} ADD COLUMN {} TEXT",
4373                                    quote_ident(name)
4374                                ),
4375                                [],
4376                            )
4377                            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4378                    }
4379                }
4380                // The reveal at every round start reads the sealed rows of the
4381                // base half through this partial index.
4382                if table.has_encrypted_columns() && !index_prefix.is_empty() {
4383                    self.conn
4384                        .execute(
4385                            &format!(
4386                                "CREATE INDEX IF NOT EXISTS {} ON {full} (\"_sync_sealed_key\") WHERE \"_sync_sealed_key\" IS NOT NULL",
4387                                quote_ident(&format!("_syncular_sealed_{}", table.name))
4388                            ),
4389                            [],
4390                        )
4391                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4392                }
4393                // Local secondary indexes (CREATE INDEX subset). Created on
4394                // both halves so mirror reads hit an index on either. Runs on
4395                // both the initial create and the §7.4.3 reset recreate path.
4396                for index in &table.indexes {
4397                    let unique = if index.unique { "UNIQUE " } else { "" };
4398                    let index_name = quote_ident(&format!("{index_prefix}{}", index.name));
4399                    let cols_sql = index
4400                        .columns
4401                        .iter()
4402                        .map(|c| quote_ident(c))
4403                        .collect::<Vec<_>>()
4404                        .join(", ");
4405                    let index_sql = format!(
4406                        "CREATE {unique}INDEX IF NOT EXISTS {index_name} ON {full} ({cols_sql})"
4407                    );
4408                    self.conn
4409                        .execute(&index_sql, [])
4410                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4411                }
4412            }
4413        }
4414        // FTS exists only on the visible half. It is a local search
4415        // projection, never a synced/base table or a wire-schema member.
4416        for table in &self.schema.tables {
4417            for index in &table.fts_indexes {
4418                self.create_fts_projection(table, index)?;
4419            }
4420        }
4421        Ok(())
4422    }
4423
4424    fn fts_projection_exists(&self, index: &FtsIndexSchema) -> Result<bool, String> {
4425        let count: i64 = self
4426            .conn
4427            .query_row(
4428                "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?1",
4429                rusqlite::params![index.name],
4430                |row| row.get(0),
4431            )
4432            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4433        Ok(count > 0)
4434    }
4435
4436    fn drop_fts_triggers(&self, index: &FtsIndexSchema) -> Result<(), String> {
4437        for suffix in ["ai", "ad", "au"] {
4438            self.conn
4439                .execute(
4440                    &format!(
4441                        "DROP TRIGGER IF EXISTS {}",
4442                        quote_ident(&format!("{}_{suffix}", index.name))
4443                    ),
4444                    [],
4445                )
4446                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4447        }
4448        Ok(())
4449    }
4450
4451    fn create_fts_triggers(
4452        &self,
4453        table: &TableSchema,
4454        index: &FtsIndexSchema,
4455    ) -> Result<(), String> {
4456        let fts = quote_ident(&index.name);
4457        let source = visible_table(&table.name);
4458        let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
4459        let pk = quote_ident(&table.primary_key);
4460        let projection_columns = std::iter::once(source_id.clone())
4461            .chain(index.columns.iter().map(|column| quote_ident(column)))
4462            .collect::<Vec<_>>()
4463            .join(", ");
4464        let new_values = std::iter::once(format!("CAST(new.{pk} AS TEXT)"))
4465            .chain(
4466                index
4467                    .columns
4468                    .iter()
4469                    .map(|column| format!("new.{}", quote_ident(column))),
4470            )
4471            .collect::<Vec<_>>()
4472            .join(", ");
4473        let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
4474        let delete_new = format!("DELETE FROM {fts} WHERE rowid = (SELECT id FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)); DELETE FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)");
4475        let delete_old = format!("DELETE FROM {fts} WHERE rowid = (SELECT id FROM {mapping} WHERE source_id = CAST(old.{pk} AS TEXT)); DELETE FROM {mapping} WHERE source_id = CAST(old.{pk} AS TEXT)");
4476        let insert_new = format!("INSERT INTO {mapping}(source_id) VALUES (CAST(new.{pk} AS TEXT)); INSERT INTO {fts} (rowid, {projection_columns}) VALUES ((SELECT id FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)), {new_values})");
4477        let bi = quote_ident(&format!("{}_bi", index.name));
4478        let ai = quote_ident(&format!("{}_ai", index.name));
4479        let ad = quote_ident(&format!("{}_ad", index.name));
4480        let au = quote_ident(&format!("{}_au", index.name));
4481        let replacement_exists = format!("EXISTS (SELECT 1 FROM {source} WHERE {pk} = new.{pk})");
4482        let sql = format!(
4483            "DROP TRIGGER IF EXISTS {bi};
4484             DROP TRIGGER IF EXISTS {ai};
4485             DROP TRIGGER IF EXISTS {ad};
4486             DROP TRIGGER IF EXISTS {au};
4487             CREATE TRIGGER {bi} BEFORE INSERT ON {source} WHEN {replacement_exists} BEGIN {delete_new}; END;
4488             CREATE TRIGGER {ai} AFTER INSERT ON {source} BEGIN {insert_new}; END;
4489             CREATE TRIGGER {ad} AFTER DELETE ON {source} BEGIN {delete_old}; END;
4490             CREATE TRIGGER {au} AFTER UPDATE ON {source} BEGIN {delete_old}; {delete_new}; {insert_new}; END;",
4491        );
4492        self.conn
4493            .execute_batch(&sql)
4494            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
4495    }
4496
4497    fn rebuild_fts_projection(
4498        &self,
4499        table: &TableSchema,
4500        index: &FtsIndexSchema,
4501    ) -> Result<(), String> {
4502        let fts = quote_ident(&index.name);
4503        let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
4504        let pk = quote_ident(&table.primary_key);
4505        let indexed_columns = index
4506            .columns
4507            .iter()
4508            .map(|column| quote_ident(column))
4509            .collect::<Vec<_>>();
4510        let projection_columns = std::iter::once(source_id.clone())
4511            .chain(indexed_columns.iter().cloned())
4512            .collect::<Vec<_>>()
4513            .join(", ");
4514        let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
4515        let sql = format!(
4516            "DELETE FROM {fts}; DELETE FROM {mapping}; INSERT INTO {fts} ({projection_columns}) SELECT CAST({pk} AS TEXT), {columns} FROM {source}; INSERT INTO {mapping}(id, source_id) SELECT rowid, {source_id} FROM {fts};",
4517            columns = indexed_columns.join(", "),
4518            source = visible_table(&table.name),
4519        );
4520        self.conn
4521            .execute_batch(&sql)
4522            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
4523    }
4524
4525    fn create_fts_projection(
4526        &self,
4527        table: &TableSchema,
4528        index: &FtsIndexSchema,
4529    ) -> Result<(), String> {
4530        self.begin_observation("syncular_fts_schema")?;
4531        let result = (|| {
4532            let existed = self.fts_projection_exists(index)?;
4533            let tokenizer = index.tokenize.replace('\'', "''");
4534            let columns = index
4535                .columns
4536                .iter()
4537                .map(|column| quote_ident(column))
4538                .collect::<Vec<_>>()
4539                .join(", ");
4540            let sql = format!(
4541            "CREATE VIRTUAL TABLE IF NOT EXISTS {fts} USING fts5({source_id} UNINDEXED, {columns}, tokenize='{tokenizer}')",
4542            fts = quote_ident(&index.name),
4543            source_id = quote_ident(FTS_SOURCE_ID_COLUMN),
4544        );
4545            self.conn.execute(&sql, []).map_err(|error| {
4546                format!(
4547                    "cannot create local FTS5 projection {:?}: {error}",
4548                    index.name
4549                )
4550            })?;
4551            let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
4552            let mapped: bool = self
4553                .conn
4554                .query_row(
4555                    "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?1)",
4556                    [format!("_syncular_fts_{}", index.name)],
4557                    |row| row.get(0),
4558                )
4559                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4560            self.conn.execute_batch(&format!("CREATE TABLE IF NOT EXISTS {mapping}(id INTEGER PRIMARY KEY, source_id TEXT NOT NULL UNIQUE)")).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4561            if !mapped && existed {
4562                self.conn
4563                    .execute_batch(&format!(
4564                        "INSERT INTO {mapping}(id, source_id) SELECT rowid, {} FROM {}",
4565                        quote_ident(FTS_SOURCE_ID_COLUMN),
4566                        quote_ident(&index.name)
4567                    ))
4568                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4569            }
4570            self.create_fts_triggers(table, index)?;
4571            // Every visible row has exactly one mapping row, so a count
4572            // mismatch marks a projection an earlier release damaged; the
4573            // open rebuilds it.
4574            let stale: bool = !existed
4575                || self
4576                    .conn
4577                    .query_row(
4578                        &format!(
4579                            "SELECT (SELECT count(*) FROM {mapping}) != (SELECT count(*) FROM {})",
4580                            visible_table(&table.name)
4581                        ),
4582                        [],
4583                        |row| row.get(0),
4584                    )
4585                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4586            if stale {
4587                self.rebuild_fts_projection(table, index)?;
4588            }
4589            Ok(())
4590        })();
4591        if let Err(error) = result {
4592            self.rollback_observation("syncular_fts_schema");
4593            return Err(error);
4594        }
4595        if let Err(error) = self.conn.execute_batch("RELEASE syncular_fts_schema") {
4596            self.rollback_observation("syncular_fts_schema");
4597            return Err(error.to_string());
4598        }
4599        Ok(())
4600    }
4601
4602    // -- persistence write-through --------------------------------------------
4603
4604    fn persist_outbox_insert(&self, commit: &OutboxCommit) -> Result<(), String> {
4605        let ops: Vec<Value> = commit
4606            .ops
4607            .iter()
4608            .map(|op| {
4609                let mut entry = serde_json::json!({
4610                    "op": if op.upsert { "upsert" } else { "delete" },
4611                    "table": op.table,
4612                    "rowId": op.row_id,
4613                    "baseVersion": op.base_version,
4614                    "values": op.values.clone().map(Value::Object),
4615                });
4616                if op.insert {
4617                    entry["insert"] = Value::Bool(true);
4618                }
4619                entry
4620            })
4621            .collect();
4622        self.conn
4623            .execute(
4624                "INSERT OR REPLACE INTO _syncular_outbox (commit_id, ops_json) VALUES (?1, ?2)",
4625                rusqlite::params![commit.client_commit_id, Value::Array(ops).to_string()],
4626            )
4627            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4628        for operation in &commit.ops {
4629            let Some(values) = &operation.values else {
4630                continue;
4631            };
4632            let Some(table) = self.schema.table(&operation.table) else {
4633                continue;
4634            };
4635            for column in table
4636                .columns
4637                .iter()
4638                .filter(|column| column.ty == ColumnType::BlobRef)
4639            {
4640                let Some(Value::String(raw)) = values.get(&column.name) else {
4641                    continue;
4642                };
4643                let Ok(blob_ref) = serde_json::from_str::<Value>(raw) else {
4644                    continue;
4645                };
4646                let Some(blob_id) = blob_ref.get("blobId").and_then(Value::as_str) else {
4647                    continue;
4648                };
4649                self.conn
4650                    .execute(
4651                        "INSERT OR IGNORE INTO _syncular_blob_commit_refs(commit_id, blob_id)
4652                         SELECT ?1, ?2 WHERE EXISTS (SELECT 1 FROM _syncular_blobs WHERE blob_id = ?2)",
4653                        rusqlite::params![commit.client_commit_id, blob_id],
4654                    )
4655                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4656            }
4657        }
4658        Ok(())
4659    }
4660
4661    fn delete_outbox_persisted(&self, client_commit_id: &str) -> Result<(), String> {
4662        self.conn
4663            .execute(
4664                "DELETE FROM _syncular_blob_commit_refs WHERE commit_id = ?1",
4665                rusqlite::params![client_commit_id],
4666            )
4667            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4668        self.conn
4669            .execute(
4670                "DELETE FROM _syncular_outbox WHERE commit_id = ?1",
4671                rusqlite::params![client_commit_id],
4672            )
4673            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4674        self.conn.execute("DELETE FROM _syncular_row_deliveries WHERE NOT EXISTS(SELECT 1 FROM _syncular_outbox,json_each(ops_json) WHERE json_extract(json_each.value,'$.table')=tbl AND json_extract(json_each.value,'$.rowId')=id)", [])
4675            .map(|_| ())
4676            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
4677    }
4678
4679    fn outcome_status_name(status: CommitOutcomeStatus) -> &'static str {
4680        match status {
4681            CommitOutcomeStatus::Applied => "applied",
4682            CommitOutcomeStatus::Cached => "cached",
4683            CommitOutcomeStatus::Conflict => "conflict",
4684            CommitOutcomeStatus::Rejected => "rejected",
4685        }
4686    }
4687
4688    fn outcome_resolution_name(resolution: CommitOutcomeResolution) -> &'static str {
4689        match resolution {
4690            CommitOutcomeResolution::Active => "active",
4691            CommitOutcomeResolution::ResolvedKeepServer => "resolved_keep_server",
4692            CommitOutcomeResolution::Superseded => "superseded",
4693            CommitOutcomeResolution::Dismissed => "dismissed",
4694        }
4695    }
4696
4697    fn parse_outcome_status(value: &str) -> Result<CommitOutcomeStatus, String> {
4698        match value {
4699            "applied" => Ok(CommitOutcomeStatus::Applied),
4700            "cached" => Ok(CommitOutcomeStatus::Cached),
4701            "conflict" => Ok(CommitOutcomeStatus::Conflict),
4702            "rejected" => Ok(CommitOutcomeStatus::Rejected),
4703            _ => Err(format!("invalid persisted commit outcome status {value:?}")),
4704        }
4705    }
4706
4707    fn parse_outcome_resolution(value: &str) -> Result<CommitOutcomeResolution, String> {
4708        match value {
4709            "active" => Ok(CommitOutcomeResolution::Active),
4710            "resolved_keep_server" => Ok(CommitOutcomeResolution::ResolvedKeepServer),
4711            "superseded" => Ok(CommitOutcomeResolution::Superseded),
4712            "dismissed" => Ok(CommitOutcomeResolution::Dismissed),
4713            _ => Err(format!(
4714                "invalid persisted commit outcome resolution {value:?}"
4715            )),
4716        }
4717    }
4718
4719    fn persist_commit_outcome(
4720        &self,
4721        client_commit_id: &str,
4722        status: CommitOutcomeStatus,
4723        results: &[CommitOperationOutcome],
4724        operations: Option<&[OutboxOp]>,
4725    ) -> Result<(), String> {
4726        let results_json = serde_json::to_string(results).map_err(|error| error.to_string())?;
4727        let operations_json = operations
4728            .map(|items| {
4729                serde_json::to_string(&items.iter().map(CommitOperation::from).collect::<Vec<_>>())
4730            })
4731            .transpose()
4732            .map_err(|error| error.to_string())?;
4733        self.conn
4734            .execute(
4735                "INSERT INTO _syncular_commit_outcomes (
4736                   client_commit_id, status, recorded_at_ms, results_json,
4737                   operations_json, resolution
4738                 ) VALUES (?1, ?2, ?3, ?4, ?5, 'active')",
4739                rusqlite::params![
4740                    client_commit_id,
4741                    Self::outcome_status_name(status),
4742                    self.clock_now_ms(),
4743                    results_json,
4744                    operations_json
4745                ],
4746            )
4747            .map(|_| ())
4748            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
4749    }
4750
4751    /// §7.2: the journal holds a stored cell only in its canonical shape. A cell
4752    /// the byte-envelope rule refuses cannot be journaled verbatim.
4753    fn stored_cell_is_representable(cell: &Value) -> bool {
4754        cell.is_null()
4755            || cell.is_string()
4756            || cell.is_boolean()
4757            || cell.is_number()
4758            || is_canonical_bytes_envelope(cell)
4759    }
4760
4761    fn stored_row_is_representable(value: &Value) -> bool {
4762        value
4763            .as_object()
4764            .is_some_and(|row| row.values().all(Self::stored_cell_is_representable))
4765    }
4766
4767    fn stored_operation_is_representable(value: &Value) -> bool {
4768        value.get("table").is_some_and(Value::is_string)
4769            && value.get("rowId").is_some_and(Value::is_string)
4770            && value.get("baseVersion").is_none_or(|v| {
4771                v.as_i64()
4772                    .is_some_and(|n| n.unsigned_abs() <= MAX_JS_SAFE_INTEGER)
4773            })
4774            && match value.get("op").and_then(Value::as_str) {
4775                Some("upsert") => {
4776                    value
4777                        .get("values")
4778                        .is_some_and(Self::stored_row_is_representable)
4779                        && value
4780                            .get("insert")
4781                            .is_none_or(|insert| *insert == Value::Bool(true))
4782                }
4783                Some("delete") => value.get("values").is_none() && value.get("insert").is_none(),
4784                _ => false,
4785            }
4786    }
4787
4788    /// §7.2/§7.4.4: a dropped commit keeps its operation envelope in the journal
4789    /// only when the journal reader accepts the serialized operation, which is the
4790    /// rule `outcome_from_row` applies to this same row later.
4791    fn outbox_op_is_journalable(operation: &OutboxOp) -> bool {
4792        serde_json::to_value(CommitOperation::from(operation))
4793            .is_ok_and(|value| Self::stored_operation_is_representable(&value))
4794    }
4795
4796    fn outcome_from_row(row: StoredCommitOutcomeRow) -> Result<CommitOutcome, String> {
4797        let StoredCommitOutcomeRow {
4798            sequence,
4799            client_commit_id,
4800            status,
4801            recorded_at_ms,
4802            results_json,
4803            operations_json,
4804            resolution,
4805            resolved_at_ms,
4806            replacement_client_commit_id,
4807        } = row;
4808        if sequence.unsigned_abs() > MAX_JS_SAFE_INTEGER
4809            || recorded_at_ms.unsigned_abs() > MAX_JS_SAFE_INTEGER
4810            || resolved_at_ms.is_some_and(|value| value.unsigned_abs() > MAX_JS_SAFE_INTEGER)
4811        {
4812            return Err("sync.local_corrupt: persisted commit outcome is invalid".to_owned());
4813        }
4814        let invalid = || "sync.local_corrupt: persisted commit outcome is invalid".to_owned();
4815        let stored: Vec<Value> = serde_json::from_str(&results_json).map_err(|_| invalid())?;
4816        for result in &stored {
4817            let nested = match result.get("status").and_then(Value::as_str) {
4818                Some("conflict") => result.get("conflict"),
4819                Some("error") => result.get("rejection"),
4820                _ => None,
4821            };
4822            if nested
4823                .and_then(|record| record.get("operation"))
4824                .is_some_and(|op| !Self::stored_operation_is_representable(op))
4825                || (result.get("status").and_then(Value::as_str) == Some("conflict")
4826                    && nested
4827                        .and_then(|record| record.get("serverRow"))
4828                        .is_some_and(|row| !Self::stored_row_is_representable(row)))
4829            {
4830                return Err(invalid());
4831            }
4832        }
4833        let results: Vec<CommitOperationOutcome> =
4834            serde_json::from_value(Value::Array(stored)).map_err(|_| invalid())?;
4835        let has_conflict = results
4836            .iter()
4837            .any(|result| matches!(result, CommitOperationOutcome::Conflict { .. }));
4838        let has_error = results
4839            .iter()
4840            .any(|result| matches!(result, CommitOperationOutcome::Error { .. }));
4841        let status = Self::parse_outcome_status(&status).map_err(|_| invalid())?;
4842        let redacted = results.is_empty() && operations_json.is_none();
4843        if match status {
4844            CommitOutcomeStatus::Applied | CommitOutcomeStatus::Cached => has_conflict || has_error,
4845            CommitOutcomeStatus::Conflict => !(has_conflict || redacted),
4846            CommitOutcomeStatus::Rejected => has_conflict || !(has_error || redacted),
4847        } {
4848            return Err(invalid());
4849        }
4850        let operations: Option<Vec<CommitOperation>> = operations_json
4851            .map(|raw| {
4852                let stored: Vec<Value> = serde_json::from_str(&raw).map_err(|_| invalid())?;
4853                if !stored.iter().all(Self::stored_operation_is_representable) {
4854                    return Err(invalid());
4855                }
4856                serde_json::from_value(Value::Array(stored)).map_err(|_| invalid())
4857            })
4858            .transpose()?;
4859        for result in &results {
4860            match result {
4861                CommitOperationOutcome::Conflict { conflict }
4862                    if conflict.server_version.unsigned_abs() > MAX_JS_SAFE_INTEGER =>
4863                {
4864                    return Err("sync.local_corrupt: persisted commit outcome is invalid".to_owned())
4865                }
4866                CommitOperationOutcome::Error { rejection } => {
4867                    if let Some(details) = &rejection.details {
4868                        let encoded = serde_json::to_string(details).map_err(|_| {
4869                            "sync.local_corrupt: persisted commit outcome is invalid".to_owned()
4870                        })?;
4871                        RejectionDetails::parse(&encoded).map_err(|_| {
4872                            "sync.local_corrupt: persisted commit outcome is invalid".to_owned()
4873                        })?;
4874                    }
4875                }
4876                _ => {}
4877            }
4878        }
4879        Ok(CommitOutcome {
4880            sequence,
4881            client_commit_id,
4882            status,
4883            recorded_at_ms,
4884            results,
4885            operations,
4886            retained_rows: None,
4887            resolution: Self::parse_outcome_resolution(&resolution)?,
4888            resolved_at_ms,
4889            replacement_client_commit_id,
4890        })
4891    }
4892
4893    /// Resolve declared unique keys through SQLite, including affinity and NULL semantics.
4894    fn unique_conflicts(
4895        &self,
4896        table: &TableSchema,
4897        values: &Map<String, Value>,
4898        base: bool,
4899    ) -> Result<Vec<RetainedUniqueConflict>, String> {
4900        let source = if base {
4901            base_table(&table.name)
4902        } else {
4903            visible_table(&table.name)
4904        };
4905        let mut conflicts = Vec::new();
4906        for index in table.indexes.iter().filter(|index| index.unique) {
4907            let mut params = vec![json_param_to_sql(
4908                values.get(&table.primary_key).ok_or("sync.local_corrupt")?,
4909            )?];
4910            for name in &index.columns {
4911                let column = table
4912                    .columns
4913                    .iter()
4914                    .find(|column| column.name == *name)
4915                    .ok_or("sync.local_corrupt")?;
4916                let cell = json_to_column_value(column, values.get(name))?;
4917                params.push(owned_sql_value(RowParam::Cell(&cell))?);
4918            }
4919            let sql = format!(
4920                "SELECT * FROM {source} WHERE {} != ? AND {}",
4921                quote_ident(&table.primary_key),
4922                index
4923                    .columns
4924                    .iter()
4925                    .map(|column| format!("{} = ?", quote_ident(column)))
4926                    .collect::<Vec<_>>()
4927                    .join(" AND ")
4928            );
4929            let mut statement = self
4930                .conn
4931                .prepare_cached(&sql)
4932                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4933            let records = statement
4934                .query_map(rusqlite::params_from_iter(params), |record| {
4935                    let values = table
4936                        .columns
4937                        .iter()
4938                        .enumerate()
4939                        .map(|(index, column)| {
4940                            let value = sql_ref_to_json(column, record.get_ref(index)?);
4941                            json_to_column_value(column, Some(&value)).map_err(|message| {
4942                                rusqlite::Error::ToSqlConversionFailure(message.into())
4943                            })
4944                        })
4945                        .collect::<Result<Row, _>>()?;
4946                    Ok((values, record.get::<_, i64>(table.columns.len())?))
4947                })
4948                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4949            for record in records {
4950                let (row, version) =
4951                    record.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4952                let row_id = render_row_id_json(Some(&column_value_to_json(&row[table.pk_index])))?;
4953                conflicts.push(RetainedUniqueConflict {
4954                    index: index.name.clone(),
4955                    columns: index.columns.clone(),
4956                    row_id,
4957                    server_row: table
4958                        .columns
4959                        .iter()
4960                        .zip(&row)
4961                        .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
4962                        .collect(),
4963                    server_version: version,
4964                });
4965            }
4966        }
4967        Ok(conflicts)
4968    }
4969
4970    fn attach_retained_rows(&self, mut outcome: CommitOutcome) -> Result<CommitOutcome, String> {
4971        if let Some((commit, initial)) = self
4972            .failed_commits
4973            .iter()
4974            .find(|(commit, _)| commit.client_commit_id == outcome.client_commit_id)
4975        {
4976            let mut rows = Vec::new();
4977            for operation in &commit.ops {
4978                // A table the current schema removed has no rows to retain.
4979                let Some(table) = self.schema.table(&operation.table) else {
4980                    continue;
4981                };
4982                let key =
4983                    serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
4984                        .map_err(|_| "sync.local_corrupt".to_owned())?;
4985                let base = self.stored_row(table, &operation.row_id, true)?;
4986                rows.push(RetainedCommitRow {
4987                    table: operation.table.clone(),
4988                    row_id: operation.row_id.clone(),
4989                    unique_conflicts: if operation.upsert {
4990                        self.unique_conflicts(
4991                            table,
4992                            initial
4993                                .get(&key)
4994                                .and_then(Value::as_object)
4995                                .or(operation.values.as_ref())
4996                                .ok_or("sync.local_corrupt")?,
4997                            true,
4998                        )?
4999                    } else {
5000                        Vec::new()
5001                    },
5002                    local_row: if operation.upsert {
5003                        initial
5004                            .get(&key)
5005                            .and_then(Value::as_object)
5006                            .cloned()
5007                            .or_else(|| operation.values.clone())
5008                    } else {
5009                        None
5010                    },
5011                    server_row: base.as_ref().map(|(values, _)| {
5012                        table
5013                            .columns
5014                            .iter()
5015                            .zip(values)
5016                            .map(|(column, value)| {
5017                                (column.name.clone(), column_value_to_json(value))
5018                            })
5019                            .collect()
5020                    }),
5021                    server_version: base.map(|(_, version)| version),
5022                });
5023            }
5024            outcome.retained_rows = Some(rows);
5025        }
5026        Ok(outcome)
5027    }
5028
5029    pub fn commit_outcome(&self, client_commit_id: &str) -> Result<Option<CommitOutcome>, String> {
5030        let row = self
5031            .conn
5032            .query_row(
5033                "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
5034                        resolution, resolved_at_ms, replacement_client_commit_id
5035                   FROM _syncular_commit_outcomes WHERE client_commit_id = ?1",
5036                rusqlite::params![client_commit_id],
5037                |row| {
5038                    Ok(StoredCommitOutcomeRow {
5039                        sequence: row.get(0)?,
5040                        client_commit_id: row.get(1)?,
5041                        status: row.get(2)?,
5042                        recorded_at_ms: row.get(3)?,
5043                        results_json: row.get(4)?,
5044                        operations_json: row.get(5)?,
5045                        resolution: row.get(6)?,
5046                        resolved_at_ms: row.get(7)?,
5047                        replacement_client_commit_id: row.get(8)?,
5048                    })
5049                },
5050            )
5051            .optional()
5052            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5053        row.map(|row| self.attach_retained_rows(Self::outcome_from_row(row)?))
5054            .transpose()
5055    }
5056
5057    /// Internal host step. The host must run its idle drive before finishing.
5058    #[doc(hidden)]
5059    pub fn prepare_current_base(
5060        &self,
5061        client_commit_id: &str,
5062        sequence: i64,
5063    ) -> Result<CurrentBasePreparation, CurrentBaseProof> {
5064        let outcome = self
5065            .commit_outcome(client_commit_id)
5066            .map_err(|_| CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed))?;
5067        let valid = outcome.as_ref().is_some_and(|outcome| {
5068            outcome.sequence == sequence
5069                && outcome.resolution == CommitOutcomeResolution::Active
5070                && matches!(
5071                    outcome.status,
5072                    CommitOutcomeStatus::Conflict | CommitOutcomeStatus::Rejected
5073                )
5074                && outcome
5075                    .operations
5076                    .as_ref()
5077                    .zip(outcome.retained_rows.as_ref())
5078                    .is_some_and(|(operations, rows)| {
5079                        !rows.is_empty() && operations.len() == rows.len()
5080                    })
5081        });
5082        if !valid {
5083            return Err(CurrentBaseProof::failure(
5084                CurrentBaseProofReason::OutcomeUnavailable,
5085            ));
5086        }
5087        if !self.transport_enabled {
5088            return Err(CurrentBaseProof::failure(CurrentBaseProofReason::Offline));
5089        }
5090        Ok(CurrentBasePreparation {
5091            client_commit_id: client_commit_id.to_owned(),
5092            sequence,
5093            units: load_units(&self.conn)
5094                .map_err(|_| CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed))?,
5095            transport_epoch: self.transport_epoch,
5096            registration_epoch: self.registration_epoch,
5097            security_epoch: self.security_epoch,
5098        })
5099    }
5100
5101    /// Internal host step after the completed post-rejection idle drive.
5102    #[doc(hidden)]
5103    pub fn finish_current_base(
5104        &mut self,
5105        prepared: CurrentBasePreparation,
5106        drive: SyncOutcome,
5107    ) -> CurrentBaseProof {
5108        if !self.transport_enabled || self.transport_epoch != prepared.transport_epoch {
5109            return CurrentBaseProof::failure(CurrentBaseProofReason::Offline);
5110        }
5111        if self.registration_epoch != prepared.registration_epoch {
5112            return CurrentBaseProof::failure(CurrentBaseProofReason::MissingCoverage);
5113        }
5114        if self.security_preflight || self.security_epoch != prepared.security_epoch {
5115            return CurrentBaseProof::failure(CurrentBaseProofReason::SyncFailed);
5116        }
5117        let report = match drive {
5118            SyncOutcome::Ok(report) => report,
5119            SyncOutcome::BudgetExhausted(_) => {
5120                return CurrentBaseProof::failure(CurrentBaseProofReason::BudgetExhausted)
5121            }
5122            _ => return CurrentBaseProof::failure(CurrentBaseProofReason::SyncFailed),
5123        };
5124        if report.schema_floor.is_some() {
5125            return CurrentBaseProof::failure(CurrentBaseProofReason::SchemaFloor);
5126        }
5127        if self
5128            .conn
5129            .execute_batch("SAVEPOINT syncular_base_proof")
5130            .is_err()
5131        {
5132            return CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed);
5133        }
5134        let proof = (|| {
5135            let snapshot = snapshot_read_connection(
5136                &self.conn,
5137                &SnapshotReadRequest {
5138                    units: prepared.units.iter().map(|unit| unit.id.clone()).collect(),
5139                    commit_ids: vec![prepared.client_commit_id.clone()],
5140                    ..Default::default()
5141                },
5142            )
5143            .map_err(|_| CurrentBaseProofReason::SnapshotFailed)?;
5144            let current = self
5145                .commit_outcome(&prepared.client_commit_id)
5146                .map_err(|_| CurrentBaseProofReason::SnapshotFailed)?;
5147            let unavailable = if report.revoked.is_empty() {
5148                CurrentBaseProofReason::OutcomeUnavailable
5149            } else {
5150                CurrentBaseProofReason::Revoked
5151            };
5152            let Some(outcome) = current else {
5153                return Err(unavailable);
5154            };
5155            if !matches!(
5156                snapshot.deliveries.first(),
5157                Some(CommitDelivery::Known { .. })
5158            ) || outcome.sequence != prepared.sequence
5159                || outcome.resolution != CommitOutcomeResolution::Active
5160                || !matches!(
5161                    outcome.status,
5162                    CommitOutcomeStatus::Conflict | CommitOutcomeStatus::Rejected
5163                )
5164            {
5165                return Err(unavailable);
5166            }
5167            let (Some(operations), Some(rows)) = (outcome.operations, outcome.retained_rows) else {
5168                return Err(unavailable);
5169            };
5170            if rows.is_empty() || operations.len() != rows.len() {
5171                return Err(unavailable);
5172            }
5173            let after =
5174                load_units(&self.conn).map_err(|_| CurrentBaseProofReason::SnapshotFailed)?;
5175            let complete: HashSet<&str> = snapshot
5176                .units
5177                .iter()
5178                .filter_map(|unit| match unit {
5179                    UnitCatchup::Known {
5180                        id,
5181                        bootstrap_complete: true,
5182                        ..
5183                    } => Some(id.as_str()),
5184                    _ => None,
5185                })
5186                .collect();
5187            // A unit id names one scope map (§4.1), so the same id before and
5188            // after the drive is the same registration.
5189            let candidates: Vec<&UnitRecord> = after
5190                .iter()
5191                .filter(|unit| {
5192                    complete.contains(unit.id.as_str())
5193                        && prepared.units.iter().any(|before| before.id == unit.id)
5194                })
5195                .collect();
5196            let covered = |table_name: &str, row: &Map<String, Value>| {
5197                let Some(table) = self.schema.table(table_name) else {
5198                    return false;
5199                };
5200                candidates.iter().any(|unit| {
5201                    covers(table, &unit.scopes)
5202                        && pair_scopes(&unit.scopes, table_name, unit.echo.as_deref()).is_some_and(
5203                            |scopes| {
5204                                scopes.iter().all(|(variable, allowed)| {
5205                                    table
5206                                        .scope_column(variable)
5207                                        .and_then(|column| match row.get(column) {
5208                                            Some(Value::String(value)) => Some(value.clone()),
5209                                            Some(Value::Bool(value)) => Some(value.to_string()),
5210                                            Some(Value::Number(value)) => {
5211                                                value.as_f64().map(|number| {
5212                                                    ryu_js::Buffer::new()
5213                                                        .format_finite(number)
5214                                                        .to_owned()
5215                                                })
5216                                            }
5217                                            _ => None,
5218                                        })
5219                                        .is_some_and(|value| {
5220                                            !value.is_empty() && allowed.contains(&value)
5221                                        })
5222                                })
5223                            },
5224                        )
5225                })
5226            };
5227            for row in &rows {
5228                let Some(base) = row.server_row.as_ref().or(row.local_row.as_ref()) else {
5229                    return Err(if report.revoked.is_empty() {
5230                        CurrentBaseProofReason::MissingCoverage
5231                    } else {
5232                        CurrentBaseProofReason::Revoked
5233                    });
5234                };
5235                if !covered(&row.table, base)
5236                    || row
5237                        .unique_conflicts
5238                        .iter()
5239                        .any(|conflict| !covered(&row.table, &conflict.server_row))
5240                {
5241                    return Err(if report.revoked.is_empty() {
5242                        CurrentBaseProofReason::MissingCoverage
5243                    } else {
5244                        CurrentBaseProofReason::Revoked
5245                    });
5246                }
5247            }
5248            Ok(rows)
5249        })();
5250        if self
5251            .conn
5252            .execute_batch("RELEASE syncular_base_proof")
5253            .is_err()
5254        {
5255            return CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed);
5256        }
5257        match proof {
5258            Ok(rows) => CurrentBaseProof {
5259                proven: true,
5260                rows: Some(rows),
5261                reason: None,
5262            },
5263            Err(reason) => CurrentBaseProof::failure(reason),
5264        }
5265    }
5266
5267    pub fn prove_current_base(
5268        &mut self,
5269        transport: &mut dyn Transport,
5270        client_commit_id: &str,
5271        sequence: i64,
5272        max_rounds: Option<u32>,
5273    ) -> CurrentBaseProof {
5274        let prepared = match self.prepare_current_base(client_commit_id, sequence) {
5275            Ok(prepared) => prepared,
5276            Err(proof) => return proof,
5277        };
5278        let drive = self.sync_until_idle(transport, max_rounds);
5279        self.finish_current_base(prepared, drive)
5280    }
5281
5282    pub fn commit_outcomes(&self, query: CommitOutcomeQuery) -> Result<Vec<CommitOutcome>, String> {
5283        if query.limit == Some(0) {
5284            return Err("sync.invalid_request: commit outcome limit must be positive".to_owned());
5285        }
5286        let mut sql = String::from(
5287            "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
5288                    resolution, resolved_at_ms, replacement_client_commit_id
5289               FROM _syncular_commit_outcomes",
5290        );
5291        if query.active_only {
5292            sql.push_str(" WHERE resolution = 'active' AND status IN ('conflict', 'rejected')");
5293        }
5294        sql.push_str(" ORDER BY seq DESC");
5295        if let Some(limit) = query.limit {
5296            sql.push_str(&format!(" LIMIT {limit}"));
5297        }
5298        let mut stmt = self
5299            .conn
5300            .prepare(&sql)
5301            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5302        let rows = stmt
5303            .query_map([], |row| {
5304                Ok(StoredCommitOutcomeRow {
5305                    sequence: row.get(0)?,
5306                    client_commit_id: row.get(1)?,
5307                    status: row.get(2)?,
5308                    recorded_at_ms: row.get(3)?,
5309                    results_json: row.get(4)?,
5310                    operations_json: row.get(5)?,
5311                    resolution: row.get(6)?,
5312                    resolved_at_ms: row.get(7)?,
5313                    replacement_client_commit_id: row.get(8)?,
5314                })
5315            })
5316            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5317        let mut outcomes = Vec::new();
5318        for row in rows {
5319            outcomes.push(self.attach_retained_rows(Self::outcome_from_row(
5320                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
5321            )?)?);
5322        }
5323        Ok(outcomes)
5324    }
5325
5326    fn prune_commit_outcomes(&self) -> Result<(), String> {
5327        #[cfg(test)]
5328        self.outcome_prune_count
5329            .set(self.outcome_prune_count.get() + 1);
5330        let max_entries = self.limits.outcome_retention_max_entries.unwrap_or(1_000);
5331        let count = self
5332            .conn
5333            .query_row(
5334                "SELECT COUNT(*) FROM _syncular_commit_outcomes",
5335                [],
5336                |row| row.get::<_, i64>(0),
5337            )
5338            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
5339            as usize;
5340        let excess = count.saturating_sub(max_entries);
5341        if excess == 0 {
5342            return Ok(());
5343        }
5344        let mut stmt = self
5345            .conn
5346            .prepare(
5347                "SELECT seq FROM _syncular_commit_outcomes
5348                  WHERE status IN ('applied', 'cached') OR resolution != 'active'
5349                  ORDER BY seq ASC LIMIT ?1",
5350            )
5351            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5352        let rows = stmt
5353            .query_map(rusqlite::params![excess as i64], |row| row.get::<_, i64>(0))
5354            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5355        let sequences = rows
5356            .collect::<Result<Vec<_>, _>>()
5357            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5358        drop(stmt);
5359        for sequence in sequences {
5360            self.conn
5361                .execute(
5362                    "DELETE FROM _syncular_commit_outcomes WHERE seq = ?1",
5363                    rusqlite::params![sequence],
5364                )
5365                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5366        }
5367        Ok(())
5368    }
5369
5370    // -- units (§4.1) ------------------------------------------------------------
5371
5372    /// Every registered unit in registration order (§4.1).
5373    pub fn units(&self) -> Result<Vec<UnitRecord>, String> {
5374        self.stored(load_units(&self.conn))
5375    }
5376
5377    pub fn unit(&self, id: &str) -> Result<Option<UnitRecord>, String> {
5378        self.stored(get_unit(&self.conn, id))
5379    }
5380
5381    /// Register one unit (§4.1) and return its id.
5382    pub fn subscribe(&mut self, scopes: UnitScopes) -> Result<String, String> {
5383        Ok(self.subscribe_many(vec![scopes])?.remove(0))
5384    }
5385
5386    /// Register several units in one local transaction and return their ids:
5387    /// one commit for all of them (on OPFS a commit costs milliseconds, so a
5388    /// cold start with hundreds of units pays it once). An error rolls back
5389    /// every registration of the call. Registering an active unit again
5390    /// changes nothing; a revoked unit re-arms (§3.3). A call that registers or
5391    /// re-arms a unit raises sync-needed with an interactive intent.
5392    pub fn subscribe_many(&mut self, units: Vec<UnitScopes>) -> Result<Vec<String>, String> {
5393        self.begin_observation("syncular_subscribe")?;
5394        let mut changed = false;
5395        let registered = (|| {
5396            let mut ids = Vec::with_capacity(units.len());
5397            for scopes in units {
5398                let (id, held) = self.hold_unit(scopes, true)?;
5399                changed |= held;
5400                ids.push(id);
5401            }
5402            self.conn
5403                .execute_batch("RELEASE syncular_subscribe")
5404                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5405            Ok(ids)
5406        })();
5407        let ids = match registered {
5408            Ok(ids) => ids,
5409            Err(error) => {
5410                self.rollback_observation("syncular_subscribe");
5411                return Err(error);
5412            }
5413        };
5414        if changed {
5415            self.registration_epoch = self.registration_epoch.wrapping_add(1);
5416            self.set_sync_needed(true, true);
5417        }
5418        Ok(ids)
5419    }
5420
5421    /// Release the explicit reference to a unit. The unit leaves the
5422    /// registration list when no window base holds it (§4.1); its rows stay.
5423    pub fn unsubscribe(&mut self, id: &str) -> Result<(), String> {
5424        let Some(unit) = self.stored(get_unit(&self.conn, id))? else {
5425            return Ok(());
5426        };
5427        if !unit.explicit {
5428            return Ok(());
5429        }
5430        if !self.stored(window_refs(&self.conn, id))?.is_empty() {
5431            return self.stored(save_unit(
5432                &self.conn,
5433                &UnitRecord {
5434                    explicit: false,
5435                    ..unit
5436                },
5437            ));
5438        }
5439        self.conn
5440            .execute("DELETE FROM _syncular_units WHERE id = ?1", [id])
5441            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5442        self.registration_epoch = self.registration_epoch.wrapping_add(1);
5443        if unit.status == UnitStatus::Active {
5444            self.set_sync_needed(true, true);
5445        }
5446        Ok(())
5447    }
5448
5449    /// Take one reference on a unit (§4.1): validate the scope map, register a
5450    /// new unit catching up with cursor `-1`, re-arm a revoked one (§3.3).
5451    /// Returns the id and whether the registration list changed.
5452    fn hold_unit(&mut self, scopes: UnitScopes, explicit: bool) -> Result<(String, bool), String> {
5453        if scopes.is_empty()
5454            || scopes.values().any(|value| value == "*")
5455            || !self
5456                .schema
5457                .tables
5458                .iter()
5459                .any(|table| covers(table, &scopes))
5460        {
5461            return Err("sync.invalid_subscription: a unit names one non-wildcard value per variable and covers a table".to_owned());
5462        }
5463        let id = ssp2::registration::unit_id(&scope_pairs(&scopes));
5464        let existing = self.stored(get_unit(&self.conn, &id))?;
5465        if let Some(existing) = existing
5466            .as_ref()
5467            .filter(|unit| unit.status != UnitStatus::Revoked)
5468        {
5469            if explicit && !existing.explicit {
5470                self.stored(save_unit(
5471                    &self.conn,
5472                    &UnitRecord {
5473                        explicit,
5474                        ..existing.clone()
5475                    },
5476                ))?;
5477            }
5478            return Ok((id, false));
5479        }
5480        self.delete_pending_evict(&id)?;
5481        self.stored(save_unit(
5482            &self.conn,
5483            &UnitRecord {
5484                id: id.clone(),
5485                scopes,
5486                explicit: explicit || existing.is_some_and(|unit| unit.explicit),
5487                status: UnitStatus::Active,
5488                reason_code: None,
5489                cursor: Some(-1),
5490                bootstrap_state: None,
5491                seeded: Vec::new(),
5492                echo: None,
5493            },
5494        ))?;
5495        Ok((id, true))
5496    }
5497
5498    // -- windows (§4.8) -----------------------------------------------------------
5499
5500    /// §4.8: set the windowed-in values of a base, one unit per value. Added
5501    /// values take a reference on their unit (a new unit bootstraps on the
5502    /// next sync); removed values release theirs, and a unit no reference
5503    /// holds leaves the registration list fused with its first eviction chunk
5504    /// (E1–E4). Idempotent; re-entry cancels a deferred eviction.
5505    pub fn set_window(
5506        &mut self,
5507        base: &WindowBase,
5508        units: &[String],
5509    ) -> Result<CommandEffects, String> {
5510        let base_key = window_base_key(base);
5511        let live = self.load_window_units(&base_key)?;
5512        let mut changed = false;
5513        for value in units {
5514            if live.iter().any(|(held, _)| held == value) {
5515                continue;
5516            }
5517            let rollback = self.capture();
5518            self.begin_observation("syncular_window")?;
5519            let widened = (|| {
5520                let (id, registered) = self.hold_unit(window_unit_scopes(base, value), false)?;
5521                self.sync_needed |= registered;
5522                self.conn
5523                    .execute(
5524                        "INSERT OR REPLACE INTO _syncular_windows(base, unit, sub_id) VALUES (?1, ?2, ?3)",
5525                        rusqlite::params![base_key, value, id],
5526                    )
5527                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5528                let mut batch = ChangeAccumulator::default();
5529                batch.window(&base_key, value);
5530                batch.status = true;
5531                self.finish_observation("syncular_window", batch)
5532            })();
5533            if let Err(error) = widened {
5534                self.rollback_observation("syncular_window");
5535                self.restore(rollback);
5536                return Err(error);
5537            }
5538            changed = true;
5539            self.registration_epoch = self.registration_epoch.wrapping_add(1);
5540        }
5541        for (value, id) in live {
5542            if units.contains(&value) {
5543                continue;
5544            }
5545            self.rebuild_overlay_if_dirty()?;
5546            let rollback = self.capture();
5547            self.begin_observation("syncular_window")?;
5548            let remaining = self.release_window_unit(&base_key, &value, &id);
5549            let remaining = match remaining {
5550                Ok(remaining) => remaining,
5551                Err(error) => {
5552                    self.rollback_observation("syncular_window");
5553                    self.restore(rollback);
5554                    return Err(error);
5555                }
5556            };
5557            changed = true;
5558            self.registration_epoch = self.registration_epoch.wrapping_add(1);
5559            if remaining {
5560                self.drain_pending_evictions()?;
5561            }
5562        }
5563        Ok(if changed && self.transport_enabled {
5564            CommandEffects::interactive()
5565        } else {
5566            CommandEffects::none()
5567        })
5568    }
5569
5570    /// §4.8 completeness oracle: the windowed-in values of a base and those
5571    /// whose unit is not complete. A value is complete once its unit is active
5572    /// and no longer bootstrapping; registration alone is not completeness.
5573    pub fn window_state(&self, base: &WindowBase) -> Result<WindowState, String> {
5574        persisted_window_state(&self.conn, base).map_err(|failure| failure.message)
5575    }
5576
5577    /// Registry rows for a base, ordered by value: (value, unit id).
5578    fn load_window_units(&self, base_key: &str) -> Result<Vec<(String, String)>, String> {
5579        let mut statement = self
5580            .conn
5581            .prepare_cached(
5582                "SELECT unit, sub_id FROM _syncular_windows WHERE base = ?1 ORDER BY unit ASC",
5583            )
5584            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5585        let rows = statement
5586            .query_map([base_key], |row| Ok((row.get(0)?, row.get(1)?)))
5587            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5588        rows.collect::<Result<_, _>>()
5589            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5590    }
5591
5592    /// Every window registry row: (base key, value).
5593    fn load_window_registry(&self) -> Result<Vec<(String, String)>, String> {
5594        let mut statement = self
5595            .conn
5596            .prepare_cached("SELECT base, unit FROM _syncular_windows ORDER BY base, unit")
5597            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5598        let rows = statement
5599            .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))
5600            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5601        rows.collect::<Result<_, _>>()
5602            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5603    }
5604
5605    /// §4.8: release a window value's reference inside the caller's
5606    /// observation. When no reference holds the unit, unregister it fused with
5607    /// its first eviction chunk (E3): delete the rows of every covered table
5608    /// matching the persisted echo, except rows pinned by the outbox (E1) or
5609    /// owned by another unit (§3.3), and park the per-table scopes of what
5610    /// remains in the pending-eviction registry. Fails closed without a local
5611    /// scope-column mapping. Returns whether unpinned rows remain.
5612    fn release_window_unit(
5613        &mut self,
5614        base_key: &str,
5615        value: &str,
5616        id: &str,
5617    ) -> Result<bool, String> {
5618        self.conn
5619            .execute(
5620                "DELETE FROM _syncular_windows WHERE base = ?1 AND unit = ?2",
5621                rusqlite::params![base_key, value],
5622            )
5623            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5624        let mut batch = ChangeAccumulator::default();
5625        batch.window(base_key, value);
5626        batch.status = true;
5627        let unit = self.stored(get_unit(&self.conn, id))?;
5628        let mut remaining = false;
5629        if let Some(unit) = unit.filter(|unit| !unit.explicit) {
5630            if self.stored(window_refs(&self.conn, id))?.is_empty() {
5631                let units = self.stored(load_units(&self.conn))?;
5632                self.conn
5633                    .execute("DELETE FROM _syncular_units WHERE id = ?1", [id])
5634                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5635                self.sync_needed = true;
5636                for table in self.schema.tables.clone() {
5637                    let Some(effective) = unit
5638                        .echo
5639                        .as_deref()
5640                        .filter(|_| covers(&table, &unit.scopes))
5641                        .and_then(|echo| pair_scopes(&unit.scopes, &table.name, Some(echo)))
5642                    else {
5643                        continue;
5644                    };
5645                    let pinned = self.pinned_row_ids(&table.name);
5646                    let owners = unit_owners(&units, &table, id);
5647                    let (more, deferred) =
5648                        self.evict_scope_rows(&table.name, &effective, &owners, &pinned)?;
5649                    remaining |= more;
5650                    if deferred {
5651                        self.save_pending_evict(id, &table.name, &effective)?;
5652                    }
5653                    self.record_scope_map(&mut batch, &table.name, &effective);
5654                }
5655            }
5656        }
5657        self.finish_observation("syncular_window", batch)?;
5658        Ok(remaining)
5659    }
5660
5661    /// §4.8 E1: delete base rows owned by the effective scopes
5662    /// ([`local_scope_predicate`]) EXCEPT pinned primary keys. Returns (more
5663    /// unpinned work, any scoped rows remain). A missing scope-column mapping
5664    /// fails without deleting rows.
5665    fn evict_scope_rows(
5666        &self,
5667        table_name: &str,
5668        effective: &[(String, Vec<String>)],
5669        owners: &[Vec<(String, Vec<String>)>],
5670        pinned: &std::collections::HashSet<String>,
5671    ) -> Result<(bool, bool), String> {
5672        let table = self.schema.table(table_name).ok_or("sync.unknown_table")?;
5673        let Some((scope, mut params)) = local_scope_predicate(table, effective, owners)
5674            .map_err(|()| "sync.scope_revoked".to_owned())?
5675        else {
5676            return Ok((false, false));
5677        };
5678        let scope_params = params.len();
5679        let pk = quote_ident(&table.primary_key);
5680        // Pinned ids are rowId text; the text form matches every key type.
5681        let predicate = if pinned.is_empty() {
5682            scope.clone()
5683        } else {
5684            params.extend(pinned.iter().cloned().map(SqlValue::Text));
5685            let holes = vec!["?"; pinned.len()].join(", ");
5686            format!("{scope} AND CAST({pk} AS TEXT) NOT IN ({holes})")
5687        };
5688        let base = base_table(table_name);
5689        let visible = visible_table(table_name);
5690        let mut ids: Vec<SqlValue> = Vec::new();
5691        for source in [&base, &visible] {
5692            let mut stmt = self
5693                .conn
5694                .prepare_cached(&format!(
5695                    "SELECT {pk} FROM {source} WHERE {predicate} LIMIT 1024"
5696                ))
5697                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5698            ids = stmt
5699                .query_map(rusqlite::params_from_iter(&params), |row| row.get(0))
5700                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
5701                .collect::<Result<_, _>>()
5702                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5703            if !ids.is_empty() {
5704                break;
5705            }
5706        }
5707        if !ids.is_empty() {
5708            for id in &ids {
5709                self.conn
5710                    .execute(
5711                        "DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2",
5712                        rusqlite::params![table_name, id],
5713                    )
5714                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5715            }
5716            let holes = vec!["?"; ids.len()].join(", ");
5717            for source in [&base, &visible] {
5718                self.conn
5719                    .execute(
5720                        &format!("DELETE FROM {source} WHERE {pk} IN ({holes})"),
5721                        rusqlite::params_from_iter(&ids),
5722                    )
5723                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5724            }
5725        }
5726        let mut remaining = false;
5727        let mut deferred = false;
5728        for source in [&base, &visible] {
5729            remaining |= self
5730                .conn
5731                .query_row(
5732                    &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {predicate})"),
5733                    rusqlite::params_from_iter(&params),
5734                    |row| row.get::<_, bool>(0),
5735                )
5736                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5737            deferred |= self
5738                .conn
5739                .query_row(
5740                    &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {scope})"),
5741                    rusqlite::params_from_iter(&params[..scope_params]),
5742                    |row| row.get::<_, bool>(0),
5743                )
5744                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5745        }
5746        Ok((remaining, deferred))
5747    }
5748
5749    /// §4.8 E1: continue deferred evictions after the outbox drains or between
5750    /// chunks, retaining outbox pins and their durable marker. A unit that
5751    /// re-entered the window has no pending entry left.
5752    fn drain_pending_evictions(&mut self) -> Result<(), String> {
5753        let pending = self.load_pending_evictions()?;
5754        if pending.is_empty() {
5755            return Ok(());
5756        }
5757        self.rebuild_overlay_if_dirty()?;
5758        for (unit_id, table_name, effective) in pending {
5759            let Some(table) = self.schema.table(&table_name).cloned() else {
5760                self.delete_pending_evict_table(&unit_id, &table_name)?;
5761                continue;
5762            };
5763            loop {
5764                std::thread::yield_now();
5765                self.begin_observation("syncular_evict")?;
5766                let chunk = (|| {
5767                    let pinned = self.pinned_row_ids(&table_name);
5768                    let owners =
5769                        unit_owners(&self.stored(load_units(&self.conn))?, &table, &unit_id);
5770                    let (remaining, deferred) =
5771                        self.evict_scope_rows(&table_name, &effective, &owners, &pinned)?;
5772                    if !deferred {
5773                        self.delete_pending_evict_table(&unit_id, &table_name)?;
5774                    }
5775                    let mut batch = ChangeAccumulator::default();
5776                    self.record_scope_map(&mut batch, &table_name, &effective);
5777                    self.finish_observation("syncular_evict", batch)?;
5778                    Ok(remaining)
5779                })();
5780                match chunk {
5781                    Ok(true) => {}
5782                    Ok(false) => break,
5783                    Err(error) => {
5784                        self.rollback_observation("syncular_evict");
5785                        return Err(error);
5786                    }
5787                }
5788            }
5789        }
5790        Ok(())
5791    }
5792
5793    /// §4.8 E1: primary keys of `table` referenced by a pending outbox
5794    /// commit — rows that MUST NOT be evicted until the commit drains.
5795    fn pinned_row_ids(&self, table: &str) -> std::collections::HashSet<String> {
5796        let mut pinned = std::collections::HashSet::new();
5797        for commit in self
5798            .outbox
5799            .iter()
5800            .map(|commit| &**commit)
5801            .chain(self.failed_commits.iter().map(|(commit, _)| commit))
5802        {
5803            for op in &commit.ops {
5804                if op.table == table {
5805                    pinned.insert(op.row_id.clone());
5806                }
5807            }
5808        }
5809        pinned
5810    }
5811
5812    fn save_pending_evict(
5813        &self,
5814        unit_id: &str,
5815        table: &str,
5816        effective: &[(String, Vec<String>)],
5817    ) -> Result<(), String> {
5818        self.conn.execute(
5819            "INSERT OR REPLACE INTO _syncular_pending_evictions(unit_id, tbl, effective_scopes) VALUES (?1, ?2, ?3)",
5820            rusqlite::params![unit_id, table, scope_map_to_json(effective).to_string()],
5821        ).map(|_| ()).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5822    }
5823
5824    /// Re-entry cancels every pending eviction of the unit.
5825    fn delete_pending_evict(&self, unit_id: &str) -> Result<(), String> {
5826        self.conn
5827            .execute(
5828                "DELETE FROM _syncular_pending_evictions WHERE unit_id = ?1",
5829                [unit_id],
5830            )
5831            .map(|_| ())
5832            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5833    }
5834
5835    fn delete_pending_evict_table(&self, unit_id: &str, table: &str) -> Result<(), String> {
5836        self.conn
5837            .execute(
5838                "DELETE FROM _syncular_pending_evictions WHERE unit_id = ?1 AND tbl = ?2",
5839                [unit_id, table],
5840            )
5841            .map(|_| ())
5842            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5843    }
5844
5845    fn load_pending_evictions(&self) -> Result<Vec<PendingEvict>, String> {
5846        let mut stmt = self
5847            .conn
5848            .prepare("SELECT unit_id, tbl, effective_scopes FROM _syncular_pending_evictions")
5849            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5850        let rows = stmt
5851            .query_map([], |row| {
5852                Ok((
5853                    row.get::<_, String>(0)?,
5854                    row.get::<_, String>(1)?,
5855                    row.get::<_, String>(2)?,
5856                ))
5857            })
5858            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5859        rows.map(|row| {
5860            let (unit_id, table, json) =
5861                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5862            let value: Value = serde_json::from_str(&json)
5863                .map_err(|_| "sync.local_corrupt: invalid pending eviction scopes".to_owned())?;
5864            let effective = json_to_scope_map(&value)
5865                .map_err(|_| "sync.local_corrupt: invalid pending eviction scopes".to_owned())?;
5866            Ok((unit_id, table, effective))
5867        })
5868        .collect()
5869    }
5870
5871    /// Select failed-intent retention; reload durable retained aggregates.
5872    pub fn set_retain_failed_commits(&mut self, enabled: bool) -> Result<(), String> {
5873        self.retain_failed_commits = enabled;
5874        for (commit, _) in self.failed_commits.iter() {
5875            for operation in &commit.ops {
5876                self.overlay_dirty.table(&operation.table);
5877            }
5878        }
5879        self.failed_commits = self.load_failed_commits()?.into();
5880        for (commit, _) in self.failed_commits.iter() {
5881            for operation in &commit.ops {
5882                self.overlay_dirty.table(&operation.table);
5883            }
5884        }
5885        Ok(())
5886    }
5887
5888    fn load_failed_commits(&self) -> Result<Vec<FailedCommit>, String> {
5889        let mut statement = self.conn.prepare("SELECT client_commit_id, operations_json, initial_json FROM _syncular_failed_commits ORDER BY seq")
5890            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5891        let rows = statement
5892            .query_map([], |row| {
5893                Ok((
5894                    row.get::<_, String>(0)?,
5895                    row.get::<_, String>(1)?,
5896                    row.get::<_, String>(2)?,
5897                ))
5898            })
5899            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5900        rows.map(|row| {
5901            let (id, operations, initial) =
5902                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5903            let operations: Vec<CommitOperation> =
5904                serde_json::from_str(&operations).map_err(|_| "sync.local_corrupt".to_owned())?;
5905            let initial =
5906                serde_json::from_str(&initial).map_err(|_| "sync.local_corrupt".to_owned())?;
5907            Ok((
5908                OutboxCommit {
5909                    client_commit_id: id,
5910                    ops: operations
5911                        .into_iter()
5912                        .map(|op| OutboxOp {
5913                            upsert: op.op == "upsert",
5914                            table: op.table,
5915                            row_id: op.row_id,
5916                            base_version: op.base_version,
5917                            values: op.values,
5918                            insert: op.insert,
5919                        })
5920                        .collect(),
5921                },
5922                initial,
5923            ))
5924        })
5925        .collect()
5926    }
5927
5928    fn retain_failed_commit(&mut self, id: &str, operations: &[OutboxOp]) -> Result<(), String> {
5929        let mut initial = Map::new();
5930        for operation in operations {
5931            // A table the current schema removed has no rows to retain.
5932            let Some(table) = self.schema.table(&operation.table) else {
5933                continue;
5934            };
5935            let initial_row = match self.stored_row(table, &operation.row_id, true)? {
5936                Some(row) => Some(row),
5937                None => self.visible_row(table, &operation.row_id)?,
5938            };
5939            if let Some((values, _)) = initial_row {
5940                let mut intended: Map<String, Value> = table
5941                    .columns
5942                    .iter()
5943                    .zip(values.iter())
5944                    .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
5945                    .collect();
5946                if let Some(patch) = &operation.values {
5947                    intended.extend(patch.clone());
5948                }
5949                initial.insert(
5950                    serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
5951                        .map_err(|_| "sync.local_corrupt".to_owned())?,
5952                    Value::Object(intended),
5953                );
5954            }
5955        }
5956        self.conn.execute("INSERT INTO _syncular_failed_commits(client_commit_id, operations_json, initial_json) VALUES (?1, ?2, ?3)", rusqlite::params![id,
5957            serde_json::to_string(&operations.iter().map(CommitOperation::from).collect::<Vec<_>>()).map_err(|_| "sync.local_corrupt".to_owned())?,
5958            serde_json::to_string(&initial).map_err(|_| "sync.local_corrupt".to_owned())?])
5959            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5960        self.failed_commits.push((
5961            OutboxCommit {
5962                client_commit_id: id.to_owned(),
5963                ops: operations.to_vec(),
5964            },
5965            initial,
5966        ));
5967        Ok(())
5968    }
5969
5970    fn drop_failed_commits(
5971        &mut self,
5972        matches: impl Fn(&str, &Map<String, Value>) -> bool,
5973    ) -> Result<Vec<OutboxCommit>, String> {
5974        let mut doomed = self.drop_acknowledged_rows(&matches)?;
5975        for (commit, initial) in self.failed_commits.iter() {
5976            for operation in &commit.ops {
5977                let key =
5978                    serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
5979                        .map_err(|_| "sync.local_corrupt".to_owned())?;
5980                if initial
5981                    .get(&key)
5982                    .and_then(Value::as_object)
5983                    .is_some_and(|values| matches(&operation.table, values))
5984                {
5985                    doomed.push(commit.clone());
5986                    break;
5987                }
5988            }
5989        }
5990        for commit in &doomed {
5991            self.conn
5992                .execute(
5993                    "DELETE FROM _syncular_failed_commits WHERE client_commit_id = ?1",
5994                    rusqlite::params![commit.client_commit_id],
5995                )
5996                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5997        }
5998        for commit in &doomed {
5999            self.conn.execute("UPDATE _syncular_commit_outcomes SET operations_json = NULL, results_json = '[]' WHERE client_commit_id = ?1", rusqlite::params![commit.client_commit_id]).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6000        }
6001        self.conflicts.retain(|record| {
6002            !doomed
6003                .iter()
6004                .any(|commit| commit.client_commit_id == record.client_commit_id)
6005        });
6006        self.rejections.retain(|record| {
6007            !doomed
6008                .iter()
6009                .any(|commit| commit.client_commit_id == record.client_commit_id)
6010        });
6011        self.failed_commits.retain(|(commit, _)| {
6012            !doomed
6013                .iter()
6014                .any(|dropped| dropped.client_commit_id == commit.client_commit_id)
6015        });
6016        for commit in &doomed {
6017            for operation in &commit.ops {
6018                self.overlay_dirty.table(&operation.table);
6019            }
6020        }
6021        Ok(doomed)
6022    }
6023
6024    /// Record one commit of local mutations. The returned commit ID is the
6025    /// §2.3 idempotency key. Failures are structured so callers read
6026    /// `code`/`message`/`details` instead of parsing a string prefix.
6027    pub fn mutate(&mut self, mutations: Vec<Mutation>) -> Result<String, ClientError> {
6028        self.mutate_with_commit_id(mutations, None)
6029    }
6030
6031    /// [`Self::mutate`] under a caller-supplied `clientCommitId` (§7.1), so an
6032    /// app that loses the reply knows its commit. The ID is non-empty (§6.1);
6033    /// one this replica still holds fails with `client.commit_id_reused`.
6034    pub fn mutate_with_commit_id(
6035        &mut self,
6036        mutations: Vec<Mutation>,
6037        client_commit_id: Option<String>,
6038    ) -> Result<String, ClientError> {
6039        let prior = self.storage_failure.borrow_mut().take();
6040        if let Some(id) = &client_commit_id {
6041            let held = if id.is_empty() {
6042                Err("sync.invalid_request: clientCommitId is empty".to_owned())
6043            } else {
6044                self.conn
6045                    .query_row(
6046                        "SELECT EXISTS(SELECT 1 FROM _syncular_outbox WHERE commit_id=?1 UNION ALL SELECT 1 FROM _syncular_commit_outcomes WHERE client_commit_id=?1)",
6047                        [id],
6048                        |row| row.get(0),
6049                    )
6050                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
6051            };
6052            match held {
6053                Ok(false) => {}
6054                Ok(true) => {
6055                    return self.finish_authoring(
6056                        prior,
6057                        Err(ClientError {
6058                            code: "client.commit_id_reused".to_owned(),
6059                            message: "clientCommitId is held".to_owned(),
6060                            details: Some(json!({ "clientCommitId": id })),
6061                            retryable: false,
6062                        }),
6063                    )
6064                }
6065                Err(message) => return self.finish_authoring(prior, Err(message.into())),
6066            }
6067        }
6068        let outcome = (|| -> Result<String, ClientError> {
6069            if mutations.is_empty() {
6070                return Err(
6071                    "sync.invalid_request: a commit must contain at least one operation (§6.1)"
6072                        .to_owned()
6073                        .into(),
6074                );
6075            }
6076            let mut ops = Vec::with_capacity(mutations.len());
6077            for mutation in mutations {
6078                match mutation {
6079                    Mutation::Patch {
6080                        table,
6081                        values,
6082                        base_version,
6083                    } => {
6084                        let schema_table = self
6085                            .schema
6086                            .table(&table)
6087                            .ok_or_else(|| "sync.unknown_table".to_owned())?;
6088                        let values = normalize_values_casing(schema_table, values)?;
6089                        let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
6090                        ops.push(self.prepare_patch(&table, &row_id, values, base_version)?);
6091                    }
6092                    Mutation::Upsert {
6093                        table,
6094                        values,
6095                        base_version,
6096                    } => {
6097                        let schema_table = self.schema.table(&table).ok_or_else(|| {
6098                            format!("sync.unknown_table: unknown table {table:?}")
6099                        })?;
6100                        // §5: value keys are accepted in snake_case AND the
6101                        // generated row types' camelCase; normalize to SQL truth
6102                        // before the pk lookup / codec see them.
6103                        let values = normalize_values_casing(schema_table, values)?;
6104                        let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
6105                        // §6.7 `mutate` marks every column present: fill missing
6106                        // nullable columns with NULL before the payload is built.
6107                        let values = full_row_values(schema_table, values)?;
6108                        self.refuse_sealed_row(schema_table, &row_id)?;
6109                        // §5.11: validate the payload encodes with the current
6110                        // codec. A key-selection or unknown-key failure is a
6111                        // durable rejection at the push seam (§10.3), never an
6112                        // author-time error. Full-row `mutate` presents every
6113                        // column, so no stored fallback is needed.
6114                        self.validate_author_encode(schema_table, &row_id, &values)?;
6115                        // §7.1 insert intent: the row is absent from the
6116                        // visible replica at call time, before this batch
6117                        // applies anything.
6118                        let insert = !self.visible_row_exists(schema_table, &row_id)?;
6119                        ops.push(OutboxOp {
6120                            upsert: true,
6121                            table,
6122                            row_id,
6123                            base_version,
6124                            values: Some(values),
6125                            insert,
6126                        });
6127                    }
6128                    Mutation::Delete {
6129                        table,
6130                        row_id,
6131                        base_version,
6132                    } => {
6133                        let schema_table = self.schema.table(&table).ok_or_else(|| {
6134                            format!("sync.unknown_table: unknown table {table:?}")
6135                        })?;
6136                        self.refuse_sealed_row(schema_table, &row_id)?;
6137                        // §2.4: the server rejects a rowId holding U+0000.
6138                        if row_id.contains('\0') {
6139                            return Err("sync.invalid_request: rowId holds U+0000 (§2.4)"
6140                                .to_owned()
6141                                .into());
6142                        }
6143                        ops.push(OutboxOp {
6144                            upsert: false,
6145                            table,
6146                            row_id,
6147                            base_version,
6148                            values: None,
6149                            insert: false,
6150                        });
6151                    }
6152                }
6153            }
6154            Ok(self.record_outbox_commit(ops, client_commit_id)?)
6155        })();
6156        self.finish_authoring(prior, outcome)
6157    }
6158
6159    /// Scope the retained `storage_failure` diagnostic to this call so an
6160    /// earlier operation's failure cannot classify a new one, and surface the
6161    /// call's own classified failure with its details. A fresh classified
6162    /// failure replaces the retained diagnostic; otherwise the earlier one is
6163    /// kept for the next sync.
6164    fn finish_authoring(
6165        &self,
6166        prior: Option<QueryReadFailure>,
6167        outcome: Result<String, ClientError>,
6168    ) -> Result<String, ClientError> {
6169        let fresh = self.storage_failure.borrow_mut().take();
6170        *self.storage_failure.borrow_mut() = fresh.clone().or(prior);
6171        outcome.map_err(|error| fresh.map(ClientError::from).unwrap_or(error))
6172    }
6173
6174    fn record_outbox_commit(
6175        &mut self,
6176        ops: Vec<OutboxOp>,
6177        client_commit_id: Option<String>,
6178    ) -> Result<String, String> {
6179        let commit = OutboxCommit {
6180            client_commit_id: client_commit_id.unwrap_or_else(|| uuid::Uuid::new_v4().to_string()),
6181            ops,
6182        };
6183        self.begin_observation("syncular_mutation")?;
6184        let mut batch = ChangeAccumulator::default();
6185        for op in &commit.ops {
6186            let mut precise = self.record_row_scopes(&mut batch, &op.table, &op.row_id, false);
6187            if let Some(values) = &op.values {
6188                if let Some(table) = self.schema.table(&op.table) {
6189                    for scope in &table.scope_variables {
6190                        if let Some(Value::String(value)) = values.get(&scope.column) {
6191                            batch.scope(&op.table, format!("{}:{value}", scope.prefix));
6192                            precise = true;
6193                        }
6194                    }
6195                }
6196            }
6197            if !precise {
6198                batch.table(&op.table);
6199            }
6200        }
6201        if let Err(error) = self.persist_outbox_insert(&commit) {
6202            self.rollback_observation("syncular_mutation");
6203            return Err(error);
6204        }
6205        // A clean overlay contains the FIFO fold of every previous commit, so
6206        // the new commit applies over it. §7.1: an operation that does not
6207        // apply (a secondary unique collision) fails the whole commit; the
6208        // savepoint rollback removes the outbox entry and every visible write.
6209        // The append undoes itself: a rollback capture would copy the outbox.
6210        let was_dirty = self.overlay_dirty.snapshot();
6211        let id = commit.client_commit_id.clone();
6212        let appended = self.rebuild_overlay_if_dirty().and_then(|()| {
6213            commit
6214                .ops
6215                .iter()
6216                .try_for_each(|op| self.apply_outbox_op(op))?;
6217            self.outbox.push(Arc::new(commit));
6218            batch.status = true;
6219            self.finish_observation("syncular_mutation", batch)
6220                .inspect_err(|_| drop(self.outbox.pop()))
6221        });
6222        if let Err(error) = appended {
6223            self.rollback_observation("syncular_mutation");
6224            self.overlay_dirty.restore(was_dirty);
6225            return Err(error);
6226        }
6227        Ok(id)
6228    }
6229
6230    /// Record one §6.1 sparse upsert: the primary key plus the supplied
6231    /// non-scope columns are present; every other column is absent and stays
6232    /// untouched on the server and in the local overlay. An absent local base
6233    /// refuses the whole batch with sync.row_missing before enqueueing (§7.1).
6234    /// Failures are structured so callers read `code`/`message`/`details`.
6235    pub fn patch(
6236        &mut self,
6237        table: &str,
6238        row_id: &str,
6239        partial: Map<String, Value>,
6240        base_version: Option<i64>,
6241    ) -> Result<String, ClientError> {
6242        let prior = self.storage_failure.borrow_mut().take();
6243        let outcome = self.perform_patch(table, row_id, partial, base_version);
6244        self.finish_authoring(prior, outcome)
6245    }
6246
6247    /// The unscoped patch body. Public authoring entry points scope
6248    /// `storage_failure` around the whole operation, including any pre-read.
6249    fn perform_patch(
6250        &mut self,
6251        table: &str,
6252        row_id: &str,
6253        partial: Map<String, Value>,
6254        base_version: Option<i64>,
6255    ) -> Result<String, ClientError> {
6256        let operation = self.prepare_patch(table, row_id, partial, base_version)?;
6257        Ok(self.record_outbox_commit(vec![operation], None)?)
6258    }
6259
6260    fn prepare_patch(
6261        &self,
6262        table: &str,
6263        row_id: &str,
6264        partial: Map<String, Value>,
6265        base_version: Option<i64>,
6266    ) -> Result<OutboxOp, ClientError> {
6267        let schema_table = self
6268            .schema
6269            .table(table)
6270            .ok_or_else(|| format!("sync.unknown_table: unknown table {table:?}"))?;
6271        let partial = normalize_values_casing(schema_table, partial)?;
6272        if let Some(pk) = partial.get(&schema_table.primary_key) {
6273            if render_row_id_json(Some(pk)).ok().as_deref() != Some(row_id) {
6274                return Err(invalid_authoring(
6275                    "patch cannot change the primary key",
6276                    json!({ "table": table }),
6277                ));
6278            }
6279        }
6280        if !self.visible_row_exists(schema_table, row_id)? {
6281            return Err("sync.row_missing: a sparse patch requires a local row"
6282                .to_owned()
6283                .into());
6284        }
6285        self.refuse_sealed_row(schema_table, row_id)?;
6286        // §3.4 rule 5 / §6.2: scope columns are immutable on update. The
6287        // server accepts a present scope column whose value equals the
6288        // stored row and applies it as a no-op, so a decoded envelope
6289        // round-trips. The local row is the only value the client can prove
6290        // equality against; when the row is absent locally the client cannot
6291        // prove equality against the server's stored row and fails closed.
6292        // A primary key that is also a scope column needs no stored-row
6293        // read: the primary key in a sparse payload is by construction the
6294        // row id being patched (§6.1), so its value is proven equal already.
6295        for scope in &schema_table.scope_variables {
6296            if !partial.contains_key(&scope.column) || scope.column == schema_table.primary_key {
6297                continue;
6298            }
6299            let Some(column) = schema_table
6300                .columns
6301                .iter()
6302                .find(|column| column.name == scope.column)
6303            else {
6304                continue;
6305            };
6306            let stored = self
6307                .conn
6308                .query_row(
6309                    &format!(
6310                        "SELECT {} FROM {} WHERE {}",
6311                        quote_ident(&scope.column),
6312                        visible_table(&schema_table.name),
6313                        row_id_predicate(schema_table)
6314                    ),
6315                    rusqlite::params![row_id],
6316                    |row| Ok(sql_ref_to_json(column, row.get_ref(0)?)),
6317                )
6318                .optional()
6319                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6320            // Decode the supplied value through the same column-type path the
6321            // rest of the patch encoding uses, so a value that only equals
6322            // the stored one in another number representation still compares
6323            // equal.
6324            let incoming = json_to_column_value(column, partial.get(&scope.column))
6325                .ok()
6326                .map(|value| column_value_to_json(&value));
6327            // A stored blob never equals a patch value, matching the server's
6328            // scope-column comparison (§3.4): bytes are unstorable as scope
6329            // values.
6330            let equal = stored.as_ref().is_some_and(|stored| {
6331                stored.get("$bytes").is_none() && incoming.as_ref() == Some(stored)
6332            });
6333            if !equal {
6334                return Err(invalid_authoring(
6335                    "patch cannot write a scope column (§3.4)",
6336                    json!({ "table": table, "column": scope.column }),
6337                ));
6338            }
6339        }
6340        let mut values = partial;
6341        // Presence-set semantics: a proven-equal scope column is dropped, so
6342        // the patch leaves the stored value untouched (§6.2).
6343        for scope in &schema_table.scope_variables {
6344            values.remove(&scope.column);
6345        }
6346        let pk_value = match schema_table.columns[schema_table.pk_index].ty {
6347            // §2.4: the rowId of an integer key is its canonical decimal form.
6348            ColumnType::Integer => match row_id.parse::<i64>() {
6349                Ok(v)
6350                    if v.to_string() == row_id
6351                        && (-ssp2::primitives::I64_SAFE_MAX..=ssp2::primitives::I64_SAFE_MAX)
6352                            .contains(&v) =>
6353                {
6354                    Value::from(v)
6355                }
6356                _ => {
6357                    return Err(format!(
6358                    "sync.invalid_request: table {table:?}: invalid integer primary key {row_id:?}"
6359                )
6360                    .into())
6361                }
6362            },
6363            ColumnType::Boolean => match row_id {
6364                "true" => Value::Bool(true),
6365                "false" => Value::Bool(false),
6366                _ => {
6367                    return Err(format!(
6368                    "sync.invalid_request: table {table:?}: invalid boolean primary key {row_id:?}"
6369                )
6370                    .into())
6371                }
6372            },
6373            _ => Value::from(row_id.to_owned()),
6374        };
6375        values.insert(schema_table.primary_key.clone(), pk_value);
6376        // §6.1: a patch cannot null a non-nullable column.
6377        if let Some(column) = schema_table
6378            .columns
6379            .iter()
6380            .find(|column| !column.nullable && values.get(&column.name) == Some(&Value::Null))
6381        {
6382            return Err(invalid_authoring(
6383                "a non-nullable column is null (§6.1)",
6384                json!({ "table": table, "column": column.name }),
6385            ));
6386        }
6387        // §5.11: structural validation only; an absent key-id selector is
6388        // resolved at the push seam from the stored local row, and a
6389        // key-selection or unknown-key failure becomes a durable
6390        // `client.encrypt_failed` rejection there (§10.3), never an
6391        // author-time error.
6392        self.validate_author_encode(schema_table, row_id, &values)?;
6393        Ok(OutboxOp {
6394            upsert: true,
6395            table: table.to_owned(),
6396            row_id: row_id.to_owned(),
6397            base_version,
6398            values: Some(values),
6399            insert: false,
6400        })
6401    }
6402
6403    /// §5.11 key-id fallback: the stored local row's selector value as a
6404    /// positional row (only the selector slot filled). Returns None when no
6405    /// selector is configured, the row is locally absent, or the stored
6406    /// value is not a usable key id. A failed storage read is an error.
6407    fn stored_key_fallback(
6408        &self,
6409        table: &TableSchema,
6410        row_id: &str,
6411    ) -> Result<Option<Row>, String> {
6412        let Some(selector) = self.encryption.key_id_columns.get(&table.name) else {
6413            return Ok(None);
6414        };
6415        let Some(index) = table
6416            .columns
6417            .iter()
6418            .position(|column| &column.name == selector)
6419        else {
6420            return Ok(None);
6421        };
6422        let sql = format!(
6423            "SELECT {} FROM {} WHERE {}",
6424            quote_ident(selector),
6425            visible_table(&table.name),
6426            row_id_predicate(table)
6427        );
6428        let key = self
6429            .conn
6430            .query_row(&sql, rusqlite::params![row_id], |row| {
6431                row.get::<_, Option<String>>(0)
6432            })
6433            .optional()
6434            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
6435            .flatten()
6436            .filter(|key| !key.is_empty());
6437        let Some(key) = key else {
6438            return Ok(None);
6439        };
6440        let mut fallback: Row = vec![None; table.columns.len()];
6441        fallback[index] = Some(ColumnValue::String(key));
6442        Ok(Some(fallback))
6443    }
6444
6445    /// §5.11/§10.3 author-time encode check: a structural failure (a
6446    /// non-nullable NULL, a value the codec rejects) is an author-time
6447    /// error; a key-selection or unknown-key failure is deferred to the push
6448    /// seam, where it becomes a durable `client.encrypt_failed` rejection.
6449    fn validate_author_encode(
6450        &self,
6451        table: &TableSchema,
6452        row_id: &str,
6453        values: &Map<String, Value>,
6454    ) -> Result<(), String> {
6455        match encode_sparse_row_json(table, row_id, values, &self.encryption, None) {
6456            Err(error) if error.starts_with(ENCRYPT_FAILED_CODE) => Ok(()),
6457            result => result.map(|_| ()),
6458        }
6459    }
6460
6461    /// §5.11/§10.3: drop every pending commit whose push encode fails with
6462    /// `client.encrypt_failed` (an unresolvable key id or an unknown key) and
6463    /// raise one durable rejection per commit. `sync()` never aborts for an
6464    /// encode failure. Mirrors §7.4.4's `drop_incompatible_outbox`, including
6465    /// undoing the dropped commit's purely-optimistic rows.
6466    /// §7.1/§7.4.4: drop pending commits whose values the current codec refuses
6467    /// (an envelope or a type an earlier version accepted). `with_encryption`
6468    /// additionally drops commits whose §5.11 encryption cannot resolve a key;
6469    /// the startup boundary leaves those to the send seam, where hosts configure
6470    /// keys after open.
6471    /// Recovery runs as one observation. An enclosing observation (the guarded
6472    /// schema reset) passes its `batch`, so the rejection, outcome, and table
6473    /// domains join that change event instead of publishing a second one.
6474    fn drop_unencodable_outbox(
6475        &mut self,
6476        with_encryption: bool,
6477        batch: Option<&mut ChangeAccumulator>,
6478    ) -> Result<bool, String> {
6479        let mut failures = Vec::new();
6480        for commit in self.outbox.iter() {
6481            for op in &commit.ops {
6482                let Some(table) = self.schema.table(&op.table).filter(|_| op.upsert) else {
6483                    continue;
6484                };
6485                // An insert lacking a non-nullable column is left to
6486                // `drop_incompatible_outbox` (§7.4.4).
6487                let Some(values) = op.presented_values(table) else {
6488                    continue;
6489                };
6490                let fallback = self.stored_key_fallback(table, &op.row_id)?;
6491                let failure = match encode_sparse_row_json(
6492                    table,
6493                    &op.row_id,
6494                    &values,
6495                    &self.encryption,
6496                    fallback.as_ref(),
6497                ) {
6498                    Err(error) if error.starts_with(ENCRYPT_FAILED_CODE) => {
6499                        with_encryption.then_some((error, ENCRYPT_FAILED_CODE))
6500                    }
6501                    // §7.1/§7.4.4: a value the current codec refuses was
6502                    // persisted by a version that accepted it; it leaves
6503                    // through the incompatible-outbox path, so the FIFO
6504                    // queue never wedges.
6505                    Err(error) if error.starts_with(INVALID_REQUEST_CODE) => Some((
6506                        LEGACY_VALUE_REFUSAL_MESSAGE.to_owned(),
6507                        OUTBOX_INCOMPATIBLE_CODE,
6508                    )),
6509                    _ => None,
6510                };
6511                if let Some((error, code)) = failure {
6512                    failures.push((commit.clone(), error, code));
6513                    break;
6514                }
6515            }
6516        }
6517        if failures.is_empty() {
6518            return Ok(false);
6519        }
6520        // §7.4.4: one observation covers the durable outcome writes, the outbox
6521        // removal, the in-memory drop, and the overlay rebuild, so a storage fault
6522        // leaves the replica and the client unchanged, a successful recovery
6523        // publishes one batch with the rejection, outcome, and table domains, and
6524        // an ordinary open with nothing to recover publishes nothing.
6525        let rollback = self.capture();
6526        let owns_observation = batch.is_none();
6527        let owns_transaction = self.conn.is_autocommit();
6528        let mut domains = ChangeAccumulator::default();
6529        if owns_observation {
6530            self.begin_observation("syncular_value_recovery")?;
6531        }
6532        let dropped = (|| -> Result<(), String> {
6533            for (commit, message, code) in failures {
6534                if self.retain_failed_commits && code == ENCRYPT_FAILED_CODE {
6535                    self.retain_failed_commit(&commit.client_commit_id, &commit.ops)?;
6536                }
6537                let representable = commit.ops.iter().all(Self::outbox_op_is_journalable);
6538                let rejection = RejectionRecord {
6539                    client_commit_id: commit.client_commit_id.clone(),
6540                    op_index: 0,
6541                    code: code.to_owned(),
6542                    message,
6543                    retryable: false,
6544                    details: (code == OUTBOX_INCOMPATIBLE_CODE).then(|| RejectionDetails {
6545                        field_paths: None,
6546                        reason: Some(INVALID_STORED_VALUES_REASON.to_owned()),
6547                        required_action: None,
6548                        references: None,
6549                    }),
6550                    operation: representable
6551                        .then(|| commit.ops.first())
6552                        .flatten()
6553                        .map(CommitOperation::from),
6554                };
6555                self.persist_commit_outcome(
6556                    &commit.client_commit_id,
6557                    CommitOutcomeStatus::Rejected,
6558                    &[CommitOperationOutcome::Error {
6559                        rejection: rejection.clone(),
6560                    }],
6561                    representable.then_some(commit.ops.as_slice()),
6562                )?;
6563                self.delete_outbox_persisted(&commit.client_commit_id)?;
6564                self.outbox
6565                    .retain(|candidate| candidate.client_commit_id != commit.client_commit_id);
6566                self.rejections.push(rejection);
6567                for operation in &commit.ops {
6568                    domains.table(operation.table.as_str());
6569                }
6570            }
6571            domains.status = true;
6572            domains.rejections = true;
6573            domains.outcomes = true;
6574            self.prune_commit_outcomes()?;
6575            // The visible projection is re-derived from the surviving base + outbox.
6576            self.overlay_dirty.set(true);
6577            self.rebuild_overlay()?;
6578            Ok(())
6579        })();
6580        let result = match batch {
6581            Some(batch) => {
6582                if dropped.is_ok() {
6583                    for table in domains.tables.keys() {
6584                        batch.table(table.as_str());
6585                    }
6586                    batch.status |= domains.status;
6587                    batch.rejections |= domains.rejections;
6588                    batch.outcomes |= domains.outcomes;
6589                }
6590                dropped
6591            }
6592            None => {
6593                dropped.and_then(|()| self.finish_observation("syncular_value_recovery", domains))
6594            }
6595        };
6596        if let Err(error) = result {
6597            if owns_observation {
6598                if owns_transaction {
6599                    if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
6600                        let mut failure = self.storage_failure.borrow_mut();
6601                        let failure =
6602                            failure.get_or_insert_with(|| QueryReadFailure::from(error.clone()));
6603                        failure.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
6604                    }
6605                } else {
6606                    self.rollback_observation("syncular_value_recovery");
6607                }
6608            }
6609            self.restore(rollback);
6610            return Err(error);
6611        }
6612        Ok(true)
6613    }
6614
6615    pub fn pending_commit_ids(&self) -> Vec<String> {
6616        self.outbox
6617            .iter()
6618            .map(|c| c.client_commit_id.clone())
6619            .collect()
6620    }
6621
6622    /// §6.1: the sparse push payload bytes of every pending upsert operation,
6623    /// FIFO — exactly what the next push round would put on the wire. Used by
6624    /// conformance B.21(h) to pin cross-core byte identity.
6625    pub fn pending_payloads(&self) -> Result<Vec<Vec<u8>>, String> {
6626        let mut payloads = Vec::new();
6627        for commit in self.outbox.iter() {
6628            for op in &commit.ops {
6629                let Some(table) = self.schema.table(&op.table).filter(|_| op.upsert) else {
6630                    continue;
6631                };
6632                let Some(values) = op.presented_values(table) else {
6633                    continue;
6634                };
6635                let fallback = self.stored_key_fallback(table, &op.row_id)?;
6636                if let Ok(payload) = encode_sparse_row_json(
6637                    table,
6638                    &op.row_id,
6639                    &values,
6640                    &self.encryption,
6641                    fallback.as_ref(),
6642                ) {
6643                    payloads.push(payload);
6644                }
6645            }
6646        }
6647        Ok(payloads)
6648    }
6649
6650    pub fn conflicts(&self) -> &[ConflictRecord] {
6651        &self.conflicts
6652    }
6653
6654    pub fn rejections(&self) -> &[RejectionRecord] {
6655        &self.rejections
6656    }
6657
6658    pub fn resolve_commit_outcome(
6659        &mut self,
6660        input: ResolveCommitOutcomeInput,
6661    ) -> Result<CommitOutcome, String> {
6662        let current = self
6663            .commit_outcome(&input.client_commit_id)?
6664            .ok_or_else(|| {
6665                format!(
6666                    "sync.outcome_not_found: no durable outcome exists for {:?}",
6667                    input.client_commit_id
6668                )
6669            })?;
6670        if current.resolution != CommitOutcomeResolution::Active {
6671            return Ok(current);
6672        }
6673        if input.resolution == CommitOutcomeResolution::Active {
6674            return Err("sync.invalid_request: resolution must leave active state".to_owned());
6675        }
6676        match input.resolution {
6677            CommitOutcomeResolution::Superseded => {
6678                let replacement = input
6679                    .replacement_client_commit_id
6680                    .as_deref()
6681                    .filter(|value| !value.is_empty() && *value != input.client_commit_id)
6682                    .ok_or_else(|| {
6683                        "sync.invalid_request: superseded outcomes require a distinct replacementClientCommitId"
6684                            .to_owned()
6685                    })?;
6686                let _ = replacement;
6687            }
6688            _ if input.replacement_client_commit_id.is_some() => {
6689                return Err(
6690                    "sync.invalid_request: replacementClientCommitId is valid only for superseded outcomes"
6691                        .to_owned(),
6692                );
6693            }
6694            _ => {}
6695        }
6696        let allowed = match current.status {
6697            CommitOutcomeStatus::Conflict => matches!(
6698                input.resolution,
6699                CommitOutcomeResolution::ResolvedKeepServer | CommitOutcomeResolution::Superseded
6700            ),
6701            CommitOutcomeStatus::Rejected => {
6702                matches!(
6703                    input.resolution,
6704                    CommitOutcomeResolution::Superseded
6705                        | CommitOutcomeResolution::ResolvedKeepServer
6706                )
6707            }
6708            CommitOutcomeStatus::Applied | CommitOutcomeStatus::Cached => {
6709                input.resolution == CommitOutcomeResolution::Dismissed
6710            }
6711        };
6712        if !allowed {
6713            return Err(format!(
6714                "sync.invalid_request: resolution {:?} is invalid for {:?} outcome",
6715                input.resolution, current.status
6716            ));
6717        }
6718
6719        let rollback = self.capture();
6720        self.begin_observation("syncular_outcome_resolution")?;
6721        let result = (|| {
6722            self.conn
6723                .execute(
6724                    "UPDATE _syncular_commit_outcomes
6725                        SET resolution = ?1, resolved_at_ms = ?2,
6726                            replacement_client_commit_id = ?3
6727                      WHERE client_commit_id = ?4 AND resolution = 'active'",
6728                    rusqlite::params![
6729                        Self::outcome_resolution_name(input.resolution),
6730                        self.clock_now_ms(),
6731                        input.replacement_client_commit_id,
6732                        input.client_commit_id
6733                    ],
6734                )
6735                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6736            let resolved = self
6737                .commit_outcome(&current.client_commit_id)?
6738                .ok_or_else(|| "sync.outcome_not_found: outcome disappeared".to_owned())?;
6739            self.conn
6740                .execute(
6741                    "DELETE FROM _syncular_failed_commits WHERE client_commit_id = ?1",
6742                    rusqlite::params![input.client_commit_id],
6743                )
6744                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6745            for (commit, _) in self.failed_commits.iter() {
6746                if commit.client_commit_id == input.client_commit_id {
6747                    for operation in &commit.ops {
6748                        self.overlay_dirty.table(&operation.table);
6749                    }
6750                }
6751            }
6752            self.failed_commits
6753                .retain(|(commit, _)| commit.client_commit_id != input.client_commit_id);
6754            self.prune_commit_outcomes()?;
6755            Ok(resolved)
6756        })();
6757        let resolved = match result {
6758            Ok(outcome) => outcome,
6759            Err(error) => {
6760                self.rollback_observation("syncular_outcome_resolution");
6761                self.restore(rollback);
6762                return Err(error);
6763            }
6764        };
6765        self.conflicts
6766            .retain(|record| record.client_commit_id != current.client_commit_id);
6767        self.rejections
6768            .retain(|record| record.client_commit_id != current.client_commit_id);
6769        let mut batch = ChangeAccumulator {
6770            conflicts: current.status == CommitOutcomeStatus::Conflict,
6771            rejections: current.status == CommitOutcomeStatus::Rejected,
6772            outcomes: true,
6773            ..ChangeAccumulator::default()
6774        };
6775        if let Some(operations) = &current.operations {
6776            for operation in operations {
6777                batch.table(&operation.table);
6778            }
6779        }
6780        if let Err(error) = self
6781            .rebuild_overlay_if_dirty()
6782            .and_then(|()| self.finish_observation("syncular_outcome_resolution", batch))
6783        {
6784            self.rollback_observation("syncular_outcome_resolution");
6785            self.restore(rollback);
6786            return Err(error);
6787        }
6788        Ok(resolved)
6789    }
6790
6791    pub fn schema_floor(&self) -> Option<&SchemaFloor> {
6792        self.schema_floor.as_ref()
6793    }
6794
6795    /// §7.3.5: the client's opaque auth-lease state, if any.
6796    pub fn lease_state(&self) -> Option<&LeaseState> {
6797        self.lease_state.as_ref()
6798    }
6799
6800    /// §7.3.5: record a request-level lease error (stop-and-surface). Only
6801    /// the two lease codes set it; other errors leave leaseState untouched.
6802    fn record_lease_error(&mut self, code: &str) -> Result<(), String> {
6803        if code != "sync.auth_lease_required" && code != "sync.auth_lease_revoked" {
6804            return Ok(());
6805        }
6806        let mut next = self.lease_state.clone().unwrap_or_default();
6807        next.error_code = Some(code.to_owned());
6808        self.set_lease_state(Some(next))
6809    }
6810
6811    fn set_lease_state(&mut self, next: Option<LeaseState>) -> Result<(), String> {
6812        if self.lease_state == next {
6813            return Ok(());
6814        }
6815        self.begin_observation("syncular_lease")?;
6816        let rollback = self.capture();
6817        self.lease_state = next;
6818        let persisted = match &self.lease_state {
6819            Some(lease) => serde_json::to_string(lease)
6820                .map_err(|error| error.to_string())
6821                .and_then(|json| self.set_meta(LEASE_STATE_KEY, &json)),
6822            None => self.delete_meta(LEASE_STATE_KEY),
6823        };
6824        let batch = ChangeAccumulator {
6825            status: true,
6826            ..ChangeAccumulator::default()
6827        };
6828        if let Err(error) =
6829            persisted.and_then(|()| self.finish_observation("syncular_lease", batch))
6830        {
6831            self.rollback_observation("syncular_lease");
6832            self.restore(rollback);
6833            return Err(error);
6834        }
6835        Ok(())
6836    }
6837
6838    fn set_schema_floor(&mut self, next: Option<SchemaFloor>) -> Result<(), String> {
6839        if self.schema_floor == next {
6840            return Ok(());
6841        }
6842        self.begin_observation("syncular_schema_floor")?;
6843        let rollback = self.capture();
6844        self.stopped = next.is_some();
6845        self.schema_floor = next;
6846        let persisted = match &self.schema_floor {
6847            Some(floor) => serde_json::to_string(floor)
6848                .map_err(|error| error.to_string())
6849                .and_then(|json| self.set_meta(SCHEMA_FLOOR_KEY, &json)),
6850            None => self.delete_meta(SCHEMA_FLOOR_KEY),
6851        };
6852        let batch = ChangeAccumulator {
6853            status: true,
6854            ..ChangeAccumulator::default()
6855        };
6856        if let Err(error) =
6857            persisted.and_then(|()| self.finish_observation("syncular_schema_floor", batch))
6858        {
6859            self.rollback_observation("syncular_schema_floor");
6860            self.restore(rollback);
6861            return Err(error);
6862        }
6863        Ok(())
6864    }
6865
6866    fn set_upgrading(&mut self, value: bool) {
6867        if self.upgrading == value {
6868            return;
6869        }
6870        if self.begin_observation("syncular_upgrading").is_err() {
6871            return;
6872        }
6873        self.upgrading = value;
6874        let batch = ChangeAccumulator {
6875            status: true,
6876            ..ChangeAccumulator::default()
6877        };
6878        if self
6879            .finish_observation("syncular_upgrading", batch)
6880            .is_err()
6881        {
6882            self.rollback_observation("syncular_upgrading");
6883        }
6884    }
6885
6886    pub fn sync_needed(&self) -> bool {
6887        self.sync_needed
6888    }
6889
6890    pub fn read_rows(&self, table: &str) -> Result<Vec<RowState>, String> {
6891        let schema_table = self
6892            .schema
6893            .table(table)
6894            .ok_or_else(|| format!("unknown table {table:?}"))?;
6895        let sql = format!(
6896            "SELECT * FROM {} ORDER BY {} ASC",
6897            visible_table(table),
6898            quote_ident(&schema_table.primary_key)
6899        );
6900        let mut stmt = self
6901            .conn
6902            .prepare(&sql)
6903            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6904        let mut rows = stmt
6905            .query([])
6906            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6907        let mut out = Vec::new();
6908        while let Some(row) = rows
6909            .next()
6910            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
6911        {
6912            let mut values = Map::new();
6913            for (i, column) in schema_table.columns.iter().enumerate() {
6914                let value = row
6915                    .get_ref(i)
6916                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6917                values.insert(column.name.clone(), sql_ref_to_json(column, value));
6918            }
6919            let version: i64 = row
6920                .get(schema_table.columns.len())
6921                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6922            let row_id = match values.get(&schema_table.primary_key) {
6923                Some(Value::String(s)) => s.clone(),
6924                Some(Value::Number(n)) => n.to_string(),
6925                Some(Value::Bool(b)) => b.to_string(),
6926                other => format!("{}", other.cloned().unwrap_or(Value::Null)),
6927            };
6928            let sealed_key = if schema_table.has_encrypted_columns() {
6929                row.get(schema_table.columns.len() + 1)
6930                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
6931            } else {
6932                None
6933            };
6934            out.push(RowState {
6935                row_id,
6936                version,
6937                values,
6938                sealed_key,
6939            });
6940        }
6941        Ok(out)
6942    }
6943
6944    // -- §5.10.5 native CRDT (the `crdt-yjs` feature) --------------------------
6945    //
6946    // The Rust face of the §5.10.4 client model: a local crdt edit loads the
6947    // current stored (server-merged ⊕ pending-overlay) column bytes, applies
6948    // the op with `yrs`, re-encodes the whole doc state, and records a
6949    // baseVersion-less sparse upsert carrying only the primary key and the
6950    // crdt column (§5.10.3, §6.6). No local merge — merging is server-side;
6951    // the overlay re-materializes the edit immediately (optimistic apply,
6952    // §7.1) and the server-merged bytes arrive on the next pull, idempotently.
6953    // Byte-compatible with `@syncular/crdt-yjs`.
6954
6955    /// The current stored value of a `crdt` column for one row — the visible
6956    /// (optimistic) bytes, or `None` when the row is absent or the column is
6957    /// NULL (the empty document, §5.10.1). Errors if the column is not a
6958    /// `crdt` column (guards the app against a typo'd column name).
6959    #[cfg(feature = "crdt-yjs")]
6960    fn crdt_column_bytes(
6961        &self,
6962        table: &str,
6963        row_id: &str,
6964        column: &str,
6965    ) -> Result<Option<Vec<u8>>, String> {
6966        let schema_table = self
6967            .schema
6968            .table(table)
6969            .ok_or_else(|| format!("unknown table {table:?}"))?;
6970        let col = schema_table
6971            .columns
6972            .iter()
6973            .find(|c| c.name == column)
6974            .ok_or_else(|| format!("table {table:?} has no column {column:?}"))?;
6975        if col.ty != ColumnType::Crdt {
6976            return Err(format!("column {column:?} is not a crdt column (§5.10.1)"));
6977        }
6978        let sql = format!(
6979            "SELECT {} FROM {} WHERE {}",
6980            quote_ident(column),
6981            visible_table(table),
6982            row_id_predicate(schema_table)
6983        );
6984        let bytes: Option<Vec<u8>> = self
6985            .conn
6986            .prepare_cached(&sql)
6987            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
6988            .query_row(rusqlite::params![row_id], |row| {
6989                row.get::<_, Option<Vec<u8>>>(0)
6990            })
6991            .map_err(|e| match e {
6992                rusqlite::Error::QueryReturnedNoRows => "no such row".to_owned(),
6993                other => Self::sqlite_failure(&self.storage_failure, other),
6994            })?;
6995        Ok(bytes)
6996    }
6997
6998    /// §5.10.4 materialize: the collaborative text of a `crdt` column, decoded
6999    /// from the stored bytes with `yrs` — `YjsColumn.text(name).toString()`.
7000    /// An absent row / NULL column is the empty document (empty string).
7001    #[cfg(feature = "crdt-yjs")]
7002    pub fn crdt_text(
7003        &self,
7004        table: &str,
7005        row_id: &str,
7006        column: &str,
7007        name: &str,
7008    ) -> Result<String, String> {
7009        let bytes = self
7010            .crdt_column_bytes(table, row_id, column)?
7011            .unwrap_or_default();
7012        crate::crdt::text(&bytes, name)
7013    }
7014
7015    /// §5.10.4 push-an-update: apply a text insert to a `crdt` column and
7016    /// record the resulting crdt-only sparse operation. Returns the enqueued
7017    /// `clientCommitId`.
7018    #[cfg(feature = "crdt-yjs")]
7019    pub fn crdt_insert_text(
7020        &mut self,
7021        table: &str,
7022        row_id: &str,
7023        column: &str,
7024        name: &str,
7025        index: u32,
7026        value: &str,
7027    ) -> Result<String, ClientError> {
7028        let prior = self.storage_failure.borrow_mut().take();
7029        let outcome = (|| -> Result<String, ClientError> {
7030            let current = self
7031                .crdt_column_bytes(table, row_id, column)?
7032                .unwrap_or_default();
7033            let update = crate::crdt::insert_text(&current, name, index, value)?;
7034            self.perform_crdt_push_update(table, row_id, column, &update)
7035        })();
7036        self.finish_authoring(prior, outcome)
7037    }
7038
7039    /// §5.10.4 push-an-update: apply a text delete to a `crdt` column and
7040    /// record the resulting crdt-only sparse operation. Returns the enqueued
7041    /// `clientCommitId`.
7042    #[cfg(feature = "crdt-yjs")]
7043    pub fn crdt_delete_text(
7044        &mut self,
7045        table: &str,
7046        row_id: &str,
7047        column: &str,
7048        name: &str,
7049        index: u32,
7050        len: u32,
7051    ) -> Result<String, ClientError> {
7052        let prior = self.storage_failure.borrow_mut().take();
7053        let outcome = (|| -> Result<String, ClientError> {
7054            let current = self
7055                .crdt_column_bytes(table, row_id, column)?
7056                .unwrap_or_default();
7057            let update = crate::crdt::delete_text(&current, name, index, len)?;
7058            self.perform_crdt_push_update(table, row_id, column, &update)
7059        })();
7060        self.finish_authoring(prior, outcome)
7061    }
7062
7063    /// §5.10.4 generic escape hatch: apply an arbitrary Yjs update onto a
7064    /// `crdt` column's current state and record the resulting crdt-only sparse
7065    /// operation. The app authored the update with its own `yrs` model.
7066    /// Returns the enqueued `clientCommitId`.
7067    #[cfg(feature = "crdt-yjs")]
7068    pub fn crdt_apply_update(
7069        &mut self,
7070        table: &str,
7071        row_id: &str,
7072        column: &str,
7073        update: &[u8],
7074    ) -> Result<String, ClientError> {
7075        let prior = self.storage_failure.borrow_mut().take();
7076        let outcome = (|| -> Result<String, ClientError> {
7077            let current = self
7078                .crdt_column_bytes(table, row_id, column)?
7079                .unwrap_or_default();
7080            let next = crate::crdt::apply_update(&current, update)?;
7081            self.perform_crdt_push_update(table, row_id, column, &next)
7082        })();
7083        self.finish_authoring(prior, outcome)
7084    }
7085
7086    /// Shared tail of the crdt edit methods: record a §6.1 sparse crdt-only
7087    /// upsert — the primary key plus the crdt column, no `baseVersion` — and
7088    /// enqueue it. The server merges the crdt column and leaves every other
7089    /// column untouched (§5.10.3), so a baseVersion-less crdt edit cannot
7090    /// clobber a concurrent edit to another column. A locally absent row
7091    /// records the same partial operation; the server answers per §6.2.
7092    #[cfg(feature = "crdt-yjs")]
7093    fn perform_crdt_push_update(
7094        &mut self,
7095        table: &str,
7096        row_id: &str,
7097        column: &str,
7098        crdt_bytes: &[u8],
7099    ) -> Result<String, ClientError> {
7100        let mut bytes_obj = Map::new();
7101        bytes_obj.insert("$bytes".to_owned(), Value::from(bytes_to_hex(crdt_bytes)));
7102        let mut partial = Map::new();
7103        partial.insert(column.to_owned(), Value::Object(bytes_obj));
7104        self.perform_patch(table, row_id, partial, None)
7105    }
7106
7107    /// Run an arbitrary read-only SQL query against the local database and
7108    /// return each row as a `column-name → JSON value` map. This is the seam
7109    /// the React `useSyncQuery` live-query API needs (it takes app-authored
7110    /// SQL over the visible tables/views, not a fixed table read like
7111    /// [`read_rows`]).
7112    ///
7113    /// Bound `params` are the driver value forms: JSON strings/numbers/bools/
7114    /// null bind directly; a `{"$bytes": hex}` object binds as a BLOB — the
7115    /// same envelope the command surface uses everywhere else. Output BLOB
7116    /// columns come back as `{"$bytes": hex}` to round-trip cleanly.
7117    ///
7118    /// The result column typing is dynamic (SQLite's stored affinity), because
7119    /// arbitrary SQL can alias, join, and compute — there is no schema column
7120    /// to consult per output cell, unlike [`read_rows`].
7121    pub fn query(&self, sql: &str, params: &[QueryValue]) -> Result<Vec<QueryRow>, String> {
7122        query_connection(&self.conn, sql, params).map_err(|failure| failure.to_string())
7123    }
7124
7125    /// Repository benchmark and conformance access, present only with `bench-internals`.
7126    #[cfg(feature = "bench-internals")]
7127    #[doc(hidden)]
7128    pub fn benchmark_connection(&mut self) -> &mut Connection {
7129        &mut self.conn
7130    }
7131
7132    /// Private benchmark intervals; absent from ordinary builds and diagnostics.
7133    #[cfg(feature = "bench-internals")]
7134    #[doc(hidden)]
7135    pub fn benchmark_phases(
7136        &self,
7137        enabled: Option<bool>,
7138        reset: bool,
7139    ) -> Result<Option<Value>, String> {
7140        self.benchmark_phases.configure(enabled, reset)?;
7141        Ok(self.benchmark_phases.snapshot())
7142    }
7143
7144    /// One serialized, read-only authority observation, including durable coverage.
7145    pub fn authority_snapshot(&self) -> Result<Value, String> {
7146        validate_authority_reads(&self.schema, &self.limits.authority_reads)?;
7147        if self.limits.authority_reads.is_empty() {
7148            return Err("client.authority_read_forbidden: no authority reads were declared at client creation".into());
7149        }
7150        self.conn
7151            .execute_batch("SAVEPOINT authority_snapshot")
7152            .map_err(|e| e.to_string())?;
7153        let result = (|| {
7154            let revision = self
7155                .get_meta(LOCAL_REVISION_KEY)?
7156                .ok_or("sync.local_corrupt: authority revision is missing")?;
7157            let parsed_revision = revision
7158                .parse::<u64>()
7159                .map_err(|_| "sync.local_corrupt: authority revision is invalid")?;
7160            if parsed_revision.to_string() != revision {
7161                return Err("sync.local_corrupt: authority revision is invalid".into());
7162            }
7163            let mut tables = Vec::new();
7164            for read in &self.limits.authority_reads {
7165                let table = self
7166                    .schema
7167                    .table(&read.table)
7168                    .ok_or("client.authority_read_forbidden: authority table is not declared")?;
7169                let scalar = |column: &str, value: &Value| -> Result<String, String> {
7170                    let ty = table
7171                        .columns
7172                        .iter()
7173                        .find(|c| c.name == column)
7174                        .expect("validated authority column")
7175                        .ty;
7176                    if ty == ColumnType::Boolean {
7177                        if let Some(value) = value.as_bool() {
7178                            return Ok(value.to_string());
7179                        }
7180                        return match value.as_i64() {
7181                            Some(0) => Ok("false".into()),
7182                            Some(1) => Ok("true".into()),
7183                            _ => Err("sync.local_corrupt: authority boolean is invalid".into()),
7184                        };
7185                    }
7186                    match value {
7187                        Value::String(s) => Ok(s.clone()),
7188                        Value::Number(n) => Ok(n.to_string()),
7189                        Value::Object(o) if ty == ColumnType::Integer && o.len() == 1 => o
7190                            .get("$bigint")
7191                            .and_then(Value::as_str)
7192                            .map(str::to_owned)
7193                            .ok_or("sync.local_corrupt: authority integer is invalid".into()),
7194                        _ => {
7195                            Err("sync.local_corrupt: authority scalar is missing or invalid".into())
7196                        }
7197                    }
7198                };
7199                let matches = |row: &QueryRow| -> Result<bool, String> {
7200                    for (variable, values) in &read.scopes {
7201                        let column = table
7202                            .scope_column(variable)
7203                            .expect("validated authority scope");
7204                        let value = row
7205                            .get(column)
7206                            .ok_or("sync.local_corrupt: authority scope column is missing")?;
7207                        if value.is_null() || !values.contains(&scalar(column, value)?) {
7208                            return Ok(false);
7209                        }
7210                    }
7211                    Ok(true)
7212                };
7213                let selected = read
7214                    .columns
7215                    .iter()
7216                    .map(|c| quote_ident(c))
7217                    .collect::<Vec<_>>()
7218                    .join(",");
7219                let mut intent = BTreeSet::new();
7220                let mut relevant = BTreeSet::new();
7221                for row in query_connection(&self.conn,
7222                    "SELECT id,values_json FROM (SELECT json_extract(value,'$.table') AS tbl,json_extract(value,'$.rowId') AS id,json_extract(value,'$.values') AS values_json FROM _syncular_outbox,json_each(ops_json) UNION ALL SELECT json_extract(value,'$.table'),json_extract(value,'$.rowId'),json_extract(value,'$.values') FROM _syncular_failed_commits,json_each(operations_json) UNION ALL SELECT tbl,id,intent_json FROM _syncular_acked_rows) WHERE tbl=?",
7223                    &[Value::String(read.table.clone())]).map_err(|e| e.to_string())? {
7224                    let id = row.get("id").and_then(Value::as_str).ok_or("sync.local_corrupt: authority intent id is invalid")?.to_owned();
7225                    intent.insert(id.clone());
7226                    if let Some(raw) = row.get("values_json").and_then(Value::as_str) {
7227                        let values: QueryRow = serde_json::from_str(raw).map_err(|_| "sync.local_corrupt: authority intent values are invalid")?;
7228                        if read.scopes.keys().all(|v| values.contains_key(table.scope_column(v).expect("validated scope"))) && matches(&values)? {relevant.insert(id);}
7229                    }
7230                }
7231                let mut rows = BTreeMap::new();
7232                for full in [base_table(&read.table), visible_table(&read.table)] {
7233                    for mut values in query_connection(
7234                        &self.conn,
7235                        &format!(
7236                            "SELECT {selected},_syncular_version AS authority_version FROM {full}"
7237                        ),
7238                        &[],
7239                    )
7240                    .map_err(|e| e.to_string())?
7241                    {
7242                        if !matches(&values)? {
7243                            continue;
7244                        }
7245                        let id = scalar(
7246                            &table.primary_key,
7247                            values
7248                                .get(&table.primary_key)
7249                                .expect("selected primary key"),
7250                        )?;
7251                        relevant.insert(id.clone());
7252                        if full == base_table(&read.table) {
7253                            let version = values
7254                                .remove("authority_version")
7255                                .expect("selected version");
7256                            let version = version
7257                                .as_i64()
7258                                .ok_or("sync.local_corrupt: authority server version is invalid")?;
7259                            if version >= 0 {
7260                                rows.insert(id.clone(), json!({"values": values, "version": version, "hasLocalIntent": intent.contains(&id)}));
7261                            }
7262                        }
7263                    }
7264                }
7265                intent.retain(|id| relevant.contains(id));
7266                let rows = rows.into_values().collect::<Vec<_>>();
7267                // Every unit covering the table, as its (unit, table) pair (§4.4).
7268                let replica = self.stored(replica_cursor(&self.conn))?;
7269                let persisted = self
7270                    .stored(load_units(&self.conn))?
7271                    .into_iter()
7272                    .filter(|unit| covers(table, &unit.scopes))
7273                    .map(|unit| {
7274                        let effective = unit.echo.as_deref().and_then(|echo| {
7275                            pair_scopes(&unit.scopes, &read.table, Some(echo))
7276                        });
7277                        let requested = read
7278                            .scopes
7279                            .keys()
7280                            .map(|v| (v.clone(), json!(unit.scopes.get(v).into_iter().collect::<Vec<_>>())))
7281                            .collect::<Map<_, _>>();
7282                        let complete = effective
7283                            .as_ref()
7284                            .is_some_and(|scopes| scopes.len() == read.scopes.len())
7285                            && unit.status == UnitStatus::Active
7286                            && !unit.bootstrapping();
7287                        let exposed = effective.map(|scopes| {
7288                            read.scopes
7289                                .keys()
7290                                .map(|v| {
7291                                    let held = scopes.iter().find(|(variable, _)| variable == v);
7292                                    (v.clone(), json!(held.map_or(&Vec::new(), |(_, values)| values)))
7293                                })
7294                                .collect::<Map<_, _>>()
7295                        });
7296                        json!({"requestedScopes": requested, "status": unit.status, "cursor": unit.cursor.unwrap_or(replica), "effectiveScopes": exposed, "complete": complete})
7297                    })
7298                    .collect::<Vec<_>>();
7299                fn covered(selectors: &[(&String, &Vec<String>)], candidates: &[&Value]) -> bool {
7300                    match selectors.split_first() {
7301                        None => !candidates.is_empty(),
7302                        Some(((variable, values), rest)) => values.iter().all(|value| {
7303                            covered(
7304                                rest,
7305                                &candidates
7306                                    .iter()
7307                                    .copied()
7308                                    .filter(|p| {
7309                                        p["effectiveScopes"][variable.as_str()]
7310                                            .as_array()
7311                                            .is_some_and(|held| {
7312                                                held.contains(&Value::String(value.clone()))
7313                                                    || held.contains(&Value::String("*".into()))
7314                                            })
7315                                    })
7316                                    .collect::<Vec<_>>(),
7317                            )
7318                        }),
7319                    }
7320                }
7321                let coverage = if covered(
7322                    &read.scopes.iter().collect::<Vec<_>>(),
7323                    &persisted
7324                        .iter()
7325                        .filter(|p| p["complete"] == true)
7326                        .collect::<Vec<_>>(),
7327                ) {
7328                    "complete"
7329                } else if persisted.iter().any(|p| {
7330                    p["status"] == "active"
7331                        && read.scopes.iter().all(|(v, values)| {
7332                            values.iter().any(|value| {
7333                                p["requestedScopes"][v].as_array().is_some_and(|held| {
7334                                    held.contains(&Value::String(value.clone()))
7335                                })
7336                            })
7337                        })
7338                }) {
7339                    "pending"
7340                } else {
7341                    "missing"
7342                };
7343                tables.push(json!({"table": read.table, "rows": rows, "localIntentRowIds": intent, "scopes": read.scopes, "coverage": coverage, "persisted": persisted}));
7344            }
7345            Ok(
7346                json!({"revision": revision, "complete": tables.iter().all(|t| t["coverage"] == "complete"), "tables": tables}),
7347            )
7348        })();
7349        match result {
7350            Ok(snapshot) => {
7351                self.conn
7352                    .execute_batch("RELEASE authority_snapshot")
7353                    .map_err(|e| e.to_string())?;
7354                Ok(snapshot)
7355            }
7356            Err(error) => {
7357                self.conn
7358                    .execute_batch("ROLLBACK TO authority_snapshot; RELEASE authority_snapshot")
7359                    .map_err(|e| format!("{error}; rollback failed: {e}"))?;
7360                Err(error)
7361            }
7362        }
7363    }
7364
7365    /// Rows, coverage, and local revision from one SQLite read snapshot.
7366    /// §7.5 atomic snapshot read. An `owner` records a failure in
7367    /// `diagnostics_snapshot().query_failures` until its next successful read.
7368    pub fn query_snapshot(
7369        &mut self,
7370        sql: &str,
7371        params: &[QueryValue],
7372        coverage: &[WindowCoverage],
7373        owner: Option<QueryOwner<'_>>,
7374    ) -> Result<QuerySnapshot, String> {
7375        let result = snapshot_connection(&self.conn, sql, params, coverage);
7376        if let Some(owner) = owner {
7377            self.record_query_read(owner, result.as_ref().err());
7378        }
7379        result.map_err(|failure| failure.to_string())
7380    }
7381
7382    /// §7.6: record the outcome of one owned snapshot read, including reads a
7383    /// host ran on a separate connection. A success removes the owner's entry;
7384    /// an identical failure leaves it unchanged; a changed failure moves last.
7385    pub fn record_query_read(&mut self, owner: QueryOwner<'_>, failure: Option<&QueryReadFailure>) {
7386        let position = self
7387            .query_failures
7388            .iter()
7389            .position(|entry| entry.id == owner.id);
7390        let Some(failure) = failure else {
7391            if let Some(index) = position {
7392                self.query_failures.remove(index);
7393            }
7394            return;
7395        };
7396        let tables: Vec<String> = BTreeSet::from_iter(owner.tables.iter().copied())
7397            .into_iter()
7398            .map(str::to_owned)
7399            .collect();
7400        let code = failure.code.unwrap_or("client.query_failed").to_owned();
7401        if position.is_some_and(|index| {
7402            let entry = &self.query_failures[index];
7403            entry.tables == tables && entry.code == code && entry.sqlite_code == failure.sqlite_code
7404        }) {
7405            return;
7406        }
7407        if let Some(index) = position {
7408            self.query_failures.remove(index);
7409        }
7410        self.query_failures.push(DiagnosticQueryFailure {
7411            id: owner.id.to_owned(),
7412            tables,
7413            code,
7414            sqlite_code: failure.sqlite_code,
7415            at_ms: self.clock_now_ms(),
7416        });
7417        if self.query_failures.len() > MAX_DIAGNOSTIC_QUERY_FAILURES {
7418            self.query_failures.remove(0);
7419        }
7420    }
7421
7422    // -- request building ---------------------------------------------------------
7423
7424    fn build_request(&self, url_capable: bool) -> Result<(Message, RequestMeta), Box<SyncOutcome>> {
7425        #[cfg(feature = "bench-internals")]
7426        let _phase = self.benchmark_phases.start(Phase::RequestPrepare);
7427        let log_epoch = self.get_meta(LOG_EPOCH_KEY).map_err(|message| {
7428            Box::new(SyncOutcome::Failed {
7429                error_code: "storage.failed".into(),
7430                message,
7431                details: None,
7432                recommended_action: None,
7433            })
7434        })?;
7435        let max_commits = self.limits.max_push_commits_per_request;
7436        let max_ops = self
7437            .limits
7438            .max_push_operations_per_request
7439            .unwrap_or(PUSH_OPS_PER_REQUEST);
7440        let byte_cap = self.limits.max_push_request_bytes;
7441        // §4.2/§5.4: bit 3 is advertised iff the transport can fetch a
7442        // bare URL — capability negotiation, decided per transport.
7443        let accept = self.limits.accept.unwrap_or(if url_capable {
7444            DEFAULT_ACCEPT | ACCEPT_SIGNED_URLS
7445        } else {
7446            DEFAULT_ACCEPT
7447        });
7448        let storage_failed = |message: String| {
7449            Box::new(SyncOutcome::Failed {
7450                error_code: "storage.failed".into(),
7451                message,
7452                details: None,
7453                recommended_action: None,
7454            })
7455        };
7456        let header = Frame::ReqHeader {
7457            client_id: self.client_id.clone(),
7458            schema_version: self.schema.version,
7459            log_epoch: log_epoch.clone(),
7460            outbox_id: self.get_meta(OUTBOX_ID_KEY).map_err(storage_failed)?,
7461        };
7462        // §1.5: the lead frames are REQ_HEADER plus the optional ROUND_FAILURE,
7463        // which stays ahead of every push, pull, unit and progress frame; the
7464        // byte-capacity probes below measure this exact sequence.
7465        let mut lead = vec![header];
7466        lead.extend(
7467            self.round_failure
7468                .iter()
7469                .map(|code| Frame::RoundFailure { code: code.clone() }),
7470        );
7471        let registered: Vec<UnitRecord> = self
7472            .stored(load_units(&self.conn))
7473            .map_err(storage_failed)?
7474            .into_iter()
7475            .filter(|unit| unit.status == UnitStatus::Active)
7476            .collect();
7477        let registration_digest = ssp2::registration::registration_digest(
7478            registered
7479                .iter()
7480                .map(|unit| scope_pairs(&unit.scopes))
7481                .collect::<Vec<_>>()
7482                .iter()
7483                .map(Vec::as_slice),
7484        );
7485        let echo_digest = self
7486            .get_meta(ECHO_DIGEST_KEY)
7487            .map_err(storage_failed)?
7488            .filter(|hex| !hex.is_empty())
7489            .map(|hex| {
7490                crate::values::hex_to_bytes(&hex)
7491                    .ok()
7492                    .and_then(|bytes| <[u8; 32]>::try_from(bytes).ok())
7493                    .ok_or_else(|| {
7494                        "sync.local_corrupt: persisted echo digest is invalid".to_owned()
7495                    })
7496            })
7497            .transpose()
7498            .map_err(storage_failed)?;
7499        let pull = Frame::PullHeader {
7500            limit_commits: self.limits.limit_commits.unwrap_or(0),
7501            limit_snapshot_rows: self.limits.limit_snapshot_rows.unwrap_or(0),
7502            max_snapshot_pages: self.limits.max_snapshot_pages.unwrap_or(0),
7503            accept,
7504            cursor: self
7505                .stored(replica_cursor(&self.conn))
7506                .map_err(storage_failed)?,
7507            registration_digest,
7508            echo_digest,
7509        };
7510        let registration_digest = bytes_to_hex(&registration_digest);
7511        // §4.1: the list travels when it changed since the server stored it,
7512        // after `unregistered` and after a reset (both clear the stored digest).
7513        let travels = self
7514            .get_meta(REGISTERED_DIGEST_KEY)
7515            .map_err(storage_failed)?
7516            .as_deref()
7517            != Some(registration_digest.as_str());
7518        let mut sub_frames: Vec<Frame> = Vec::new();
7519        if travels {
7520            sub_frames.extend(registered.iter().map(|unit| Frame::Unit {
7521                scopes: scope_pairs(&unit.scopes),
7522            }));
7523        }
7524        // §4.3: one PROGRESS per catching-up unit, in registration order.
7525        sub_frames.extend(registered.iter().filter_map(|unit| {
7526            Some(Frame::Progress {
7527                id: unit.id.clone(),
7528                cursor: unit.cursor?,
7529                bootstrap_state: unit.bootstrap_state.clone().map(RawJson),
7530            })
7531        }));
7532        // The fixed frames (order-independent size) and the header+pull probe
7533        // both encode canonically, so the byte budget is the exact wire size.
7534        let fixed_bytes = match byte_cap {
7535            Some(_) => {
7536                let mut frames = lead.clone();
7537                frames.push(pull.clone());
7538                frames.extend(sub_frames.iter().cloned());
7539                encode_message(&Message {
7540                    wire_version: WIRE_VERSION,
7541                    msg_kind: MsgKind::Request,
7542                    frames,
7543                })
7544                .len()
7545            }
7546            None => 0,
7547        };
7548        if let Some(cap) = byte_cap {
7549            if fixed_bytes > cap {
7550                return Err(Self::push_capacity_outcome("bytes", cap, fixed_bytes, None));
7551            }
7552        }
7553        let base_bytes = match byte_cap {
7554            Some(_) => {
7555                let mut frames = lead.clone();
7556                frames.push(pull.clone());
7557                encode_message(&Message {
7558                    wire_version: WIRE_VERSION,
7559                    msg_kind: MsgKind::Request,
7560                    frames,
7561                })
7562                .len()
7563            }
7564            None => 0,
7565        };
7566        let total_outbox = if log_epoch.is_some() {
7567            self.outbox.len()
7568        } else {
7569            0
7570        };
7571        let mut push_frames: Vec<Frame> = Vec::new();
7572        let mut pushed_ids = Vec::new();
7573        let mut ops_in_request = 0usize;
7574        let mut bytes_in_request = 0usize;
7575        let mut deferred_commits = 0usize;
7576        #[cfg(feature = "bench-internals")]
7577        let outbox_phase = self.benchmark_phases.start(Phase::OutboxEncode);
7578        for (index, commit) in self.outbox.iter().take(total_outbox).enumerate() {
7579            // §6.1/§7.1: the commit-count and operation caps are hard. A
7580            // first commit over the operation cap is a typed capacity error
7581            // and stays queued; a later one defers with the complete suffix.
7582            if max_commits.is_some_and(|cap| push_frames.len() >= cap) {
7583                deferred_commits = total_outbox - index;
7584                break;
7585            }
7586            if ops_in_request + commit.ops.len() > max_ops {
7587                if push_frames.is_empty() {
7588                    return Err(Self::push_capacity_outcome(
7589                        "operations",
7590                        max_ops,
7591                        commit.ops.len(),
7592                        Some(commit.client_commit_id.as_str()),
7593                    ));
7594                }
7595                deferred_commits = total_outbox - index;
7596                break;
7597            }
7598            // §7.1 chained base: the commit waits for the change of the commit
7599            // it chains on, which supplies its rebased version.
7600            if self.outbox[..index]
7601                .iter()
7602                .flat_map(|earlier| &earlier.ops)
7603                .any(|earlier| commit.ops.iter().any(|op| chains_on(earlier, op)))
7604            {
7605                deferred_commits = total_outbox - index;
7606                break;
7607            }
7608            let operations = commit
7609                .ops
7610                .iter()
7611                .map(|op| {
7612                    let payload = match self
7613                        .schema
7614                        .table(&op.table)
7615                        .and_then(|table| Some((table, op.presented_values(table)?)))
7616                    {
7617                        Some((table, values)) => {
7618                            // §0: outbox entries encode at send time with the
7619                            // current codec (validated at mutate()). §6.1: the
7620                            // values map's keys are the sparse presence set; an
7621                            // insert-intent operation presents every column (§7.1).
7622                            // §5.11: encrypted columns are encrypted here, with
7623                            // an absent key-id selector read from the stored row.
7624                            // `sync_inner` runs `drop_unencodable_outbox` first,
7625                            // so an encrypt failure cannot reach this encode; the
7626                            // author seam validated every other failure.
7627                            let fallback =
7628                                self.stored_key_fallback(table, &op.row_id)
7629                                    .map_err(|message| {
7630                                        Box::new(SyncOutcome::Failed {
7631                                            error_code: "storage.failed".into(),
7632                                            message,
7633                                            details: None,
7634                                            recommended_action: None,
7635                                        })
7636                                    })?;
7637                            encode_sparse_row_json(
7638                                table,
7639                                &op.row_id,
7640                                &values,
7641                                &self.encryption,
7642                                fallback.as_ref(),
7643                            )
7644                            .ok()
7645                        }
7646                        None => None,
7647                    };
7648                    Ok(ssp2::model::Operation {
7649                        table: op.table.clone(),
7650                        row_id: op.row_id.clone(),
7651                        op: if op.upsert { Op::Upsert } else { Op::Delete },
7652                        base_version: op.base_version,
7653                        payload,
7654                    })
7655                })
7656                .collect::<Result<Vec<_>, Box<SyncOutcome>>>()?;
7657            // §6.3.2, §7.1: the commit's outbox sequence.
7658            let client_seq = self
7659                .conn
7660                .prepare_cached("SELECT seq FROM _syncular_outbox WHERE commit_id = ?1")
7661                .and_then(|mut statement| {
7662                    statement.query_row([&commit.client_commit_id], |row| row.get(0))
7663                })
7664                .map_err(|error| {
7665                    storage_failed(Self::sqlite_failure(&self.storage_failure, error))
7666                })?;
7667            let frame = Frame::PushCommit {
7668                client_commit_id: commit.client_commit_id.clone(),
7669                client_seq,
7670                operations,
7671            };
7672            let frame_bytes = match byte_cap {
7673                Some(_) => {
7674                    let mut frames = lead.clone();
7675                    frames.push(frame.clone());
7676                    frames.push(pull.clone());
7677                    encode_message(&Message {
7678                        wire_version: WIRE_VERSION,
7679                        msg_kind: MsgKind::Request,
7680                        frames,
7681                    })
7682                    .len()
7683                        - base_bytes
7684                }
7685                None => 0,
7686            };
7687            // The head is always encoded so the caller can report the full
7688            // projected request size; a later commit defers once it no longer
7689            // fits.
7690            if let Some(cap) = byte_cap {
7691                if !push_frames.is_empty() && fixed_bytes + bytes_in_request + frame_bytes > cap {
7692                    deferred_commits = total_outbox - index;
7693                    break;
7694                }
7695            }
7696            push_frames.push(frame);
7697            pushed_ids.push(commit.client_commit_id.clone());
7698            ops_in_request += commit.ops.len();
7699            bytes_in_request += frame_bytes;
7700        }
7701        #[cfg(feature = "bench-internals")]
7702        drop(outbox_phase);
7703        if let Some(cap) = byte_cap {
7704            if fixed_bytes + bytes_in_request > cap {
7705                // The head alone does not fit this budget; the commit stays
7706                // queued and no transport round is attempted.
7707                let blocked = if push_frames.is_empty() {
7708                    None
7709                } else {
7710                    self.outbox.first().map(|c| c.client_commit_id.as_str())
7711                };
7712                return Err(Self::push_capacity_outcome(
7713                    "bytes",
7714                    cap,
7715                    fixed_bytes + bytes_in_request,
7716                    blocked,
7717                ));
7718            }
7719        }
7720        let mut frames = lead;
7721        frames.extend(push_frames);
7722        frames.push(pull);
7723        frames.extend(sub_frames);
7724        let message = Message {
7725            wire_version: WIRE_VERSION,
7726            msg_kind: MsgKind::Request,
7727            frames,
7728        };
7729        Ok((
7730            message,
7731            RequestMeta {
7732                pushed_ids,
7733                registration_digest,
7734                accept,
7735                deferred_commits,
7736                applied: BTreeMap::new(),
7737                deferred: BTreeSet::new(),
7738            },
7739        ))
7740    }
7741
7742    /// §7.1: a configured push capacity limit cannot carry this request.
7743    fn push_capacity_outcome(
7744        kind: &str,
7745        limit: usize,
7746        size: usize,
7747        client_commit_id: Option<&str>,
7748    ) -> Box<SyncOutcome> {
7749        let mut details = Map::new();
7750        details.insert("kind".to_owned(), Value::from(kind));
7751        details.insert("limit".to_owned(), Value::from(limit as u64));
7752        details.insert("size".to_owned(), Value::from(size as u64));
7753        if let Some(id) = client_commit_id {
7754            details.insert("clientCommitId".to_owned(), Value::from(id));
7755        }
7756        Box::new(SyncOutcome::Failed {
7757            error_code: "client.push_request_too_large".into(),
7758            message: "the push request exceeds a configured capacity limit".into(),
7759            details: Some(Value::Object(details)),
7760            recommended_action: None,
7761        })
7762    }
7763
7764    // -- sync -------------------------------------------------------------------
7765
7766    /// Cloneable observation handle; no client lock is needed to read it.
7767    pub fn progress(&self) -> ProgressObserver {
7768        self.progress.clone()
7769    }
7770
7771    pub fn sync(&mut self, transport: &mut dyn Transport) -> SyncOutcome {
7772        self.sync_bounded(transport, None)
7773    }
7774
7775    /// One round; `idle_bound` is the idle drive's round bound (§7.8).
7776    fn sync_bounded(
7777        &mut self,
7778        transport: &mut dyn Transport,
7779        idle_bound: Option<(u64, u32)>,
7780    ) -> SyncOutcome {
7781        let mut prepared = match self.prepare_sync_round(transport.supports_url_fetch()) {
7782            Ok(prepared) => prepared,
7783            Err(outcome) => return *outcome,
7784        };
7785        prepared.idle_bound = idle_bound;
7786        // The round's network deadline rides `PreparedSyncRound` and is scoped
7787        // to each exchange, so uploads, continuations, the main request, and
7788        // every segment fetch share one monotonic anchor without leaving
7789        // deadline state on a reused transport.
7790        let mut next = prepared;
7791        loop {
7792            match self.apply_sync_round(next.exchange(transport)) {
7793                crate::AppliedSyncRound::Continue(prepared) => next = prepared,
7794                crate::AppliedSyncRound::Complete {
7795                    outcome, controls, ..
7796                } => {
7797                    if self.transport_enabled {
7798                        for text in controls {
7799                            let _ = transport.realtime_send(&text);
7800                        }
7801                    } else {
7802                        self.disconnect_realtime(transport);
7803                    }
7804                    return outcome;
7805                }
7806            }
7807        }
7808    }
7809
7810    fn finish_sync_round(&mut self, started_at_ms: i64, mut outcome: SyncOutcome) -> SyncOutcome {
7811        if let SyncOutcome::Failed {
7812            error_code,
7813            message,
7814            details,
7815            ..
7816        } = &mut outcome
7817        {
7818            if let Some((failure, code)) = self
7819                .storage_failure
7820                .borrow()
7821                .as_ref()
7822                .and_then(|failure| failure.code.map(|code| (failure, code)))
7823            {
7824                *error_code = code.to_owned();
7825                *message = match code {
7826                    "client.storage_busy" => "local SQLite storage is busy",
7827                    "client.storage_full" => "local SQLite storage is full",
7828                    "client.storage_io" => "local SQLite storage I/O failed",
7829                    "client.storage_corrupt" => "local SQLite storage is corrupt",
7830                    _ => unreachable!("unrecognized classified storage failure"),
7831                }
7832                .to_owned();
7833                *details = failure.details();
7834            }
7835        }
7836        let details = match &outcome {
7837            SyncOutcome::Failed { details, .. } => details.as_ref(),
7838            _ => None,
7839        };
7840        let cause_kind = details
7841            .and_then(|details| details.get("causeKind"))
7842            .and_then(Value::as_str)
7843            .and_then(|cause| match cause {
7844                "network" => Some(crate::TransportCauseKind::Network),
7845                "connect" => Some(crate::TransportCauseKind::Connect),
7846                "tls" => Some(crate::TransportCauseKind::Tls),
7847                "io" => Some(crate::TransportCauseKind::Io),
7848                "request" => Some(crate::TransportCauseKind::Request),
7849                "protocol" => Some(crate::TransportCauseKind::Protocol),
7850                "timeout" => Some(crate::TransportCauseKind::Timeout),
7851                "round_deadline" => Some(crate::TransportCauseKind::RoundDeadline),
7852                "aborted" => Some(crate::TransportCauseKind::Aborted),
7853                "body" => Some(crate::TransportCauseKind::Body),
7854                "status" => Some(crate::TransportCauseKind::Status),
7855                "redirect" => Some(crate::TransportCauseKind::Redirect),
7856                "unknown" => Some(crate::TransportCauseKind::Unknown),
7857                _ => None,
7858            });
7859        let http_status = if matches!(
7860            cause_kind,
7861            Some(
7862                crate::TransportCauseKind::Status
7863                    | crate::TransportCauseKind::Body
7864                    | crate::TransportCauseKind::Redirect
7865            )
7866        ) {
7867            details
7868                .and_then(|details| details.get("httpStatus"))
7869                .and_then(Value::as_f64)
7870                .filter(|status| status.fract() == 0.0 && (100.0..=599.0).contains(status))
7871                .map(|status| status as u16)
7872        } else {
7873            None
7874        };
7875        let retry_delay_ms = self.round_retry_delay_ms;
7876        self.progress.update(|p| match &outcome {
7877            SyncOutcome::Ok(report) | SyncOutcome::BudgetExhausted(report)
7878                if report.failed.is_empty() =>
7879            {
7880                p.state = ProgressState::Complete
7881            }
7882            SyncOutcome::Ok(_) | SyncOutcome::BudgetExhausted(_) => {
7883                p.state = ProgressState::Failed;
7884                p.error_code = Some("sync.scope_revoked".into());
7885            }
7886            SyncOutcome::Failed { error_code, .. } => {
7887                p.state = ProgressState::Failed;
7888                p.error_code = Some(Self::diagnostic_code(error_code));
7889                p.cause_kind = cause_kind;
7890                p.http_status = http_status;
7891                p.retry_delay_ms = retry_delay_ms;
7892            }
7893            SyncOutcome::RealtimeUnavailable { .. } => {
7894                p.state = ProgressState::Failed;
7895                p.error_code = Some(REALTIME_UNAVAILABLE_CODE.to_owned());
7896                p.retry_delay_ms = retry_delay_ms;
7897            }
7898        });
7899        let completed_at_ms = self.clock_now_ms();
7900        self.last_round = Some(match &outcome {
7901            SyncOutcome::Ok(report) | SyncOutcome::BudgetExhausted(report) => DiagnosticLastRound {
7902                status: "succeeded".to_owned(),
7903                started_at_ms,
7904                completed_at_ms,
7905                duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
7906                counters: Some(DiagnosticRoundCounters {
7907                    pushed: report.pushed,
7908                    applied: report.applied.len(),
7909                    rejected: report.rejected.len(),
7910                    retryable: report.retryable.len(),
7911                    conflicts: report.conflicts,
7912                    commits_applied: report.commits_applied,
7913                    segment_rows_applied: report.segment_rows_applied,
7914                    bootstrapping: report.bootstrapping.len(),
7915                    resets: report.resets.len(),
7916                    revoked: report.revoked.len(),
7917                    failed: report.failed.len(),
7918                    deferred_commits: report.deferred_commits,
7919                }),
7920                error_code: None,
7921                cause_kind: None,
7922                http_status: None,
7923            },
7924            SyncOutcome::Failed { error_code, .. } => DiagnosticLastRound {
7925                status: "failed".to_owned(),
7926                started_at_ms,
7927                completed_at_ms,
7928                duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
7929                counters: None,
7930                error_code: Some(Self::diagnostic_code(error_code)),
7931                cause_kind,
7932                http_status,
7933            },
7934            SyncOutcome::RealtimeUnavailable { .. } => DiagnosticLastRound {
7935                status: "failed".to_owned(),
7936                started_at_ms,
7937                completed_at_ms,
7938                duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
7939                counters: None,
7940                error_code: Some(REALTIME_UNAVAILABLE_CODE.to_owned()),
7941                cause_kind: None,
7942                http_status: None,
7943            },
7944        });
7945        outcome
7946    }
7947
7948    /// Capture one request on the client owner. Mutations made after this
7949    /// call remain in the outbox for a later round; no network I/O occurs here.
7950    pub fn prepare_sync_round(
7951        &mut self,
7952        url_capable: bool,
7953    ) -> Result<crate::PreparedSyncRound, Box<SyncOutcome>> {
7954        if !self.transport_enabled {
7955            return Err(Box::new(SyncOutcome::Failed {
7956                error_code: "sync.offline".into(),
7957                message: "transport gate is closed".into(),
7958                details: None,
7959                recommended_action: None,
7960            }));
7961        }
7962        if self.active_round.is_some() {
7963            return Err(Box::new(SyncOutcome::Failed {
7964                error_code: "client.round_in_progress".into(),
7965                message: "a sync round is already in flight".into(),
7966                details: None,
7967                recommended_action: None,
7968            }));
7969        }
7970        self.storage_failure.borrow_mut().take();
7971        self.progress.start();
7972        self.round_retry_delay_ms = None;
7973        let started_at_ms = self.clock_now_ms();
7974        let prepared = (|| {
7975            if self.stopped {
7976                return Err(Box::new(SyncOutcome::Ok(SyncReport {
7977                    schema_floor: self.schema_floor.clone(),
7978                    ..SyncReport::default()
7979                })));
7980            }
7981            if self.security_preflight {
7982                return Err(Box::new(SyncOutcome::Failed {
7983                    error_code: SECURITY_PREFLIGHT_REQUIRED_CODE.into(),
7984                    message: "security preflight must complete before syncing".into(),
7985                    details: None,
7986                    recommended_action: None,
7987                }));
7988            }
7989            self.set_sync_needed(false, false);
7990            self.drain_pending_evictions()
7991                .and_then(|_| self.drop_incompatible_outbox())
7992                .and_then(|_| self.drop_unencodable_outbox(true, None))
7993                .map_err(|message| {
7994                    Box::new(SyncOutcome::Failed {
7995                        error_code: "storage.failed".into(),
7996                        message,
7997                        details: None,
7998                        recommended_action: None,
7999                    })
8000                })?;
8001            // §5.11: reveal rows sealed under a key the keyring gained since.
8002            self.reveal_sealed_rows().map_err(|error| {
8003                Box::new(SyncOutcome::Failed {
8004                    error_code: error.code,
8005                    message: error.message,
8006                    details: error.details,
8007                    recommended_action: None,
8008                })
8009            })?;
8010            let has_log_epoch = self.get_meta(LOG_EPOCH_KEY).map_err(|message| {
8011                Box::new(SyncOutcome::Failed {
8012                    error_code: "storage.failed".into(),
8013                    message,
8014                    details: None,
8015                    recommended_action: None,
8016                })
8017            })?;
8018            let uploads = if has_log_epoch.is_some() && self.schema_has_blobs() {
8019                self.pending_blob_uploads().map_err(|e| {
8020                    Box::new(SyncOutcome::Failed {
8021                        error_code: e.code,
8022                        message: e.message,
8023                        details: e.details,
8024                        recommended_action: None,
8025                    })
8026                })?
8027            } else {
8028                Vec::new()
8029            };
8030            if self.realtime_state != RealtimeState::Connected
8031                && self.realtime_policy == RealtimePolicy::Required
8032            {
8033                let retry_delay_ms = self.schedule_background_retry();
8034                self.realtime_retry_delay_ms = Some(retry_delay_ms);
8035                return Err(Box::new(SyncOutcome::RealtimeUnavailable {
8036                    state: self.realtime_state,
8037                    reason_code: self.realtime_reason_code.clone(),
8038                    retry_delay_ms,
8039                }));
8040            }
8041            let (message, mut meta) = self.build_request(url_capable)?;
8042            let units = self.stored(load_units(&self.conn)).map_err(|message| {
8043                Box::new(SyncOutcome::Failed {
8044                    error_code: "storage.failed".into(),
8045                    message,
8046                    details: None,
8047                    recommended_action: None,
8048                })
8049            })?;
8050            for unit in &units {
8051                let applied =
8052                    self.stored(applied_rows(&self.conn, &unit.id))
8053                        .map_err(|message| {
8054                            Box::new(SyncOutcome::Failed {
8055                                error_code: "storage.failed".into(),
8056                                message,
8057                                details: None,
8058                                recommended_action: None,
8059                            })
8060                        })?;
8061                if !applied.is_empty() {
8062                    meta.applied.insert(unit.id.clone(), applied);
8063                }
8064            }
8065            let id = uuid::Uuid::new_v4();
8066            self.active_round = Some(id);
8067            let phase = if uploads.is_empty() {
8068                crate::round::RoundPhase::Sync
8069            } else {
8070                crate::round::RoundPhase::UploadGrant
8071            };
8072            Ok(crate::PreparedSyncRound {
8073                id,
8074                started_at_ms,
8075                message,
8076                meta,
8077                uploads: uploads.into(),
8078                phase,
8079                encoded: None,
8080                continuation: None,
8081                completed: None,
8082                realtime: self.realtime_state == RealtimeState::Connected,
8083                // §5.5: the owning unit's scope map, each value a one-element list.
8084                scopes: units
8085                    .into_iter()
8086                    .map(|unit| {
8087                        (
8088                            unit.id,
8089                            canonical_scope_json(&unit_scope_lists(&unit.scopes)),
8090                        )
8091                    })
8092                    .collect(),
8093                #[cfg(feature = "bench-internals")]
8094                benchmark_phases: self.benchmark_phases.clone(),
8095                progress: self.progress.clone(),
8096                fixed_now: self
8097                    .now_ms
8098                    .as_ref()
8099                    .map(|clock| clock.load(std::sync::atomic::Ordering::Relaxed)),
8100                round_deadline_at: None,
8101                idle_bound: None,
8102            })
8103        })();
8104        prepared.map_err(|outcome| Box::new(self.finish_sync_round(started_at_ms, *outcome)))
8105    }
8106
8107    /// Discard the reply of an old authorization or registration context. Its
8108    /// submitted commits stay durable and can be retried with the same ids.
8109    pub fn cancel_sync_round(&mut self) {
8110        self.active_round = None;
8111    }
8112
8113    /// Apply captured network results on the owner, preserving all normal
8114    /// version, schema, content-address and cursor checks. No network I/O.
8115    pub fn apply_sync_round(
8116        &mut self,
8117        completed: crate::CompletedSyncRound,
8118    ) -> crate::AppliedSyncRound {
8119        let mut prepared = completed.prepared;
8120        if self.active_round != Some(prepared.id) || self.security_preflight {
8121            return crate::AppliedSyncRound::Complete {
8122                outcome: SyncOutcome::Failed {
8123                    error_code: "client.round_cancelled".into(),
8124                    message: "sync round belongs to a released client context".into(),
8125                    details: None,
8126                    recommended_action: None,
8127                },
8128                controls: Vec::new(),
8129                more: false,
8130                bootstrap_advanced: false,
8131            };
8132        }
8133        self.storage_failure.borrow_mut().take();
8134        let (response, mut downloads, transport_failed) = match completed.exchange {
8135            crate::round::ExchangeResult::Upload { id, result } => {
8136                let result = match result {
8137                    // §5.9.7: a non-retryable server refusal (`blob.too_large`)
8138                    // is the body's final outcome. The commits naming it take
8139                    // the incompatible-commit path with the server's code and
8140                    // leave this round's request; the round continues.
8141                    Err(error)
8142                        if error
8143                            .details
8144                            .as_ref()
8145                            .and_then(|details| details.get("retryable"))
8146                            == Some(&Value::Bool(false)) =>
8147                    {
8148                        self.conn
8149                            .prepare(
8150                                "SELECT commit_id FROM _syncular_blob_commit_refs WHERE blob_id = ?",
8151                            )
8152                            .and_then(|mut statement| {
8153                                statement
8154                                    .query_map(rusqlite::params![id], |row| row.get::<_, String>(0))?
8155                                    .collect::<Result<HashSet<_>, _>>()
8156                            })
8157                            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
8158                            .and_then(|refused| {
8159                                let commits = self
8160                                    .outbox
8161                                    .iter()
8162                                    .filter(|commit| refused.contains(&commit.client_commit_id))
8163                                    .cloned()
8164                                    .collect::<Vec<_>>();
8165                                if !commits.is_empty() {
8166                                    self.drop_outbox_commits(&commits, &error.code, &error.message)?;
8167                                }
8168                                prepared.message.frames.retain(|frame| {
8169                                    !matches!(frame, Frame::PushCommit { client_commit_id, .. }
8170                                        if refused.contains(client_commit_id))
8171                                });
8172                                prepared.meta.pushed_ids.retain(|id| !refused.contains(id));
8173                                prepared.continuation = Some(crate::round::RoundPhase::UploadGrant);
8174                                Ok(())
8175                            })
8176                            .map_err(|message| TransportError::new("client.failed", message))
8177                    }
8178                    result => result,
8179                };
8180                let result = result.and_then(|()| {
8181                    self.conn
8182                        .execute(
8183                            "DELETE FROM _syncular_blob_uploads WHERE blob_id = ?",
8184                            rusqlite::params![id],
8185                        )
8186                        .map(|_| ())
8187                        .map_err(|error| {
8188                            TransportError::new(
8189                                "client.failed",
8190                                Self::sqlite_failure(&self.storage_failure, error),
8191                            )
8192                        })
8193                });
8194                match result {
8195                    Ok(()) => return crate::AppliedSyncRound::Continue(prepared),
8196                    Err(TransportError {
8197                        code,
8198                        message,
8199                        details,
8200                    }) => {
8201                        self.active_round = None;
8202                        if Self::unreachable_transport(&details) {
8203                            // The round's work is still owed; the host's
8204                            // connectivity resume runs it.
8205                            self.set_sync_needed(true, false);
8206                        } else if Self::retryable_failure_code(&code) {
8207                            self.schedule_background_retry();
8208                        }
8209                        let recommended_action = recommended_action_of(&details);
8210                        let outcome = self.finish_sync_round(
8211                            prepared.started_at_ms,
8212                            SyncOutcome::Failed {
8213                                error_code: code,
8214                                message,
8215                                details,
8216                                recommended_action,
8217                            },
8218                        );
8219                        return crate::AppliedSyncRound::Complete {
8220                            outcome,
8221                            controls: Vec::new(),
8222                            more: false,
8223                            bootstrap_advanced: false,
8224                        };
8225                    }
8226                }
8227            }
8228            crate::round::ExchangeResult::Reply {
8229                response,
8230                downloads,
8231                transport_failed,
8232            } => (response, downloads, transport_failed),
8233        };
8234        self.active_round = None;
8235        let before = self.resume_tokens();
8236        let outcome = (|| {
8237            let response = match response {
8238                Ok(response) => {
8239                    // §1.5: a complete, decodable response ends the report, an
8240                    // in-band ERROR included (the server already recorded it).
8241                    self.round_failure = None;
8242                    response
8243                }
8244                Err(TransportError {
8245                    code,
8246                    message,
8247                    details,
8248                }) => {
8249                    // §1.5: the server cannot see a round whose transport
8250                    // failed or whose response never decoded, so its code rides
8251                    // the following requests. A code the frame grammar rejects
8252                    // is not reportable, and the previous code stays; the
8253                    // client-local drive timeout never goes on the wire (§7.8).
8254                    if code != "sync.idle_timeout" && ssp2::model::valid_round_failure_code(&code) {
8255                        self.round_failure = Some(code.clone());
8256                    }
8257                    if transport_failed {
8258                        if prepared.realtime {
8259                            self.set_realtime_state(RealtimeState::Lost, Some(&code));
8260                        }
8261                        if let Err(message) = self.record_lease_error(&code) {
8262                            return SyncOutcome::Failed {
8263                                error_code: "storage.failed".to_owned(),
8264                                message,
8265                                details: None,
8266                                recommended_action: None,
8267                            };
8268                        }
8269                    }
8270                    if Self::retryable_failure_code(&code) && !Self::unreachable_transport(&details)
8271                    {
8272                        let delay = self.schedule_background_retry();
8273                        if prepared.realtime && transport_failed {
8274                            self.realtime_retry_delay_ms = Some(delay);
8275                        }
8276                    }
8277                    let recommended_action = recommended_action_of(&details);
8278                    return SyncOutcome::Failed {
8279                        error_code: code,
8280                        message,
8281                        details,
8282                        recommended_action,
8283                    };
8284                }
8285            };
8286            let mut outcome = self.process_response(&mut downloads, response, &prepared.meta);
8287            if let SyncOutcome::Ok(report) = &mut outcome {
8288                report.deferred_commits = prepared.meta.deferred_commits;
8289            }
8290            match &outcome {
8291                SyncOutcome::Ok(_) => self.reset_background_retry(),
8292                SyncOutcome::Failed {
8293                    error_code,
8294                    details,
8295                    ..
8296                } if Self::retryable_failure_code(error_code)
8297                    && !Self::unreachable_transport(details) =>
8298                {
8299                    self.schedule_background_retry();
8300                }
8301                _ => {}
8302            }
8303            if prepared.meta.deferred_commits > 0 {
8304                self.set_sync_needed(true, true);
8305            }
8306            self.previous_version_lifetime_check();
8307            outcome
8308        })();
8309        match &outcome {
8310            // §7.1: a unit still catching up needs another round, also when the
8311            // drive that started it ends early.
8312            SyncOutcome::Ok(report) if !report.bootstrapping.is_empty() => {
8313                self.set_sync_needed(true, false)
8314            }
8315            // The round's work is still owed; the host's connectivity resume
8316            // runs it.
8317            SyncOutcome::Failed { details, .. } if Self::unreachable_transport(details) => {
8318                self.set_sync_needed(true, false)
8319            }
8320            _ => {}
8321        }
8322        let more = self.transport_enabled
8323            && matches!(&outcome, SyncOutcome::Ok(report)
8324                if report.schema_floor.is_none()
8325                    && (!report.bootstrapping.is_empty()
8326                        || report.commits_applied > 0
8327                        || report.segment_rows_applied > 0
8328                        || !report.resets.is_empty()
8329                        || report.deferred_commits > 0
8330                        || self.sync_needed));
8331        let bootstrap_advanced = matches!(&outcome, SyncOutcome::Ok(report) if report.segment_rows_applied > 0 && !report.bootstrapping.is_empty())
8332            && before != self.resume_tokens();
8333        crate::AppliedSyncRound::Complete {
8334            outcome: self.finish_sync_round(prepared.started_at_ms, outcome),
8335            controls: if self.transport_enabled {
8336                downloads.controls
8337            } else {
8338                Vec::new()
8339            },
8340            more,
8341            bootstrap_advanced,
8342        }
8343    }
8344
8345    /// Every unit's resume token and applied-segment record: a bootstrap
8346    /// that advanced changes one (§4.7), also when it applied images of a
8347    /// section it deferred (§5.3, §5.6).
8348    #[allow(clippy::type_complexity)]
8349    fn resume_tokens(&self) -> Option<Vec<(String, Option<String>, Option<String>)>> {
8350        self.conn
8351            .prepare_cached(
8352                "SELECT id, bootstrap_state, applied FROM _syncular_units ORDER BY rowid",
8353            )
8354            .and_then(|mut stmt| {
8355                stmt.query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))?
8356                    .collect()
8357            })
8358            .ok()
8359    }
8360
8361    /// §7.8: one idle drive's default total deadline, the TS client's value.
8362    pub const SYNC_IDLE_TIMEOUT_MS: u64 = 60_000;
8363
8364    pub fn sync_until_idle(
8365        &mut self,
8366        transport: &mut dyn Transport,
8367        max_rounds: Option<u32>,
8368    ) -> SyncOutcome {
8369        self.sync_until_idle_with_deadline(transport, max_rounds, Some(Self::SYNC_IDLE_TIMEOUT_MS))
8370    }
8371
8372    /// As [`SyncClient::sync_until_idle`], with an explicit total deadline in
8373    /// milliseconds (`None` disables it). The deadline is checked against the
8374    /// client clock between rounds: a drive past it after a completed round
8375    /// ends as `BudgetExhausted`. Each round is bounded by `timeout_ms` on its
8376    /// own, or the transport's tighter whole-round deadline (§1.8); a round
8377    /// that reaches the drive bound fails with `sync.idle_timeout`.
8378    pub fn sync_until_idle_with_deadline(
8379        &mut self,
8380        transport: &mut dyn Transport,
8381        max_rounds: Option<u32>,
8382        timeout_ms: Option<u64>,
8383    ) -> SyncOutcome {
8384        // §7.8: an explicit budget is a strict positive round count. Zero is a
8385        // caller error, not an exhaustion.
8386        if max_rounds == Some(0) {
8387            return SyncOutcome::Failed {
8388                details: None,
8389                error_code: "sync.invalid_request".into(),
8390                message: "maxRounds must be a positive integer".into(),
8391                recommended_action: None,
8392            };
8393        }
8394        if timeout_ms.is_some_and(|ms| ms == 0 || i64::try_from(ms).is_err()) {
8395            return SyncOutcome::Failed {
8396                details: None,
8397                error_code: "sync.invalid_request".into(),
8398                message: "timeoutMs must be a positive integer".into(),
8399                recommended_action: None,
8400            };
8401        }
8402        let started_at_ms = self.clock_now_ms();
8403        let rounds = max_rounds.unwrap_or(20);
8404        let mut aggregate = SyncReport::default();
8405        let mut spent = 0;
8406        let mut completed = false;
8407        while spent < rounds {
8408            // §7.8: a deadline that elapsed across completed rounds ends the
8409            // drive like an exhausted round budget; the next drive resumes.
8410            if completed
8411                && timeout_ms.is_some_and(|ms| {
8412                    self.clock_now_ms() >= started_at_ms.saturating_add(ms as i64)
8413                })
8414            {
8415                return SyncOutcome::BudgetExhausted(aggregate);
8416            }
8417            let idle_bound = timeout_ms.map(|ms| (ms, spent));
8418            spent += 1;
8419            let before = self.resume_tokens();
8420            match self.sync_bounded(transport, idle_bound) {
8421                SyncOutcome::Failed {
8422                    error_code,
8423                    message,
8424                    details,
8425                    recommended_action,
8426                } => {
8427                    return SyncOutcome::Failed {
8428                        details,
8429                        error_code,
8430                        message,
8431                        recommended_action,
8432                    };
8433                }
8434                outcome @ SyncOutcome::RealtimeUnavailable { .. } => return outcome,
8435                SyncOutcome::BudgetExhausted(_) => {
8436                    unreachable!("sync() never exhausts a round budget")
8437                }
8438                SyncOutcome::Ok(report) => {
8439                    completed = true;
8440                    if max_rounds.is_none()
8441                        && report.segment_rows_applied > 0
8442                        && !report.bootstrapping.is_empty()
8443                        && before != self.resume_tokens()
8444                    {
8445                        spent = 0;
8446                    }
8447                    // TS parity: a disabled transport or a schema floor ends
8448                    // the loop. The latest round still merges first.
8449                    let stop = !self.transport_enabled() || report.schema_floor.is_some();
8450                    aggregate.merge(&report);
8451                    if stop {
8452                        return SyncOutcome::Ok(aggregate);
8453                    }
8454                    // §4.5: pull again whenever the response contained
8455                    // commits or segments; resets re-bootstrap; a pending
8456                    // resume token continues paging (§4.7); deferred outbox
8457                    // commits and a raised sync-needed signal push on the next
8458                    // round.
8459                    let more = !report.bootstrapping.is_empty()
8460                        || report.commits_applied > 0
8461                        || report.segment_rows_applied > 0
8462                        || !report.resets.is_empty()
8463                        || report.deferred_commits > 0
8464                        || self.sync_needed;
8465                    if !more {
8466                        return SyncOutcome::Ok(aggregate);
8467                    }
8468                }
8469            }
8470        }
8471        // §7.8: partial success. The aggregate report is retained and the run
8472        // is explicitly not idle.
8473        SyncOutcome::BudgetExhausted(aggregate)
8474    }
8475
8476    fn process_response(
8477        &mut self,
8478        transport: &mut dyn Transport,
8479        response: Message,
8480        meta: &RequestMeta,
8481    ) -> SyncOutcome {
8482        #[cfg(feature = "bench-internals")]
8483        let _phase = self.benchmark_phases.start(Phase::ResponseApply);
8484        let mut report = SyncReport {
8485            pushed: meta.pushed_ids.len() as u32,
8486            ..SyncReport::default()
8487        };
8488        let mut rejection_details_by_commit: HashMap<String, BTreeMap<i32, RejectionDetails>> =
8489            HashMap::new();
8490        let pushed_ids = meta
8491            .pushed_ids
8492            .iter()
8493            .map(String::as_str)
8494            .collect::<HashSet<_>>();
8495        let mut last_final_push_result_id: Option<String> = None;
8496        for frame in &response.frames {
8497            match frame {
8498                Frame::PushResultDetails {
8499                    client_commit_id,
8500                    entries,
8501                } => {
8502                    let details = rejection_details_by_commit
8503                        .entry(client_commit_id.clone())
8504                        .or_default();
8505                    for entry in entries {
8506                        let parsed = match RejectionDetails::parse(&entry.details.0) {
8507                            Ok(value) => value,
8508                            Err(message) => {
8509                                return SyncOutcome::Failed {
8510                                    details: None,
8511                                    error_code: "sync.invalid_request".to_owned(),
8512                                    message,
8513                                    recommended_action: None,
8514                                };
8515                            }
8516                        };
8517                        details.insert(entry.op_index, parsed);
8518                    }
8519                }
8520                Frame::PushResult {
8521                    client_commit_id,
8522                    status,
8523                    results,
8524                    ..
8525                } if pushed_ids.contains(client_commit_id.as_str())
8526                    && Self::push_result_is_final(*status, results) =>
8527                {
8528                    last_final_push_result_id = Some(client_commit_id.clone());
8529                }
8530                _ => {}
8531            }
8532        }
8533        let mut frames = response.frames.into_iter().peekable();
8534        match frames.next() {
8535            Some(Frame::RespHeader {
8536                required_schema_version,
8537                latest_schema_version,
8538                log_epoch,
8539                reset_required,
8540            }) => {
8541                if let Some(required) = required_schema_version {
8542                    // §1.6 schema-floor response: nothing else is processed;
8543                    // stop syncing and surface the upgrade requirement.
8544                    let floor = SchemaFloor {
8545                        required_schema_version: Some(required),
8546                        latest_schema_version,
8547                    };
8548                    if let Err(message) = self.set_schema_floor(Some(floor.clone())) {
8549                        return SyncOutcome::Failed {
8550                            details: None,
8551                            error_code: "storage.failed".to_owned(),
8552                            message,
8553                            recommended_action: None,
8554                        };
8555                    }
8556                    report.schema_floor = Some(floor);
8557                    return SyncOutcome::Ok(report);
8558                }
8559                if reset_required {
8560                    if frames.next().is_some() {
8561                        return SyncOutcome::Failed {
8562                            details: None,
8563                            error_code: "client.invalid_host_response".to_owned(),
8564                            message: "log-epoch reset response contains body frames".to_owned(),
8565                            recommended_action: None,
8566                        };
8567                    }
8568                    match self.run_log_epoch_reset(&log_epoch) {
8569                        Ok(resets) => {
8570                            report.resets = resets;
8571                            return SyncOutcome::Ok(report);
8572                        }
8573                        Err(message) => {
8574                            return SyncOutcome::Failed {
8575                                details: None,
8576                                error_code: "sync.local_corrupt".to_owned(),
8577                                message,
8578                                recommended_action: None,
8579                            };
8580                        }
8581                    }
8582                }
8583                let stored_epoch = match self.get_meta(LOG_EPOCH_KEY) {
8584                    Ok(stored) => stored,
8585                    Err(message) => {
8586                        return SyncOutcome::Failed {
8587                            details: None,
8588                            error_code: "storage.failed".to_owned(),
8589                            message,
8590                            recommended_action: None,
8591                        };
8592                    }
8593                };
8594                if stored_epoch.as_deref() != Some(log_epoch.as_str()) {
8595                    return SyncOutcome::Failed {
8596                        details: None,
8597                        error_code: "client.invalid_host_response".to_owned(),
8598                        message: "server changed logEpoch without requiring a reset".to_owned(),
8599                        recommended_action: None,
8600                    };
8601                }
8602            }
8603            _ => {
8604                return SyncOutcome::Failed {
8605                    details: None,
8606                    error_code: "sync.invalid_request".to_owned(),
8607                    message: "response does not start with RESP_HEADER".to_owned(),
8608                    recommended_action: None,
8609                };
8610            }
8611        }
8612
8613        if let Err(message) = self.begin_coalesced_apply() {
8614            return SyncOutcome::Failed {
8615                details: None,
8616                error_code: "storage.failed".into(),
8617                message,
8618                recommended_action: None,
8619            };
8620        }
8621        let mut failure: Option<(String, String, Option<String>, Option<Value>)> = None;
8622        // §4.5 client application: the units whose persisted echo a live
8623        // change must match, fixed at LIVE_START after every SUB_END persisted.
8624        let mut live: Option<(LiveStatus, String, Vec<UnitRecord>)> = None;
8625        let mut live_ended = false;
8626        while let Some(frame) = frames.next() {
8627            match frame {
8628                frame @ Frame::PushResult { .. } => {
8629                    let successful = matches!(
8630                        &frame,
8631                        Frame::PushResult {
8632                            status: PushStatus::Applied | PushStatus::Cached,
8633                            ..
8634                        }
8635                    );
8636                    let mut results = vec![frame];
8637                    if successful {
8638                        while matches!(
8639                            frames.peek(),
8640                            Some(Frame::PushResult {
8641                                status: PushStatus::Applied | PushStatus::Cached,
8642                                ..
8643                            })
8644                        ) {
8645                            if let Some(next) = frames.next() {
8646                                results.push(next);
8647                            }
8648                        }
8649                    }
8650                    if self
8651                        .handle_push_results(
8652                            &results,
8653                            &pushed_ids,
8654                            &rejection_details_by_commit,
8655                            &mut report,
8656                            last_final_push_result_id.as_deref(),
8657                        )
8658                        .is_err()
8659                    {
8660                        failure = Some((
8661                            "client.outcome_persistence_failed".to_owned(),
8662                            "local commit outcome could not be persisted".to_owned(),
8663                            None,
8664                            None,
8665                        ));
8666                        break;
8667                    }
8668                }
8669                Frame::PushResultDetails { .. } => {}
8670                Frame::SubStart {
8671                    id,
8672                    status,
8673                    reason_code,
8674                    bootstrap,
8675                    fenced,
8676                } => {
8677                    // §1.4 rule 5: an ERROR closes the section; its body still
8678                    // applied, its SUB_END values are never persisted.
8679                    let mut body = Vec::new();
8680                    let mut sub_end: Option<(i64, Option<String>)> = None;
8681                    let mut error: Option<Frame> = None;
8682                    for inner in frames.by_ref() {
8683                        match inner {
8684                            Frame::SubEnd {
8685                                next_cursor,
8686                                bootstrap_state,
8687                            } => {
8688                                sub_end = Some((next_cursor, bootstrap_state.map(|r| r.0)));
8689                                break;
8690                            }
8691                            inner @ Frame::Error { .. } => {
8692                                error = Some(inner);
8693                                break;
8694                            }
8695                            Frame::Unknown { .. } => {}
8696                            other => body.push(other),
8697                        }
8698                    }
8699                    let ended = sub_end.is_some();
8700                    let section = SectionStart {
8701                        id,
8702                        status,
8703                        reason_code,
8704                        bootstrap,
8705                        fenced,
8706                    };
8707                    match self.process_section(
8708                        transport,
8709                        &section,
8710                        body,
8711                        sub_end,
8712                        meta,
8713                        &mut report,
8714                    ) {
8715                        Err(SectionError::Abort(code, message)) => {
8716                            failure = Some((code, message, None, None));
8717                            break;
8718                        }
8719                        Err(SectionError::Transport(error)) => {
8720                            failure = Some((error.code, error.message, None, error.details));
8721                            break;
8722                        }
8723                        _ => {}
8724                    }
8725                    if ended {
8726                        // The counters stay: `complete` and `failed` report them.
8727                        self.progress.update(|p| {
8728                            p.unit_id = None;
8729                            p.table = None;
8730                            p.segment_id = None;
8731                        });
8732                    }
8733                    match error {
8734                        Some(Frame::Error {
8735                            code,
8736                            message,
8737                            recommended_action,
8738                            details,
8739                            ..
8740                        }) => {
8741                            failure = Some(Self::error_frame_failure(
8742                                code,
8743                                message,
8744                                recommended_action,
8745                                details,
8746                            ));
8747                            break;
8748                        }
8749                        Some(_) => unreachable!("only an ERROR closes a section early"),
8750                        None if !ended => {
8751                            failure = Some((
8752                                "sync.invalid_request".to_owned(),
8753                                "unit section without SUB_END".to_owned(),
8754                                None,
8755                                None,
8756                            ));
8757                            break;
8758                        }
8759                        None => {}
8760                    }
8761                }
8762                Frame::LiveStart {
8763                    status,
8764                    reason_code,
8765                } => {
8766                    let receivers = match self.stored(load_units(&self.conn)) {
8767                        Ok(units) => units
8768                            .into_iter()
8769                            .filter(|unit| {
8770                                unit.status == UnitStatus::Active
8771                                    && unit.echo.is_some()
8772                                    && !unit.bootstrapping()
8773                            })
8774                            .collect(),
8775                        Err(message) => {
8776                            failure = Some(("storage.failed".to_owned(), message, None, None));
8777                            break;
8778                        }
8779                    };
8780                    live = Some((status, reason_code, receivers));
8781                }
8782                Frame::Commit {
8783                    commit_seq,
8784                    tables,
8785                    changes,
8786                    ..
8787                } => {
8788                    let Some((LiveStatus::Active, _, receivers)) = &live else {
8789                        continue;
8790                    };
8791                    let receivers = receivers.clone();
8792                    match self.apply_live_commit(
8793                        &receivers,
8794                        &tables,
8795                        changes,
8796                        commit_seq,
8797                        AppliedServerCommitSource::Pull,
8798                    ) {
8799                        Ok(applied) => report.commits_applied += u32::from(applied),
8800                        Err(error) => {
8801                            failure = Some((error.code, error.message, None, error.details));
8802                            break;
8803                        }
8804                    }
8805                }
8806                Frame::LiveEnd {
8807                    through_seq,
8808                    echo_digest,
8809                } => {
8810                    let Some((status, reason_code, _)) = live.take() else {
8811                        continue;
8812                    };
8813                    live_ended = true;
8814                    live = Some((status, String::new(), Vec::new()));
8815                    if let Err(message) = self.finish_live_section(
8816                        status,
8817                        &reason_code,
8818                        through_seq,
8819                        echo_digest,
8820                        &meta.registration_digest,
8821                        &mut report,
8822                    ) {
8823                        failure = Some(("storage.failed".to_owned(), message, None, None));
8824                        break;
8825                    }
8826                }
8827                Frame::Lease {
8828                    lease_id,
8829                    expires_at_ms,
8830                } => {
8831                    // §7.3.5: persist the opaque lease; a fresh lease clears
8832                    // any prior lease error (the outage/revocation is over).
8833                    if let Err(message) = self.set_lease_state(Some(LeaseState {
8834                        lease_id: Some(lease_id),
8835                        expires_at_ms: Some(expires_at_ms),
8836                        error_code: None,
8837                    })) {
8838                        failure = Some(("storage.failed".to_owned(), message, None, None));
8839                        break;
8840                    }
8841                }
8842                Frame::Error {
8843                    code,
8844                    message,
8845                    recommended_action,
8846                    details,
8847                    ..
8848                } => {
8849                    // §1.6: the whole request failed; already-completed
8850                    // units keep their applied data and cursors.
8851                    failure = Some(Self::error_frame_failure(
8852                        code,
8853                        message,
8854                        recommended_action,
8855                        details,
8856                    ));
8857                    break;
8858                }
8859                Frame::Unknown { .. } => {}
8860                _ => {
8861                    failure = Some((
8862                        "sync.invalid_request".to_owned(),
8863                        "unexpected frame in response".to_owned(),
8864                        None,
8865                        None,
8866                    ));
8867                    break;
8868                }
8869            }
8870        }
8871
8872        let applied = match failure {
8873            Some(failure) => Err(failure),
8874            // Local apply boundaries already reconciled their affected overlay.
8875            // Refuse round success if any remaining dirty work fails to
8876            // reconcile. §4.8 E1: the push half may have drained commits that
8877            // pinned rows of a shrunk window unit, so retry deferred evictions.
8878            None => self
8879                .rebuild_overlay_if_dirty()
8880                .and_then(|()| self.drain_pending_evictions())
8881                .map_err(|message| ("storage.failed".to_owned(), message, None, None)),
8882        };
8883        // §1.4 rule 4: frames completed before a failure stay committed; a
8884        // failed COMMIT of the coalesced transaction fails the round.
8885        if let Err(message) = self.finish_coalesced_apply() {
8886            return SyncOutcome::Failed {
8887                details: None,
8888                error_code: "storage.failed".into(),
8889                message,
8890                recommended_action: None,
8891            };
8892        }
8893        if let Err((error_code, message, recommended_action, details)) = applied {
8894            return SyncOutcome::Failed {
8895                details,
8896                error_code,
8897                message,
8898                recommended_action,
8899            };
8900        }
8901        let catching_up = match self.stored(load_units(&self.conn)) {
8902            Ok(units) => units
8903                .into_iter()
8904                .filter(|unit| unit.status == UnitStatus::Active && unit.cursor.is_some())
8905                .map(|unit| unit.id)
8906                .collect::<Vec<_>>(),
8907            Err(message) => {
8908                return SyncOutcome::Failed {
8909                    details: None,
8910                    error_code: "storage.failed".into(),
8911                    message,
8912                    recommended_action: None,
8913                }
8914            }
8915        };
8916        // §7.1: a response without LIVE_END, one answered `unregistered` or
8917        // `reset`, or a unit still catching up leaves sync-needed raised;
8918        // after `unregistered` the next round starts at once.
8919        let status = live.map(|(status, _, _)| status);
8920        if !live_ended || status != Some(LiveStatus::Active) || !catching_up.is_empty() {
8921            self.set_sync_needed(true, status == Some(LiveStatus::Unregistered));
8922        }
8923        self.ack_after_pull(transport);
8924        // §7.4.5: the reset is over once the first post-reset pull round
8925        // leaves no unit catching up: the tables are rebuilt and current.
8926        if self.upgrading && catching_up.is_empty() {
8927            self.set_upgrading(false);
8928        }
8929        report.bootstrapping = catching_up;
8930        SyncOutcome::Ok(report)
8931    }
8932
8933    /// §1.6 / §10.1: the failure an in-band `ERROR` frame reports.
8934    fn error_frame_failure(
8935        code: String,
8936        message: String,
8937        recommended_action: String,
8938        details: Option<RawJson>,
8939    ) -> (String, String, Option<String>, Option<Value>) {
8940        let recommended_action = (!recommended_action.is_empty()).then_some(recommended_action);
8941        let details = details
8942            .and_then(|details| serde_json::from_str::<Value>(&details.0).ok())
8943            .filter(Value::is_object);
8944        (code, message, recommended_action, details)
8945    }
8946
8947    /// §4.5 at `LIVE_END` of a pull: join rule J for an `active` section, the
8948    /// §4.6 replica reset for `reset`, nothing for `unregistered`; then the
8949    /// replica cursor, the echo digest and the digest of the list the server
8950    /// now stores.
8951    fn finish_live_section(
8952        &mut self,
8953        status: LiveStatus,
8954        reason_code: &str,
8955        through_seq: i64,
8956        echo_digest: Option<[u8; 32]>,
8957        registration_digest: &str,
8958        report: &mut SyncReport,
8959    ) -> Result<(), String> {
8960        let rollback = self.capture();
8961        let resets = report.resets.len();
8962        self.begin_observation("syncular_live_end")?;
8963        let finished = (|| {
8964            let mut batch = ChangeAccumulator::default();
8965            let before = self.stored(replica_cursor(&self.conn))?;
8966            let mut cursor = if status == LiveStatus::Unregistered {
8967                before
8968            } else {
8969                -1
8970            };
8971            let joined = before.max(through_seq);
8972            for unit in self.stored(load_units(&self.conn))? {
8973                if unit.status != UnitStatus::Active || unit.bootstrapping() {
8974                    continue;
8975                }
8976                match status {
8977                    // §4.6 replica reset: every live unit bootstraps again;
8978                    // post-bootstrap units keep their cursors.
8979                    LiveStatus::Reset if unit.cursor.is_none() => {
8980                        for (base_key, value) in self.stored(window_refs(&self.conn, &unit.id))? {
8981                            batch.window(&base_key, &value);
8982                        }
8983                        report.resets.push(unit.id.clone());
8984                        self.stored(save_unit(
8985                            &self.conn,
8986                            &UnitRecord {
8987                                reason_code: Some(reason_code.to_owned()),
8988                                cursor: Some(-1),
8989                                seeded: Vec::new(),
8990                                ..unit
8991                            },
8992                        ))?;
8993                    }
8994                    // §4.5 join rule J.
8995                    LiveStatus::Active => {
8996                        let next = unit.cursor.unwrap_or(joined).max(through_seq);
8997                        if next >= joined {
8998                            cursor = joined;
8999                        }
9000                        if unit.cursor.is_some() {
9001                            self.stored(save_unit(
9002                                &self.conn,
9003                                &UnitRecord {
9004                                    cursor: (next < joined).then_some(next),
9005                                    ..unit
9006                                },
9007                            ))?;
9008                        }
9009                    }
9010                    _ => {}
9011                }
9012            }
9013            self.set_meta(REPLICA_CURSOR_KEY, &cursor.to_string())?;
9014            self.set_meta(
9015                ECHO_DIGEST_KEY,
9016                &echo_digest
9017                    .map(|digest| bytes_to_hex(&digest))
9018                    .unwrap_or_default(),
9019            )?;
9020            if status == LiveStatus::Unregistered {
9021                self.delete_meta(REGISTERED_DIGEST_KEY)?;
9022            } else {
9023                self.set_meta(REGISTERED_DIGEST_KEY, registration_digest)?;
9024            }
9025            self.finish_observation("syncular_live_end", batch)
9026        })();
9027        if finished.is_err() {
9028            self.rollback_observation("syncular_live_end");
9029            self.restore(rollback);
9030            report.resets.truncate(resets);
9031        }
9032        finished
9033    }
9034
9035    // -- push results (§6.3, §7.2) ------------------------------------------------
9036
9037    fn handle_push_results(
9038        &mut self,
9039        frames: &[Frame],
9040        pushed_ids: &HashSet<&str>,
9041        rejection_details_by_commit: &HashMap<String, BTreeMap<i32, RejectionDetails>>,
9042        report: &mut SyncReport,
9043        last_final_push_result_id: Option<&str>,
9044    ) -> Result<(), String> {
9045        self.begin_observation("syncular_push_results")?;
9046        let rollback = self.capture();
9047        let conflict_count = self.conflicts.len();
9048        let applied_count = report.applied.len();
9049        let rejected_count = report.rejected.len();
9050        let retryable_count = report.retryable.len();
9051        let mut drained = Vec::new();
9052        let mut batch = ChangeAccumulator::default();
9053        let persisted = (|| {
9054            for frame in frames {
9055                let Frame::PushResult {
9056                    client_commit_id,
9057                    status,
9058                    results,
9059                    commit_seq,
9060                } = frame
9061                else {
9062                    return Err("unexpected acknowledgement frame".to_owned());
9063                };
9064                if !pushed_ids.contains(client_commit_id.as_str()) {
9065                    continue;
9066                }
9067                let Some(index) = self
9068                    .outbox
9069                    .iter()
9070                    .position(|commit| commit.client_commit_id == *client_commit_id)
9071                else {
9072                    continue;
9073                };
9074                let status = *status;
9075                let rejection_details = rejection_details_by_commit.get(client_commit_id);
9076                let operations = self.outbox[index].ops.clone();
9077                match status {
9078                    PushStatus::Applied | PushStatus::Cached => {
9079                        // §7.2: a lost ack replays as `cached` — proceed as if the
9080                        // ack had arrived.
9081                        let sequence =
9082                            commit_seq.ok_or_else(|| "ACK lacks a commit sequence".to_owned())?;
9083                        let affected: HashSet<_> = operations
9084                            .iter()
9085                            .map(|op| (op.table.as_str(), op.row_id.as_str()))
9086                            .collect();
9087                        let prior = self.acknowledged_operations(Some(&affected), None)?;
9088                        for (op_index, operation) in operations.iter().enumerate() {
9089                            let delivered: bool = self.conn.query_row("SELECT EXISTS(SELECT 1 FROM _syncular_row_deliveries WHERE tbl=?1 AND id=?2 AND commit_seq>=?3)", rusqlite::params![operation.table, operation.row_id, sequence], |row| row.get(0))
9090                                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9091                            if delivered {
9092                                continue;
9093                            }
9094                            let table = self
9095                                .schema
9096                                .table(&operation.table)
9097                                .ok_or_else(|| "sync.unknown_table".to_owned())?;
9098                            let mut intended: Map<String, Value> = self
9099                                .stored_row(table, &operation.row_id, true)?
9100                                .map(|(row, _)| {
9101                                    table
9102                                        .columns
9103                                        .iter()
9104                                        .zip(&row)
9105                                        .map(|(column, value)| {
9106                                            (column.name.clone(), column_value_to_json(value))
9107                                        })
9108                                        .collect()
9109                                })
9110                                .unwrap_or_default();
9111                            let mut scope_image = intended.clone();
9112                            for op in prior
9113                                .iter()
9114                                .chain(
9115                                    self.failed_commits
9116                                        .iter()
9117                                        .flat_map(|(commit, _)| &commit.ops),
9118                                )
9119                                .chain(
9120                                    self.outbox
9121                                        .iter()
9122                                        .take(index)
9123                                        .flat_map(|commit| &commit.ops),
9124                                )
9125                                .chain(operations.iter().take(op_index + 1))
9126                            {
9127                                if op.table != operation.table || op.row_id != operation.row_id {
9128                                    continue;
9129                                }
9130                                if let Some(values) = &op.values {
9131                                    intended.extend(values.clone());
9132                                } else {
9133                                    scope_image = intended.clone();
9134                                    intended.clear();
9135                                }
9136                            }
9137                            self.conn.execute("INSERT INTO _syncular_acked_rows(commit_id,idx,tbl,id,commit_seq,op_json,intent_json) VALUES (?1,?2,?3,?4,?5,?6,?7)",
9138                                rusqlite::params![client_commit_id, op_index as i64, operation.table, operation.row_id, sequence,
9139                                    serde_json::to_string(&CommitOperation::from(operation)).map_err(|_| "sync.local_corrupt".to_owned())?,
9140                                    serde_json::to_string(if operation.upsert { &intended } else { &scope_image }).map_err(|_| "sync.local_corrupt".to_owned())?])
9141                                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9142                        }
9143                        let journal_results = results
9144                            .iter()
9145                            .map(|result| {
9146                                let op_index = match result {
9147                                    OpResult::Applied { op_index }
9148                                    | OpResult::Conflict { op_index, .. }
9149                                    | OpResult::Error { op_index, .. } => *op_index,
9150                                };
9151                                CommitOperationOutcome::Applied { op_index }
9152                            })
9153                            .collect::<Vec<_>>();
9154                        let outcome_status = if status == PushStatus::Applied {
9155                            CommitOutcomeStatus::Applied
9156                        } else {
9157                            CommitOutcomeStatus::Cached
9158                        };
9159                        let persisted = self
9160                            .persist_commit_outcome(
9161                                client_commit_id,
9162                                outcome_status,
9163                                &journal_results,
9164                                None,
9165                            )
9166                            .and_then(|()| self.delete_outbox_persisted(client_commit_id));
9167                        persisted?;
9168                        self.sync_needed = true;
9169                        batch.status = true;
9170                        batch.outcomes = true;
9171                    }
9172                    PushStatus::Rejected => {
9173                        if results.iter().any(|result| {
9174                            matches!(
9175                                result,
9176                                OpResult::Error {
9177                                    code,
9178                                    retryable: true,
9179                                    ..
9180                                } if code == "sync.idempotency_cache_miss"
9181                            )
9182                        }) {
9183                            // §6.3/§7.2: a serving failure, not an outcome — keep the
9184                            // exact commit queued for an identical retry.
9185                            report.retryable.push(client_commit_id.to_owned());
9186                            continue;
9187                        }
9188
9189                        let mut journal_results = Vec::with_capacity(results.len());
9190                        let mut conflicts = Vec::new();
9191                        let mut rejections = Vec::new();
9192                        for result in results {
9193                            match result {
9194                                OpResult::Applied { op_index } => {
9195                                    journal_results.push(CommitOperationOutcome::Applied {
9196                                        op_index: *op_index,
9197                                    });
9198                                }
9199                                OpResult::Conflict {
9200                                    op_index,
9201                                    code,
9202                                    message,
9203                                    server_version,
9204                                    server_row,
9205                                    conflict_columns: conflict_columns_bitmap,
9206                                } => {
9207                                    let operation = operations
9208                                        .get(*op_index as usize)
9209                                        .map(CommitOperation::from);
9210                                    let (table, row_id) = operation
9211                                        .as_ref()
9212                                        .map(|op| (op.table.clone(), op.row_id.clone()))
9213                                        .unwrap_or_default();
9214                                    let server_row_json = self
9215                                        .schema
9216                                        .table(&table)
9217                                        .and_then(|t| {
9218                                            decode_row_bytes(t, server_row, &self.encryption)
9219                                                .ok()
9220                                                .map(|(row, _)| (t, row))
9221                                        })
9222                                        .map(|(t, row)| {
9223                                            let mut map = Map::new();
9224                                            for (i, column) in t.columns.iter().enumerate() {
9225                                                map.insert(
9226                                                    column.name.clone(),
9227                                                    column_value_to_json(
9228                                                        row.get(i).unwrap_or(&None),
9229                                                    ),
9230                                                );
9231                                            }
9232                                            map
9233                                        })
9234                                        .unwrap_or_default();
9235                                    // §6.3: decode the presence-layout bitmap
9236                                    // into the marked column names.
9237                                    let conflict_columns = self
9238                                        .schema
9239                                        .table(&table)
9240                                        .map(|t| {
9241                                            t.columns
9242                                                .iter()
9243                                                .enumerate()
9244                                                .filter(|(index, _)| {
9245                                                    conflict_columns_bitmap
9246                                                        .get(index / 8)
9247                                                        .is_some_and(|byte| {
9248                                                            byte & (1 << (index % 8)) != 0
9249                                                        })
9250                                                })
9251                                                .map(|(_, column)| column.name.clone())
9252                                                .collect::<Vec<_>>()
9253                                        })
9254                                        .unwrap_or_default();
9255                                    let conflict = ConflictRecord {
9256                                        client_commit_id: client_commit_id.to_owned(),
9257                                        op_index: *op_index,
9258                                        table,
9259                                        row_id,
9260                                        code: code.clone(),
9261                                        message: message.clone(),
9262                                        server_version: *server_version,
9263                                        server_row: server_row_json,
9264                                        conflict_columns,
9265                                        operation,
9266                                    };
9267                                    journal_results.push(CommitOperationOutcome::Conflict {
9268                                        conflict: conflict.clone(),
9269                                    });
9270                                    conflicts.push(conflict);
9271                                }
9272                                OpResult::Error {
9273                                    op_index,
9274                                    code,
9275                                    message,
9276                                    retryable,
9277                                } => {
9278                                    let rejection = RejectionRecord {
9279                                        client_commit_id: client_commit_id.to_owned(),
9280                                        op_index: *op_index,
9281                                        code: code.clone(),
9282                                        message: message.clone(),
9283                                        retryable: *retryable,
9284                                        details: rejection_details
9285                                            .and_then(|details| details.get(op_index))
9286                                            .cloned(),
9287                                        operation: operations
9288                                            .get(*op_index as usize)
9289                                            .map(CommitOperation::from),
9290                                    };
9291                                    journal_results.push(CommitOperationOutcome::Error {
9292                                        rejection: rejection.clone(),
9293                                    });
9294                                    rejections.push(rejection);
9295                                }
9296                            }
9297                        }
9298                        if self.retain_failed_commits {
9299                            self.retain_failed_commit(client_commit_id, &operations)?;
9300                        }
9301                        let outcome_status = if conflicts.is_empty() {
9302                            CommitOutcomeStatus::Rejected
9303                        } else {
9304                            CommitOutcomeStatus::Conflict
9305                        };
9306                        let persisted = self
9307                            .persist_commit_outcome(
9308                                client_commit_id,
9309                                outcome_status,
9310                                &journal_results,
9311                                Some(&operations),
9312                            )
9313                            .and_then(|()| self.delete_outbox_persisted(client_commit_id));
9314                        persisted?;
9315                        batch.conflicts = !conflicts.is_empty();
9316                        batch.rejections = !rejections.is_empty();
9317                        batch.status = true;
9318                        batch.outcomes = true;
9319                        self.conflicts.extend(conflicts);
9320                        self.rejections.extend(rejections);
9321                    }
9322                }
9323                drained.push(self.outbox.remove(index));
9324                if batch.status {
9325                    for operation in &operations {
9326                        if !self.record_row_scopes(
9327                            &mut batch,
9328                            &operation.table,
9329                            &operation.row_id,
9330                            false,
9331                        ) {
9332                            batch.table(&operation.table);
9333                        }
9334                    }
9335                }
9336                match status {
9337                    PushStatus::Applied | PushStatus::Cached => {
9338                        report.applied.push(client_commit_id.to_owned())
9339                    }
9340                    PushStatus::Rejected => report.rejected.push(client_commit_id.to_owned()),
9341                }
9342            }
9343            if batch.outcomes && frames.iter().any(|frame| matches!(frame, Frame::PushResult { client_commit_id, .. } if Some(client_commit_id.as_str()) == last_final_push_result_id)) {
9344                self.prune_commit_outcomes()?;
9345            }
9346            self.reconcile_rows(&drained)?;
9347            self.rebuild_overlay_if_dirty()?;
9348            self.finish_observation("syncular_push_results", batch)
9349        })();
9350        if let Err(error) = persisted {
9351            self.rollback_observation("syncular_push_results");
9352            self.restore(rollback);
9353            report.applied.truncate(applied_count);
9354            report.rejected.truncate(rejected_count);
9355            report.retryable.truncate(retryable_count);
9356            return Err(error);
9357        }
9358        if !rollback.sync_needed && self.sync_needed {
9359            self.sync_intent_queue.push_back(SyncIntent::Interactive);
9360        }
9361        report.conflicts += (self.conflicts.len() - conflict_count) as u32;
9362        Ok(())
9363    }
9364
9365    fn push_result_is_final(status: PushStatus, results: &[OpResult]) -> bool {
9366        status != PushStatus::Rejected
9367            || !results.iter().any(|result| {
9368                matches!(
9369                    result,
9370                    OpResult::Error {
9371                        code,
9372                        retryable: true,
9373                        ..
9374                    } if code == "sync.idempotency_cache_miss"
9375                )
9376            })
9377    }
9378
9379    // -- unit sections and the live section ------------------------------------------
9380
9381    /// One unit section (§4.4): apply its segments, then persist its
9382    /// `SUB_END` (§4.6 reset, §3.3 revocation and purge, the echo, cursor and
9383    /// resume token). A section of a unit the app released while the round was
9384    /// in flight is skipped; a fail-closed first page fails the unit and skips
9385    /// the rest of its section (§5.6).
9386    fn process_section(
9387        &mut self,
9388        transport: &mut dyn Transport,
9389        start: &SectionStart,
9390        body: Vec<Frame>,
9391        sub_end: Option<(i64, Option<String>)>,
9392        meta: &RequestMeta,
9393        report: &mut SyncReport,
9394    ) -> Result<(), SectionError> {
9395        let storage = |message| SectionError::Abort("storage.failed".to_owned(), message);
9396        let unit = self
9397            .stored(get_unit(&self.conn, &start.id))
9398            .map_err(storage)?
9399            .filter(|unit| unit.status == UnitStatus::Active);
9400        // §7.7: a section boundary ends the previous payload.
9401        self.progress.update(|p| {
9402            p.phase = ProgressPhase::Import;
9403            p.unit_id = unit.as_ref().map(|unit| unit.id.clone());
9404            p.table = None;
9405            p.segment_id = None;
9406            p.bytes_received = 0;
9407            p.bytes_total = None;
9408            p.rows_processed = 0;
9409            p.rows_total = None;
9410            p.import_write_ms = None;
9411            p.import_commit_ms = None;
9412        });
9413        let Some(unit) = unit else {
9414            return Ok(());
9415        };
9416        let echo: Vec<EchoRecord> = start
9417            .fenced
9418            .iter()
9419            .map(|record| EchoRecord {
9420                table: record.table.clone(),
9421                status: record.status.name().to_owned(),
9422                scopes: record.scopes.iter().cloned().collect(),
9423            })
9424            .collect();
9425        // §5.6: rows earlier attempts at this position applied.
9426        let applied_record = self
9427            .stored(applied_rows(&self.conn, &unit.id))
9428            .map_err(storage)?;
9429        // §5.6: a resumed bootstrap keeps the tables it already seeded.
9430        let mut seeded: BTreeSet<String> = if unit.bootstrap_state.is_none() {
9431            BTreeSet::new()
9432        } else {
9433            unit.seeded.iter().cloned().collect()
9434        };
9435        let applied = (|| {
9436            for frame in body {
9437                match frame {
9438                    Frame::SegmentInline { payload } => {
9439                        let segment = decode_rows_segment(&payload).map_err(|e| {
9440                            SectionError::Abort(e.code.as_str().to_owned(), e.detail)
9441                        })?;
9442                        self.progress.update(|p| {
9443                            p.phase = ProgressPhase::Import;
9444                            p.unit_id = Some(unit.id.clone());
9445                            p.table = Some(segment.table.clone());
9446                            p.segment_id = None;
9447                            p.bytes_received = payload.len() as u64;
9448                            p.bytes_total = Some(payload.len() as u64);
9449                            p.rows_processed = 0;
9450                            p.rows_total =
9451                                Some(segment.blocks.iter().map(|block| block.len() as u64).sum());
9452                            p.import_write_ms = None;
9453                            p.import_commit_ms = None;
9454                        });
9455                        let (pair, clear, effective) =
9456                            self.segment_pair(&unit, &echo, &seeded, &segment.table)?;
9457                        report.segment_rows_applied +=
9458                            self.apply_segment(&pair, &segment, clear, &effective, &unit.id)?;
9459                        seeded.insert(segment.table);
9460                    }
9461                    Frame::SegmentRef {
9462                        segment_id,
9463                        byte_length,
9464                        media_type,
9465                        table,
9466                        row_count,
9467                        as_of_commit_seq,
9468                        scope_digest,
9469                        row_cursor,
9470                        next_row_cursor,
9471                        url,
9472                        url_expires_at_ms,
9473                        ..
9474                    } => {
9475                        // §4.2: reject a descriptor whose mediaType was not
9476                        // advertised — never skip or guess.
9477                        let advertised = match media_type {
9478                            MediaType::Rows => {
9479                                meta.accept & ACCEPT_EXTERNAL_ROWS != 0
9480                                    || meta.accept & ACCEPT_INLINE_ROWS != 0
9481                            }
9482                            MediaType::Sqlite => meta.accept & ACCEPT_SQLITE != 0,
9483                        };
9484                        if !advertised {
9485                            return Err(SectionError::Abort(
9486                                "sync.invalid_request".to_owned(),
9487                                format!(
9488                                    "SEGMENT_REF mediaType {} was not advertised in accept (§4.2)",
9489                                    media_type.name()
9490                                ),
9491                            ));
9492                        }
9493                        // §5.6 applied-segment rule: a partly applied image
9494                        // already ran its table's first-page clear.
9495                        let resumed = applied_record.get(&segment_id).copied();
9496                        if resumed.is_some() {
9497                            seeded.insert(table.clone());
9498                        }
9499                        if resumed == Some(row_count) {
9500                            continue;
9501                        }
9502                        if meta
9503                            .deferred
9504                            .contains(&(unit.id.clone(), segment_id.clone()))
9505                        {
9506                            return Err(SectionError::Deferred);
9507                        }
9508                        self.progress.update(|p| {
9509                            p.phase = ProgressPhase::Download;
9510                            p.unit_id = Some(unit.id.clone());
9511                            p.table = Some(table.clone());
9512                            p.segment_id = Some(segment_id.clone());
9513                            p.bytes_received = 0;
9514                            p.bytes_total = u64::try_from(byte_length).ok();
9515                            p.rows_processed = 0;
9516                            p.rows_total = u64::try_from(row_count).ok();
9517                            p.import_write_ms = None;
9518                            p.import_commit_ms = None;
9519                        });
9520                        let observer = self.progress.clone();
9521                        let mut on_progress = |bytes| observer.update(|p| p.bytes_received = bytes);
9522                        let bytes = if let Some(url) = url {
9523                            // §5.4: a url-carrying descriptor MUST be fetched
9524                            // from that URL; failure invalidates the whole
9525                            // descriptor (no fall-through to §5.5 — re-pull
9526                            // recovers, §1.4 rule 5).
9527                            if meta.accept & ACCEPT_SIGNED_URLS == 0 {
9528                                return Err(SectionError::Abort(
9529                                    "sync.invalid_request".to_owned(),
9530                                    "SEGMENT_REF carries a url but accept bit 3 was not advertised (§5.4)"
9531                                        .to_owned(),
9532                                ));
9533                            }
9534                            // §5.4: MUST NOT start a fetch at/past expiry.
9535                            if url_expires_at_ms.is_some_and(|exp| {
9536                                exp <= transport
9537                                    .segment_fetch_started_at(&url)
9538                                    .unwrap_or_else(|| self.clock_now_ms())
9539                            }) {
9540                                return Err(SectionError::Abort(
9541                                    "sync.segment_expired".to_owned(),
9542                                    format!(
9543                                        "signed URL for segment {segment_id} expired before fetch — re-pull mints fresh descriptors (§5.4)"
9544                                    ),
9545                                ));
9546                            }
9547                            transport
9548                                .fetch_url(
9549                                    &url,
9550                                    u64::try_from(byte_length).unwrap_or(0),
9551                                    &mut on_progress,
9552                                )
9553                                .map_err(|e| self.segment_fetch_failed(e))?
9554                        } else {
9555                            // §5.5: the owning unit's scope map, each value a one-element list.
9556                            let requested_scopes_json =
9557                                canonical_scope_json(&unit_scope_lists(&unit.scopes));
9558                            transport
9559                                .download_segment(
9560                                    &SegmentRequest {
9561                                        segment_id: segment_id.clone(),
9562                                        table: table.clone(),
9563                                        requested_scopes_json,
9564                                        byte_length: u64::try_from(byte_length).unwrap_or(0),
9565                                    },
9566                                    &mut on_progress,
9567                                )
9568                                .map_err(|e| {
9569                                    // §5.5: access changed since the response;
9570                                    // the next round's SUB_START settles the unit.
9571                                    if e.code == "sync.forbidden" {
9572                                        SectionError::Refused
9573                                    } else {
9574                                        self.segment_fetch_failed(e)
9575                                    }
9576                                })?
9577                        };
9578                        // §5.4: the descriptor's byteLength caps the body before
9579                        // §5.1 verifies the content address; the client discards
9580                        // the segment and re-pulls on either failure.
9581                        if bytes.len() as i64 > byte_length {
9582                            self.schedule_background_retry();
9583                            return Err(SectionError::Abort(
9584                                "sync.invalid_request".to_owned(),
9585                                "segment exceeds its byteLength".to_owned(),
9586                            ));
9587                        }
9588                        on_progress(bytes.len() as u64);
9589                        let digest = Sha256::digest(&bytes);
9590                        let expected = segment_id
9591                            .strip_prefix("sha256:")
9592                            .unwrap_or(segment_id.as_str());
9593                        if bytes_to_hex(&digest) != expected {
9594                            // §5.1: the client discards the segment and re-pulls.
9595                            self.schedule_background_retry();
9596                            return Err(SectionError::Abort(
9597                                "sync.invalid_request".to_owned(),
9598                                "segment bytes do not match the content address (§5.1)".to_owned(),
9599                            ));
9600                        }
9601                        self.progress.update(|p| p.phase = ProgressPhase::Import);
9602                        if media_type == MediaType::Sqlite {
9603                            // §5.3: a range image keeps its position in the resume
9604                            // token; a descriptor row cursor is invalid.
9605                            if row_cursor.is_some() || next_row_cursor.is_some() {
9606                                return Err(SectionError::Abort(
9607                                    "sync.invalid_request".to_owned(),
9608                                    "sqlite segments are whole-table: rowCursor/nextRowCursor must be absent (§5.3)"
9609                                        .to_owned(),
9610                                ));
9611                            }
9612                            let (pair, clear, effective) =
9613                                self.segment_pair(&unit, &echo, &seeded, &table)?;
9614                            report.segment_rows_applied += self.apply_sqlite_segment(
9615                                &pair,
9616                                &bytes,
9617                                clear,
9618                                &effective,
9619                                &unit.id,
9620                                &segment_id,
9621                                resumed.unwrap_or(0),
9622                                row_count,
9623                                as_of_commit_seq,
9624                                &scope_digest,
9625                            )?;
9626                            seeded.insert(table);
9627                        } else {
9628                            let segment = decode_rows_segment(&bytes).map_err(|e| {
9629                                SectionError::Abort(e.code.as_str().to_owned(), e.detail)
9630                            })?;
9631                            let (pair, clear, effective) =
9632                                self.segment_pair(&unit, &echo, &seeded, &segment.table)?;
9633                            report.segment_rows_applied +=
9634                                self.apply_segment(&pair, &segment, clear, &effective, &unit.id)?;
9635                            seeded.insert(segment.table);
9636                            // An image records its rows with each chunk.
9637                            self.stored(record_applied_rows(
9638                                &self.conn,
9639                                &unit.id,
9640                                &segment_id,
9641                                row_count,
9642                            ))
9643                            .map_err(storage)?;
9644                        }
9645                    }
9646                    _ => {
9647                        return Err(SectionError::Abort(
9648                            "sync.invalid_request".to_owned(),
9649                            "unexpected frame inside a unit section".to_owned(),
9650                        ));
9651                    }
9652                }
9653            }
9654            Ok(())
9655        })();
9656        match applied {
9657            Ok(()) => {}
9658            // The unit keeps catching up with its cursor unpersisted.
9659            Err(SectionError::Refused | SectionError::Deferred) => return Ok(()),
9660            Err(SectionError::FailClosed) => {
9661                // §5.6: the failure is unit-local; the rest of the response
9662                // still applies and the failed unit leaves the list.
9663                self.begin_observation("syncular_section_failure")
9664                    .map_err(storage)?;
9665                let failed = (|| {
9666                    let mut batch = ChangeAccumulator::default();
9667                    for (base_key, value) in self.stored(window_refs(&self.conn, &unit.id))? {
9668                        batch.window(&base_key, &value);
9669                    }
9670                    batch.status = true;
9671                    self.stored(save_unit(
9672                        &self.conn,
9673                        &UnitRecord {
9674                            status: UnitStatus::Failed,
9675                            reason_code: Some("sync.scope_revoked".to_owned()),
9676                            ..unit.clone()
9677                        },
9678                    ))?;
9679                    self.finish_observation("syncular_section_failure", batch)
9680                })();
9681                if let Err(message) = failed {
9682                    self.rollback_observation("syncular_section_failure");
9683                    return Err(storage(message));
9684                }
9685                report.failed.push(unit.id);
9686                return Ok(());
9687            }
9688            Err(error) => return Err(error),
9689        }
9690        let Some((next_cursor, bootstrap_state)) = sub_end else {
9691            return Ok(());
9692        };
9693        let rollback = self.capture();
9694        let counts = (
9695            report.resets.len(),
9696            report.revoked.len(),
9697            report.failed.len(),
9698        );
9699        self.begin_observation("syncular_sub_end")
9700            .map_err(storage)?;
9701        let finished = self
9702            .finish_section(start, echo, next_cursor, bootstrap_state, seeded, report)
9703            .and_then(|batch| {
9704                self.rebuild_overlay_if_dirty()?;
9705                self.finish_observation("syncular_sub_end", batch)
9706            });
9707        if let Err(message) = finished {
9708            self.rollback_observation("syncular_sub_end");
9709            self.restore(rollback);
9710            report.resets.truncate(counts.0);
9711            report.revoked.truncate(counts.1);
9712            report.failed.truncate(counts.2);
9713            return Err(storage(message));
9714        }
9715        Ok(())
9716    }
9717
9718    /// §5.6: the table, first-page flag and effective scopes of one segment
9719    /// of `unit`'s section.
9720    #[allow(clippy::type_complexity)]
9721    fn segment_pair(
9722        &self,
9723        unit: &UnitRecord,
9724        echo: &[EchoRecord],
9725        seeded: &BTreeSet<String>,
9726        name: &str,
9727    ) -> Result<(TableSchema, bool, Vec<(String, Vec<String>)>), SectionError> {
9728        let table = self.schema.table(name).cloned().ok_or_else(|| {
9729            SectionError::Abort(
9730                "sync.schema_mismatch".to_owned(),
9731                format!("segment table {name:?} is not in the client schema"),
9732            )
9733        })?;
9734        let effective = covers(&table, &unit.scopes)
9735            .then(|| pair_scopes(&unit.scopes, name, Some(echo)))
9736            .flatten()
9737            .ok_or_else(|| {
9738                SectionError::Abort(
9739                    "sync.invalid_request".to_owned(),
9740                    "a segment names no active pair of its unit".to_owned(),
9741                )
9742            })?;
9743        Ok((table, !seeded.contains(name), effective))
9744    }
9745
9746    /// §4.4 `SUB_END` of an active, revoked or reset section, inside the
9747    /// caller's observation: a reset discards cursor, resume token and
9748    /// first-page record and keeps the echo; a revoked pair purges by its last
9749    /// echo (§3.3); otherwise the echo replaces the persisted one, a revoked
9750    /// fenced record that turns active again re-bootstraps the unit, and a
9751    /// bootstrap section persists its pin, resume token and seeded tables.
9752    fn finish_section(
9753        &mut self,
9754        start: &SectionStart,
9755        echo: Vec<EchoRecord>,
9756        next_cursor: i64,
9757        bootstrap_state: Option<String>,
9758        seeded: BTreeSet<String>,
9759        report: &mut SyncReport,
9760    ) -> Result<ChangeAccumulator, String> {
9761        #[cfg(feature = "bench-internals")]
9762        let _phase = self.benchmark_phases.start(Phase::CursorPersist);
9763        let mut batch = ChangeAccumulator::default();
9764        // §4.4: SUB_END writes only into a unit that is still registered.
9765        let Some(unit) = self
9766            .stored(get_unit(&self.conn, &start.id))?
9767            .filter(|unit| unit.status == UnitStatus::Active)
9768        else {
9769            return Ok(batch);
9770        };
9771        for (base_key, value) in self.stored(window_refs(&self.conn, &unit.id))? {
9772            batch.window(&base_key, &value);
9773        }
9774        if start.status == SubStatus::Reset {
9775            // §4.6: staleness, never a purge; the rows stay until the
9776            // bootstrap replaces them.
9777            report.resets.push(unit.id.clone());
9778            self.stored(save_unit(
9779                &self.conn,
9780                &UnitRecord {
9781                    reason_code: Some(start.reason_code.clone()),
9782                    cursor: Some(-1),
9783                    bootstrap_state: None,
9784                    seeded: Vec::new(),
9785                    ..unit
9786                },
9787            ))?;
9788            return Ok(batch);
9789        }
9790        let revoked = start.status == SubStatus::Revoked;
9791        let mut units = None;
9792        let mut failed = false;
9793        let mut rearmed = false;
9794        for table in self.schema.tables.clone() {
9795            if !covers(&table, &unit.scopes) {
9796                continue;
9797            }
9798            let last = unit
9799                .echo
9800                .as_deref()
9801                .and_then(|last| pair_scopes(&unit.scopes, &table.name, Some(last)));
9802            let next = (!revoked)
9803                .then(|| pair_scopes(&unit.scopes, &table.name, Some(&echo)))
9804                .flatten();
9805            // §3.3: a revoked fenced record that turns active again re-bootstraps.
9806            rearmed |= unit.echo.is_some() && last.is_none() && next.is_some();
9807            // §3.3: purge a pair the response revokes, keyed on the echo the
9808            // client persisted while the pair was active.
9809            if let (Some(last), None) = (&last, &next) {
9810                let units = match units {
9811                    Some(ref units) => units,
9812                    None => units.insert(self.stored(load_units(&self.conn))?),
9813                };
9814                let owners = unit_owners(units, &table, &unit.id);
9815                failed |= !self.purge_pair(&mut batch, &table, last, &owners)?;
9816            }
9817        }
9818        if revoked || failed {
9819            if revoked {
9820                report.revoked.push(unit.id.clone());
9821            }
9822            if failed {
9823                report.failed.push(unit.id.clone());
9824            }
9825            batch.status = true;
9826            self.stored(save_unit(
9827                &self.conn,
9828                &UnitRecord {
9829                    status: if failed {
9830                        UnitStatus::Failed
9831                    } else {
9832                        UnitStatus::Revoked
9833                    },
9834                    reason_code: Some(if failed {
9835                        "sync.scope_revoked".to_owned()
9836                    } else {
9837                        start.reason_code.clone()
9838                    }),
9839                    cursor: Some(-1),
9840                    bootstrap_state: None,
9841                    seeded: Vec::new(),
9842                    echo: None,
9843                    ..unit
9844                },
9845            ))?;
9846            return Ok(batch);
9847        }
9848        let completed = start.bootstrap && !rearmed && bootstrap_state.is_none();
9849        let (cursor, bootstrap_state, seeded) = if rearmed {
9850            (Some(-1), None, Vec::new())
9851        } else if start.bootstrap {
9852            let seeded = if bootstrap_state.is_none() {
9853                Vec::new()
9854            } else {
9855                seeded.into_iter().collect()
9856            };
9857            (Some(next_cursor), bootstrap_state, seeded)
9858        } else {
9859            (
9860                unit.cursor,
9861                unit.bootstrap_state.clone(),
9862                unit.seeded.clone(),
9863            )
9864        };
9865        self.stored(save_unit(
9866            &self.conn,
9867            &UnitRecord {
9868                reason_code: None,
9869                echo: Some(echo.clone()),
9870                cursor,
9871                bootstrap_state,
9872                seeded,
9873                ..unit.clone()
9874            },
9875        ))?;
9876        if !completed {
9877            return Ok(batch);
9878        }
9879        // §7.1: a completed bootstrap covers acknowledged intent of its pairs
9880        // at or below its pin.
9881        for table in self.schema.tables.clone() {
9882            let Some(effective) = covers(&table, &unit.scopes)
9883                .then(|| pair_scopes(&unit.scopes, &table.name, Some(&echo)))
9884                .flatten()
9885            else {
9886                continue;
9887            };
9888            let mut statement = self
9889                .conn
9890                .prepare("SELECT id,intent_json FROM _syncular_acked_rows WHERE tbl=?1 AND commit_seq<=?2")
9891                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9892            let rows = statement
9893                .query_map(rusqlite::params![table.name, next_cursor], |row| {
9894                    Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
9895                })
9896                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9897            let mut covered = HashSet::new();
9898            for row in rows {
9899                let (id, intent) =
9900                    row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9901                let values: Map<String, Value> =
9902                    serde_json::from_str(&intent).map_err(|_| "sync.local_corrupt".to_owned())?;
9903                if owns_local_row(&table, &effective, &[], &values) {
9904                    covered.insert(id);
9905                }
9906            }
9907            drop(statement);
9908            for id in covered {
9909                self.conn
9910                    .execute(
9911                        "DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2 AND commit_seq<=?3",
9912                        rusqlite::params![table.name, id, next_cursor],
9913                    )
9914                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9915                self.overlay_dirty.table(&table.name);
9916                batch.table(&table.name);
9917            }
9918        }
9919        Ok(batch)
9920    }
9921
9922    /// §3.3: purge one revoked pair inside the caller's observation: delete
9923    /// the rows matching its last effective scopes that no owner holds, drop
9924    /// the retained intent and every pending commit that provably writes into
9925    /// them, and delete now-unauthorized blob bodies. `Ok(false)` when the
9926    /// table has no local scope-column mapping (fail closed: nothing deleted).
9927    fn purge_pair(
9928        &mut self,
9929        batch: &mut ChangeAccumulator,
9930        table: &TableSchema,
9931        last: &[(String, Vec<String>)],
9932        owners: &[Vec<(String, Vec<String>)>],
9933    ) -> Result<bool, String> {
9934        if local_scope_predicate(table, last, owners).is_err() {
9935            return Ok(false);
9936        }
9937        for commit in self.drop_failed_commits(|name, values| {
9938            name == table.name && owns_local_row(table, last, owners, values)
9939        })? {
9940            batch.conflicts = true;
9941            batch.rejections = true;
9942            batch.outcomes = true;
9943            for operation in commit.ops {
9944                batch.table(&operation.table);
9945            }
9946        }
9947        if !self.purge_scope_rows(&table.name, last, owners)? {
9948            return Err("storage.failed: the revoked scope purge failed".to_owned());
9949        }
9950        self.record_scope_map(batch, &table.name, last);
9951        if self.drop_doomed_outbox(table, last, owners)? {
9952            batch.status = true;
9953            batch.rejections = true;
9954            batch.outcomes = true;
9955        }
9956        // §5.9.7 B2: revocation deletes now-unauthorized blob bodies.
9957        self.delete_unreferenced_cached_blobs()?;
9958        Ok(true)
9959    }
9960
9961    /// §4.5 client application: apply one live-section or delta `COMMIT`
9962    /// with only the changes that match the persisted echo of a receiving
9963    /// unit; every applied change then goes through the version guard.
9964    /// Returns whether any change of the frame applied.
9965    fn apply_live_commit(
9966        &mut self,
9967        receivers: &[UnitRecord],
9968        tables: &[String],
9969        changes: Vec<ssp2::model::Change>,
9970        commit_seq: i64,
9971        source: AppliedServerCommitSource,
9972    ) -> Result<bool, TransportError> {
9973        let changes: Vec<ssp2::model::Change> = changes
9974            .into_iter()
9975            .filter(|change| {
9976                // An unknown table fails loud in the apply below.
9977                let Some(table) = tables
9978                    .get(change.table_index as usize)
9979                    .and_then(|name| self.schema.table(name))
9980                else {
9981                    return true;
9982                };
9983                receivers.iter().any(|unit| {
9984                    covers(table, &unit.scopes)
9985                        && pair_scopes(&unit.scopes, &table.name, unit.echo.as_deref()).is_some_and(
9986                            |scopes| {
9987                                scopes.iter().all(|(variable, values)| {
9988                                    change
9989                                        .scopes
9990                                        .iter()
9991                                        .find(|(key, _)| key == variable)
9992                                        .is_some_and(|(_, value)| values.contains(value))
9993                                })
9994                            },
9995                        )
9996                })
9997            })
9998            .collect();
9999        if changes.is_empty() {
10000            return Ok(false);
10001        }
10002        let affected_tables = changes
10003            .iter()
10004            .filter_map(|change| tables.get(change.table_index as usize))
10005            .cloned()
10006            .collect::<BTreeSet<_>>();
10007        let previous_batches = self.change_queue.len();
10008        self.apply_commit_frame(tables, &changes, commit_seq)?;
10009        if self.change_queue.len() > previous_batches {
10010            if let Some(batch) = self.change_queue.back_mut() {
10011                batch
10012                    .applied_commits
10013                    .extend(
10014                        affected_tables
10015                            .into_iter()
10016                            .map(|table| AppliedServerCommit {
10017                                table,
10018                                commit_seq,
10019                                source,
10020                            }),
10021                    );
10022            }
10023        }
10024        Ok(true)
10025    }
10026
10027    // Both pull and realtime frames use the same durable observation boundary.
10028    fn apply_commit_frame(
10029        &mut self,
10030        tables: &[String],
10031        changes: &[ssp2::model::Change],
10032        sequence: i64,
10033    ) -> Result<(), TransportError> {
10034        #[cfg(feature = "bench-internals")]
10035        let _phase = self.benchmark_phases.start(Phase::CommitApply);
10036        let rollback = self.capture();
10037        self.coalesce_unit(changes.len())
10038            .and_then(|()| self.begin_observation("syncular_commit"))
10039            .map_err(|message| TransportError::new("storage.failed", message))?;
10040        let applied = (|| {
10041            let mut batch = ChangeAccumulator::default();
10042            self.record_commit_changes(&mut batch, tables, changes);
10043            // §7.1 chained base: a change names the version an acknowledged own
10044            // commit wrote; pending upserts chained on it move to that version.
10045            for change in changes {
10046                let Some(table) = tables.get(change.table_index as usize) else {
10047                    continue;
10048                };
10049                let acknowledged = self
10050                    .conn
10051                    .prepare_cached("SELECT op_json FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2 AND commit_seq=?3")
10052                    .and_then(|mut statement| {
10053                        statement
10054                            .query_map(rusqlite::params![table, change.row_id, sequence], |row| row.get::<_, String>(0))?
10055                            .collect::<Result<Vec<_>, _>>()
10056                    })
10057                    .map_err(|error| TransportError::new("storage.failed", Self::sqlite_failure(&self.storage_failure, error)))?
10058                    .iter()
10059                    .map(|op| serde_json::from_str::<CommitOperation>(op).map(OutboxOp::from))
10060                    .collect::<Result<Vec<_>, _>>()
10061                    .map_err(|_| TransportError::new("storage.failed", "sync.local_corrupt: invalid acknowledged operation".to_owned()))?;
10062                for index in 0..self.outbox.len() {
10063                    if !self.outbox[index]
10064                        .ops
10065                        .iter()
10066                        .any(|op| acknowledged.iter().any(|earlier| chains_on(earlier, op)))
10067                    {
10068                        continue;
10069                    }
10070                    let commit = Arc::make_mut(&mut self.outbox[index]);
10071                    for op in &mut commit.ops {
10072                        if acknowledged.iter().any(|earlier| chains_on(earlier, op)) {
10073                            op.base_version = Some(change.row_version);
10074                        }
10075                    }
10076                    let ops = serde_json::to_string(
10077                        &commit
10078                            .ops
10079                            .iter()
10080                            .map(CommitOperation::from)
10081                            .collect::<Vec<_>>(),
10082                    )
10083                    .map_err(|_| {
10084                        TransportError::new(
10085                            "storage.failed",
10086                            "sync.local_corrupt: unencodable outbox".to_owned(),
10087                        )
10088                    })?;
10089                    let id = commit.client_commit_id.clone();
10090                    self.conn
10091                        .execute(
10092                            "UPDATE _syncular_outbox SET ops_json=?1 WHERE commit_id=?2",
10093                            rusqlite::params![ops, id],
10094                        )
10095                        .map_err(|error| {
10096                            TransportError::new(
10097                                "storage.failed",
10098                                Self::sqlite_failure(&self.storage_failure, error),
10099                            )
10100                        })?;
10101                }
10102            }
10103            {
10104                let storage_failed = |error| {
10105                    TransportError::new(
10106                        "storage.failed",
10107                        Self::sqlite_failure(&self.storage_failure, error),
10108                    )
10109                };
10110                let pinned: HashSet<(&str, &str)> = self
10111                    .outbox
10112                    .iter()
10113                    .flat_map(|commit| &commit.ops)
10114                    .map(|op| (op.table.as_str(), op.row_id.as_str()))
10115                    .collect();
10116                for change in changes {
10117                    let table = tables.get(change.table_index as usize).ok_or_else(|| {
10118                        TransportError::new(
10119                            "sync.invalid_request",
10120                            "change tableIndex out of range".to_owned(),
10121                        )
10122                    })?;
10123                    if pinned.contains(&(table.as_str(), change.row_id.as_str())) {
10124                        self.conn.prepare_cached("INSERT INTO _syncular_row_deliveries(tbl,id,commit_seq) VALUES (?1,?2,?3) ON CONFLICT(tbl,id) DO UPDATE SET commit_seq=MAX(commit_seq,excluded.commit_seq)")
10125                            .and_then(|mut statement| statement.execute(rusqlite::params![table, change.row_id, sequence]))
10126                            .map_err(storage_failed)?;
10127                    }
10128                    self.conn.prepare_cached("DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2 AND commit_seq<=?3")
10129                        .and_then(|mut statement| statement.execute(rusqlite::params![table, change.row_id, sequence]))
10130                        .map_err(storage_failed)?;
10131                }
10132            }
10133            self.apply_commit_changes(tables, changes)?;
10134            self.rebuild_overlay_if_dirty()
10135                .map_err(|message| TransportError::new("storage.failed", message))?;
10136            self.finish_observation("syncular_commit", batch)
10137                .map_err(|message| TransportError::new("storage.failed", message))
10138        })();
10139        if applied.is_err() {
10140            self.rollback_observation("syncular_commit");
10141            self.restore(rollback);
10142        }
10143        applied
10144    }
10145
10146    fn apply_commit_changes(
10147        &mut self,
10148        tables: &[String],
10149        changes: &[ssp2::model::Change],
10150    ) -> Result<(), TransportError> {
10151        // A clean overlay can reconcile changed primary keys independently
10152        // when their tables have no secondary unique constraints. Such a
10153        // constraint can change whether another row's write succeeds, so a
10154        // table with one and any local intent takes the table rebuild.
10155        let changed_tables: BTreeSet<&str> = changes
10156            .iter()
10157            .filter_map(|change| tables.get(change.table_index as usize))
10158            .map(String::as_str)
10159            .collect();
10160        let mut mirror_visible = !self.overlay_dirty.get();
10161        for name in &changed_tables {
10162            if mirror_visible
10163                && self
10164                    .schema
10165                    .table(name)
10166                    .is_some_and(|table| table.indexes.iter().any(|index| index.unique))
10167            {
10168                let (pending, protected) = self
10169                    .table_intent(name)
10170                    .map_err(|message| TransportError::new("storage.failed", message))?;
10171                mirror_visible = !pending && !protected;
10172            }
10173        }
10174        for change in changes {
10175            let table_name = tables.get(change.table_index as usize).ok_or_else(|| {
10176                TransportError::new(
10177                    "sync.invalid_request",
10178                    "change tableIndex out of range".to_owned(),
10179                )
10180            })?;
10181            let table = self.schema.table(table_name).ok_or_else(|| {
10182                TransportError::new(
10183                    "sync.schema_mismatch",
10184                    format!("change targets unknown table {table_name:?}"),
10185                )
10186            })?;
10187            let landed = match change.op {
10188                Op::Upsert => {
10189                    let payload = change.row.as_ref().ok_or_else(|| {
10190                        TransportError::new(
10191                            "sync.invalid_request",
10192                            "upsert change without row payload".to_owned(),
10193                        )
10194                    })?;
10195                    // §5.11: decrypt encrypted columns on apply, or seal the row.
10196                    let (row, sealed) = {
10197                        #[cfg(feature = "bench-internals")]
10198                        let _phase = self.benchmark_phases.start(Phase::RowDecode);
10199                        // §5.11: a decrypt failure keeps its own identity.
10200                        decode_row_bytes(table, payload, &self.encryption)?
10201                    };
10202                    let table_name = table.name.clone();
10203                    // §4.5 version guard: only a newer upsert lands.
10204                    let landed = self
10205                        .write_base_row(&table_name, &row, change.row_version, sealed.as_ref())
10206                        .map_err(|m| TransportError::new("sync.invalid_request", m))?;
10207                    if mirror_visible && landed {
10208                        self.write_row(
10209                            &visible_table(&table_name),
10210                            &table_name,
10211                            &row,
10212                            change.row_version,
10213                            sealed.as_ref(),
10214                        )
10215                        .map_err(|m| TransportError::new("sync.invalid_request", m))?;
10216                    }
10217                    landed
10218                }
10219                // §4.5 version guard: a delete at or below the local version
10220                // is skipped, a delete of an absent row is a no-op.
10221                Op::Delete => self
10222                    .delete_base_row(table_name, &change.row_id, change.row_version)
10223                    .map_err(|m| TransportError::new("sync.invalid_request", m))?,
10224            };
10225            // The change dropped the row's acknowledged intent either way
10226            // (§7.1), so the overlay of a skipped change rebuilds from the
10227            // unchanged base.
10228            self.overlay_dirty.table(table_name);
10229            if mirror_visible && !(landed && change.op == Op::Upsert) {
10230                let predicate = row_id_predicate(table);
10231                let visible = visible_table(table_name);
10232                let mut statements = vec![format!("DELETE FROM {visible} WHERE {predicate}")];
10233                if !landed {
10234                    statements.push(format!(
10235                        "INSERT INTO {visible} SELECT * FROM {} WHERE {predicate}",
10236                        base_table(table_name)
10237                    ));
10238                }
10239                for statement in &statements {
10240                    self.conn
10241                        .prepare_cached(statement)
10242                        .and_then(|mut statement| {
10243                            statement.execute(rusqlite::params![change.row_id])
10244                        })
10245                        .map_err(|error| {
10246                            TransportError::new("sync.invalid_request", error.to_string())
10247                        })?;
10248                }
10249            }
10250        }
10251        if mirror_visible {
10252            let changed_rows: HashSet<(&str, &str)> = changes
10253                .iter()
10254                .map(|change| {
10255                    (
10256                        tables[change.table_index as usize].as_str(),
10257                        change.row_id.as_str(),
10258                    )
10259                })
10260                .collect();
10261            self.replay_overlay(&changed_tables, Some(&changed_rows))
10262                .map_err(|message| TransportError::new("storage.failed", message))?;
10263            self.overlay_dirty.set(false);
10264        }
10265        Ok(())
10266    }
10267
10268    /// §5.6 segment application: validate against the generated schema,
10269    /// clear the pair's scope on the table's first page (fail closed
10270    /// without a mapping), then upsert each row with its segment-carried
10271    /// server version under the §4.5 version guard (§5.2).
10272    fn apply_segment(
10273        &mut self,
10274        table: &TableSchema,
10275        segment: &RowsSegment,
10276        first_page: bool,
10277        effective: &[(String, Vec<String>)],
10278        unit_id: &str,
10279    ) -> Result<u32, SectionError> {
10280        // §5.2: the column table validates against the generated schema —
10281        // order, names, types, nullability; mismatch is fatal. §5.11: the
10282        // server sends the WIRE types (bytes for an encrypted column), so
10283        // validate against wire_columns.
10284        let column = if segment.columns.len() != table.wire_columns.len() {
10285            Some(Value::from("count"))
10286        } else {
10287            segment
10288                .columns
10289                .iter()
10290                .zip(table.wire_columns.iter())
10291                .position(|(a, b)| a.name != b.name || a.ty != b.ty || a.nullable != b.nullable)
10292                .map(Value::from)
10293        };
10294        if segment.table != table.name
10295            || segment.schema_version != self.schema.version
10296            || column.is_some()
10297        {
10298            let mut details = Map::new();
10299            details.insert("table".to_owned(), Value::from(table.name.as_str()));
10300            if let Some(column) = column {
10301                details.insert("column".to_owned(), column);
10302            }
10303            return Err(SectionError::Transport(TransportError {
10304                code: "sync.schema_mismatch".to_owned(),
10305                message: "rows segment columns do not match the generated schema (§5.2)".to_owned(),
10306                details: Some(Value::Object(details)),
10307            }));
10308        }
10309        let mut applied = 0u32;
10310        let empty = Vec::new();
10311        let blocks = if segment.blocks.is_empty() {
10312            std::slice::from_ref(&empty)
10313        } else {
10314            &segment.blocks
10315        };
10316        for (index, block) in blocks.iter().enumerate() {
10317            let clear = first_page && index == 0;
10318            if !clear && block.is_empty() {
10319                continue;
10320            }
10321            let rollback = self.capture();
10322            let (pending, protected) = self
10323                .table_intent(&table.name)
10324                .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10325            let reconcile_pending =
10326                !protected && pending && table.indexes.iter().any(|index| index.unique);
10327            let mirror_visible = !protected && !reconcile_pending;
10328            self.coalesce_unit(block.len())
10329                .and_then(|()| self.begin_observation("syncular_segment_block"))
10330                .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10331            let outcome = (|| {
10332                self.rebuild_overlay_if_dirty()
10333                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10334                let mut changed_keys = Vec::new();
10335                let mut batch = ChangeAccumulator::default();
10336                if clear && self.clear_scope(table, effective, unit_id)? || !block.is_empty() {
10337                    batch.table(&table.name);
10338                }
10339                for (row_index, row) in block.iter().enumerate() {
10340                    let decrypted;
10341                    let mut sealed = None;
10342                    let values = if table.has_encrypted_columns() {
10343                        let mut values = row.values.clone();
10344                        sealed = crate::values::decrypt_segment_row(
10345                            table,
10346                            &mut values,
10347                            &self.encryption,
10348                        )
10349                        .map_err(SectionError::Transport)?;
10350                        decrypted = values;
10351                        &decrypted
10352                    } else {
10353                        &row.values
10354                    };
10355                    if !self
10356                        .write_base_row(&table.name, values, row.server_version, sealed.as_ref())
10357                        .map_err(|message| {
10358                            SectionError::Abort("sync.invalid_request".into(), message)
10359                        })?
10360                    {
10361                        continue;
10362                    }
10363                    if reconcile_pending {
10364                        changed_keys.push(
10365                            owned_sql_value(RowParam::Cell(&values[table.pk_index])).map_err(
10366                                |message| SectionError::Abort("storage.failed".into(), message),
10367                            )?,
10368                        );
10369                    }
10370                    if mirror_visible {
10371                        self.write_row(
10372                            &visible_table(&table.name),
10373                            &table.name,
10374                            values,
10375                            row.server_version,
10376                            sealed.as_ref(),
10377                        )
10378                        .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10379                    }
10380                    let processed = u64::from(applied) + row_index as u64 + 1;
10381                    if processed.is_multiple_of(1024) {
10382                        self.progress.update(|p| p.rows_processed = processed);
10383                    }
10384                }
10385                self.reconcile_import(table, &mut changed_keys, protected, reconcile_pending)
10386                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10387                self.finish_observation("syncular_segment_block", batch)
10388                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))
10389            })();
10390            if let Err(error) = outcome {
10391                self.rollback_observation("syncular_segment_block");
10392                self.restore(rollback);
10393                return Err(error);
10394            }
10395            applied += block.len() as u32;
10396            self.progress
10397                .update(|p| p.rows_processed = u64::from(applied));
10398            std::thread::yield_now();
10399        }
10400        self.progress
10401            .update(|p| p.rows_processed = u64::from(applied));
10402        Ok(applied)
10403    }
10404
10405    /// §5.3 sqlite-image application: validate the in-file metadata
10406    /// against the descriptor, validate column names/order against the
10407    /// generated schema, run the §5.6 first-page clear when fresh, then
10408    /// replace-or-upsert every image row with its `_syncular_version`.
10409    /// Mechanics: the image lands in a temp file read through a second
10410    /// rusqlite connection, equivalent to ATTACH + INSERT…SELECT.
10411    /// Each image chunk commits its writes and revision before readers continue.
10412    #[allow(clippy::too_many_arguments)]
10413    fn apply_sqlite_segment(
10414        &mut self,
10415        table: &TableSchema,
10416        bytes: &[u8],
10417        first_page: bool,
10418        effective: &[(String, Vec<String>)],
10419        unit_id: &str,
10420        segment_id: &str,
10421        resumed: i64,
10422        row_count: i64,
10423        as_of_commit_seq: i64,
10424        scope_digest: &str,
10425    ) -> Result<u32, SectionError> {
10426        let invalid = |detail: &str| {
10427            SectionError::Abort(
10428                "sync.invalid_request".to_owned(),
10429                format!("sqlite segment rejected: {detail} (§5.3)"),
10430            )
10431        };
10432        // §5.11: an image bypasses the decrypt seam, so a server cannot serve
10433        // one for a table with encrypted columns.
10434        if table.has_encrypted_columns() {
10435            return Err(invalid("table has encrypted columns"));
10436        }
10437
10438        // The image lands beside the replica so it shares the replica's
10439        // directory permissions and volume; an in-memory replica has no
10440        // directory and uses the system temp dir. The file is owner-only.
10441        let directory = self
10442            .previous_version_replica_path()
10443            .and_then(|replica| std::path::Path::new(&replica).parent().map(Into::into))
10444            .unwrap_or_else(std::env::temp_dir);
10445        let path = directory.join(format!(".syncular-image-{}.tmp", uuid::Uuid::new_v4()));
10446        let outcome = (|| {
10447            let mut options = std::fs::OpenOptions::new();
10448            options.write(true).create_new(true);
10449            #[cfg(unix)]
10450            std::os::unix::fs::OpenOptionsExt::mode(&mut options, 0o600);
10451            std::io::Write::write_all(
10452                &mut options
10453                    .open(&path)
10454                    .map_err(|_| invalid("image temp file write failed"))?,
10455                bytes,
10456            )
10457            .map_err(|_| invalid("image temp file write failed"))?;
10458            let img = rusqlite::Connection::open_with_flags(
10459                &path,
10460                rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY,
10461            )
10462            .map_err(|_| invalid("bytes do not open as a SQLite database"))?;
10463            self.apply_sqlite_image(
10464                &img,
10465                table,
10466                first_page,
10467                effective,
10468                unit_id,
10469                segment_id,
10470                resumed,
10471                row_count,
10472                as_of_commit_seq,
10473                scope_digest,
10474            )
10475        })();
10476        let _ = std::fs::remove_file(&path);
10477        outcome
10478    }
10479
10480    #[allow(clippy::too_many_arguments)]
10481    fn apply_sqlite_image(
10482        &mut self,
10483        img: &rusqlite::Connection,
10484        table: &crate::schema::TableSchema,
10485        first_fresh_page: bool,
10486        effective: &[(String, Vec<String>)],
10487        unit_id: &str,
10488        segment_id: &str,
10489        resumed: i64,
10490        row_count: i64,
10491        as_of_commit_seq: i64,
10492        scope_digest: &str,
10493    ) -> Result<u32, SectionError> {
10494        let invalid = |detail: String| {
10495            SectionError::Abort(
10496                "sync.invalid_request".to_owned(),
10497                format!("sqlite segment rejected: {detail} (§5.3)"),
10498            )
10499        };
10500
10501        // 1. Metadata vs descriptor + client state (§5.3 rule 2; exactly
10502        //    one row).
10503        type MetaRow = (i64, String, i64, i64, String, i64, i64);
10504        let meta: MetaRow = img
10505            .query_row(
10506                "SELECT format, \"table\", \"schemaVersion\", \"asOfCommitSeq\",
10507                        \"scopeDigest\", \"rowCount\",
10508                        (SELECT count(*) FROM _syncular_segment)
10509                 FROM _syncular_segment",
10510                [],
10511                |row| {
10512                    Ok((
10513                        row.get(0)?,
10514                        row.get(1)?,
10515                        row.get(2)?,
10516                        row.get(3)?,
10517                        row.get(4)?,
10518                        row.get(5)?,
10519                        row.get(6)?,
10520                    ))
10521                },
10522            )
10523            .map_err(|_| invalid("missing or unreadable _syncular_segment metadata".to_owned()))?;
10524        let (format, meta_table, schema_version, pin, digest, meta_rows, meta_count) = meta;
10525        if meta_count != 1 {
10526            return Err(invalid(format!(
10527                "_syncular_segment must contain exactly one row, found {meta_count}"
10528            )));
10529        }
10530        if format != 1 {
10531            return Err(invalid(format!("format {format}")));
10532        }
10533        if meta_table != table.name {
10534            return Err(invalid(format!("image table {meta_table:?}")));
10535        }
10536        if schema_version != i64::from(self.schema.version) {
10537            return Err(invalid(format!("schemaVersion {schema_version}")));
10538        }
10539        if pin != as_of_commit_seq {
10540            return Err(invalid(format!("asOfCommitSeq {pin}")));
10541        }
10542        if digest != scope_digest {
10543            return Err(invalid("scopeDigest mismatch".to_owned()));
10544        }
10545        if meta_rows != row_count {
10546            return Err(invalid(format!("rowCount {meta_rows}")));
10547        }
10548
10549        // 2. Column names and order vs the generated schema (§5.3 rule 3).
10550        let mut names: Vec<String> = Vec::new();
10551        let mut primary_keys: Vec<String> = Vec::new();
10552        {
10553            let mut stmt = img
10554                .prepare(&format!("PRAGMA table_info({})", quote_ident(&table.name)))
10555                .map_err(|_| invalid("image data table missing".to_owned()))?;
10556            let mut rows = stmt
10557                .query([])
10558                .map_err(|_| invalid("image data table unreadable".to_owned()))?;
10559            while let Some(row) = rows
10560                .next()
10561                .map_err(|_| invalid("image data table unreadable".to_owned()))?
10562            {
10563                if row
10564                    .get::<_, i64>(5)
10565                    .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?
10566                    > 0
10567                {
10568                    primary_keys.push(row.get(1).map_err(|error| {
10569                        invalid(Self::sqlite_failure(&self.storage_failure, error))
10570                    })?);
10571                }
10572                names.push(
10573                    row.get::<_, String>(1)
10574                        .map_err(|_| invalid("image data table unreadable".to_owned()))?,
10575                );
10576            }
10577        }
10578        let mut expected: Vec<&str> = table.columns.iter().map(|c| c.name.as_str()).collect();
10579        expected.push("_syncular_version");
10580        if names.len() != expected.len() || names.iter().zip(expected.iter()).any(|(a, b)| a != b) {
10581            return Err(SectionError::Transport(TransportError {
10582                code: "sync.schema_mismatch".to_owned(),
10583                message: "sqlite segment columns do not match the generated schema (§5.3)"
10584                    .to_owned(),
10585                details: Some(serde_json::json!({ "table": table.name })),
10586            }));
10587        }
10588
10589        if primary_keys != vec![table.primary_key.clone()] {
10590            return Err(invalid(
10591                "image primary key does not match generated schema".into(),
10592            ));
10593        }
10594        let actual_count: i64 = img
10595            .query_row(
10596                &format!("SELECT count(*) FROM {}", quote_ident(&table.name)),
10597                [],
10598                |row| row.get(0),
10599            )
10600            .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
10601        if actual_count != row_count {
10602            return Err(invalid("image row count does not match descriptor".into()));
10603        }
10604        let insert = self.insert_row_sql(&base_table(&table.name), table);
10605        let visible_insert = self.insert_row_sql(&visible_table(&table.name), table);
10606        let column_list = names
10607            .iter()
10608            .map(|n| quote_ident(n))
10609            .collect::<Vec<_>>()
10610            .join(", ");
10611        // §5.6: an earlier attempt at this position committed the first
10612        // `resumed` rows in primary-key order; the content address pins them.
10613        let mut stmt = img
10614            .prepare(&format!(
10615                "SELECT {column_list} FROM {} ORDER BY {} LIMIT -1 OFFSET ?1",
10616                quote_ident(&table.name),
10617                quote_ident(&table.primary_key)
10618            ))
10619            .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
10620        let mut rows = stmt
10621            .query([resumed])
10622            .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
10623        let resumed = u32::try_from(resumed)
10624            .ok()
10625            .filter(|rows| i64::from(*rows) <= actual_count)
10626            .ok_or_else(|| invalid("applied-segment record exceeds the image".into()))?;
10627        let mut applied = resumed;
10628        let mut limit = if first_fresh_page {
10629            1024
10630        } else {
10631            self.image_chunk_rows
10632                .get(&table.name)
10633                .copied()
10634                .unwrap_or(1024)
10635        };
10636        let (mut write, mut commit) = (std::time::Duration::ZERO, std::time::Duration::ZERO);
10637        loop {
10638            let clear = applied == 0 && first_fresh_page;
10639            let rollback = self.capture();
10640            let (pending, protected) = self
10641                .table_intent(&table.name)
10642                .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10643            let reconcile_pending =
10644                !protected && pending && table.indexes.iter().any(|index| index.unique);
10645            let mirror_visible = !protected && !reconcile_pending;
10646            // §5.3: a table's first chunk holds 1,024 rows; a later range image
10647            // starts at the size its previous range reached. A later chunk holds at
10648            // least 1,024 rows, at most twice the rows of the one before, at
10649            // most 65,536, and at most the rows the one before applied per
10650            // 250 ms from its start to its COMMIT; a chunk that replays local
10651            // writes holds at most 1,024.
10652            let chunk = if pending || protected {
10653                limit.min(1024)
10654            } else {
10655                limit
10656            };
10657            self.coalesce_unit(chunk as usize)
10658                .and_then(|()| self.begin_observation("syncular_image_chunk"))
10659                .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10660            let started = std::time::Instant::now();
10661            let result = (|| {
10662                self.rebuild_overlay_if_dirty()
10663                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10664                let mut changed_keys = Vec::new();
10665                let mut batch = ChangeAccumulator::default();
10666                if clear && self.clear_scope(table, effective, unit_id)? {
10667                    batch.table(&table.name);
10668                }
10669                let mut processed = 0u32;
10670                {
10671                    let mut ins = self.conn.prepare_cached(&insert).map_err(|error| {
10672                        invalid(Self::sqlite_failure(&self.storage_failure, error))
10673                    })?;
10674                    let mut mirror = if mirror_visible {
10675                        Some(self.conn.prepare_cached(&visible_insert).map_err(|error| {
10676                            invalid(Self::sqlite_failure(&self.storage_failure, error))
10677                        })?)
10678                    } else {
10679                        None
10680                    };
10681                    for _ in 0..chunk {
10682                        let Some(row) = rows.next().map_err(|error| {
10683                            invalid(Self::sqlite_failure(&self.storage_failure, error))
10684                        })?
10685                        else {
10686                            break;
10687                        };
10688                        for (i, column) in table.columns.iter().enumerate() {
10689                            let cell = row.get_ref(i).map_err(|error| {
10690                                invalid(Self::sqlite_failure(&self.storage_failure, error))
10691                            })?;
10692                            let param = image_cell_param(column, cell).map_err(invalid)?;
10693                            ins.raw_bind_parameter(i + 1, &param).map_err(|error| {
10694                                invalid(Self::sqlite_failure(&self.storage_failure, error))
10695                            })?;
10696                            if let Some(mirror) = &mut mirror {
10697                                mirror.raw_bind_parameter(i + 1, &param).map_err(|error| {
10698                                    invalid(Self::sqlite_failure(&self.storage_failure, error))
10699                                })?;
10700                            }
10701                        }
10702                        let version_index = table.columns.len();
10703                        let version: i64 = row.get(version_index).map_err(|error| {
10704                            invalid(Self::sqlite_failure(&self.storage_failure, error))
10705                        })?;
10706                        if version < 1 {
10707                            return Err(invalid("image row version must be positive".into()));
10708                        }
10709                        ins.raw_bind_parameter(version_index + 1, version)
10710                            .map_err(|error| {
10711                                invalid(Self::sqlite_failure(&self.storage_failure, error))
10712                            })?;
10713                        // §4.5 version guard: the base upsert decides.
10714                        let landed = ins.raw_execute().map_err(|error| {
10715                            invalid(Self::sqlite_failure(&self.storage_failure, error))
10716                        })? > 0;
10717                        processed += 1;
10718                        if processed.is_multiple_of(1024) && processed < chunk {
10719                            self.progress
10720                                .update(|p| p.rows_processed = u64::from(applied + processed));
10721                        }
10722                        if !landed {
10723                            continue;
10724                        }
10725                        if let Some(mirror) = &mut mirror {
10726                            mirror
10727                                .raw_bind_parameter(version_index + 1, version)
10728                                .map_err(|error| {
10729                                    invalid(Self::sqlite_failure(&self.storage_failure, error))
10730                                })?;
10731                            mirror.raw_execute().map_err(|error| {
10732                                invalid(Self::sqlite_failure(&self.storage_failure, error))
10733                            })?;
10734                        }
10735                        if reconcile_pending {
10736                            changed_keys.push(row.get::<_, SqlValue>(table.pk_index).map_err(
10737                                |error| invalid(Self::sqlite_failure(&self.storage_failure, error)),
10738                            )?);
10739                        }
10740                    }
10741                }
10742                if processed > 0 {
10743                    batch.table(&table.name);
10744                    self.overlay_dirty.table(&table.name);
10745                }
10746                self.stored(record_applied_rows(
10747                    &self.conn,
10748                    unit_id,
10749                    segment_id,
10750                    i64::from(applied + processed),
10751                ))
10752                .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10753                self.reconcile_import(table, &mut changed_keys, protected, reconcile_pending)
10754                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10755                let written = std::time::Instant::now();
10756                self.finish_observation("syncular_image_chunk", batch)
10757                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10758                Ok((processed, written))
10759            })();
10760            let (processed, written) = match result {
10761                Ok(done) => done,
10762                Err(error) => {
10763                    self.rollback_observation("syncular_image_chunk");
10764                    self.restore(rollback);
10765                    return Err(error);
10766                }
10767            };
10768            // The chunk commits before the yield, also inside a coalesced apply.
10769            let coalesced = self.coalesced.is_some();
10770            if coalesced {
10771                self.commit_coalesced_apply()
10772                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10773            }
10774            write += written - started;
10775            commit += written.elapsed();
10776            applied += processed;
10777            let last = processed < chunk || i64::from(applied) == actual_count;
10778            // A file replica's WAL keeps its high-water size until truncated.
10779            if last && self.conn.is_autocommit() {
10780                self.conn
10781                    .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))
10782                    .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
10783            }
10784            if coalesced {
10785                self.begin_coalesced_apply()
10786                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10787            }
10788            self.progress.update(|p| {
10789                p.rows_processed = u64::from(applied);
10790                p.import_write_ms = u64::try_from(write.as_millis()).ok();
10791                p.import_commit_ms = u64::try_from(commit.as_millis()).ok();
10792            });
10793            let duration = started.elapsed().as_secs_f64() * 1000.0;
10794            #[cfg(test)]
10795            let duration = self.image_chunk_ms.get().unwrap_or(duration);
10796            std::thread::yield_now();
10797            if last {
10798                break;
10799            }
10800            // A zero duration saturates the cast and leaves the doubling.
10801            limit = (processed * 2)
10802                .min(65_536)
10803                .min((f64::from(processed) * 250.0 / duration) as u32)
10804                .max(1024);
10805            self.image_chunk_rows.insert(table.name.clone(), limit);
10806        }
10807        if i64::from(applied) != actual_count {
10808            return Err(invalid(format!(
10809                "image yielded {applied} rows, expected {actual_count}"
10810            )));
10811        }
10812        Ok(applied - resumed)
10813    }
10814
10815    /// §7.1 keyed reconciliation of the rows `drained` commits touch: restore
10816    /// them from the base and replay their remaining intent. A table with a
10817    /// secondary unique index restores and replays every row with intent,
10818    /// since one row's value can admit or defer another row's write.
10819    fn reconcile_rows(&self, drained: &[Arc<OutboxCommit>]) -> Result<(), String> {
10820        let names: BTreeSet<&str> = drained
10821            .iter()
10822            .flat_map(|commit| &commit.ops)
10823            .map(|op| op.table.as_str())
10824            .collect();
10825        for name in names {
10826            let Some(table) = self.schema.table(name) else {
10827                continue;
10828            };
10829            let ops = drained.iter().flat_map(|commit| &commit.ops);
10830            let tables = BTreeSet::from([name]);
10831            let rows: HashSet<_> = ops
10832                .clone()
10833                .map(|op| (op.table.as_str(), op.row_id.as_str()))
10834                .collect();
10835            let unique = table.indexes.iter().any(|index| index.unique);
10836            let acknowledged = if unique {
10837                self.acknowledged_operations(None, Some(&tables))?
10838            } else {
10839                Vec::new()
10840            };
10841            let peers = acknowledged
10842                .iter()
10843                .chain(
10844                    self.failed_commits
10845                        .iter()
10846                        .flat_map(|(commit, _)| &commit.ops),
10847                )
10848                .chain(self.outbox.iter().flat_map(|commit| &commit.ops))
10849                .filter(|_| unique);
10850            self.restore_base_rows(table, &mut Vec::new(), peers.chain(ops.clone()))?;
10851            self.replay_overlay(&tables, (!unique).then_some(&rows))?;
10852        }
10853        Ok(())
10854    }
10855
10856    /// Whether `table` has pending outbox intent, and whether it has protected
10857    /// (acknowledged or retained failed) intent.
10858    fn table_intent(&self, table: &str) -> Result<(bool, bool), String> {
10859        let touches = |op: &OutboxOp| op.table == table;
10860        Ok((
10861            self.outbox
10862                .iter()
10863                .flat_map(|commit| &commit.ops)
10864                .any(touches),
10865            self.failed_commits
10866                .iter()
10867                .flat_map(|(commit, _)| &commit.ops)
10868                .any(touches)
10869                || self
10870                    .conn
10871                    .query_row(
10872                        "SELECT EXISTS(SELECT 1 FROM _syncular_acked_rows WHERE tbl=?1)",
10873                        [table],
10874                        |row| row.get(0),
10875                    )
10876                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
10877        ))
10878    }
10879
10880    /// §7.1 reconciliation after an import block or chunk. Keyed replay covers
10881    /// only pending intent, so a table with protected intent rebuilds; a
10882    /// `restore` restores the imported `keys` and every pending row first.
10883    fn reconcile_import(
10884        &mut self,
10885        table: &TableSchema,
10886        keys: &mut Vec<SqlValue>,
10887        protected: bool,
10888        restore: bool,
10889    ) -> Result<(), String> {
10890        if protected {
10891            return self.rebuild_overlay_if_dirty();
10892        }
10893        let pending = self.outbox.iter().flat_map(|commit| &commit.ops);
10894        if restore {
10895            self.restore_base_rows(table, keys, pending.clone())?;
10896        }
10897        self.apply_outbox_ops(pending.filter(|op| op.table == table.name))?;
10898        self.overlay_dirty.set(false);
10899        Ok(())
10900    }
10901
10902    /// Restore the visible rows of `keys` and of the rows `ops` touch in
10903    /// `table` from the base before the caller replays their intent. The
10904    /// caller owns the transaction and publishes its revision after replay.
10905    fn restore_base_rows<'a>(
10906        &self,
10907        table: &TableSchema,
10908        keys: &mut Vec<SqlValue>,
10909        ops: impl IntoIterator<Item = &'a OutboxOp>,
10910    ) -> Result<(), String> {
10911        let visible = visible_table(&table.name);
10912        let base = base_table(&table.name);
10913        let pk = quote_ident(&table.primary_key);
10914        let predicate = row_id_predicate(table);
10915        let mut lookup = self.conn.prepare_cached(&format!(
10916            "SELECT {pk} FROM {base} WHERE {predicate} UNION SELECT {pk} FROM {visible} WHERE {predicate}"
10917        )).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
10918        let mut deleted = HashSet::new();
10919        for op in ops.into_iter().filter(|op| op.table == table.name) {
10920            if op.upsert {
10921                if let Some(values) = &op.values {
10922                    let key = json_to_column_value(
10923                        &table.columns[table.pk_index],
10924                        values.get(&table.primary_key),
10925                    )?;
10926                    keys.push(owned_sql_value(RowParam::Cell(&key))?);
10927                }
10928            } else if deleted.insert(&op.row_id) {
10929                keys.extend(
10930                    lookup
10931                        .query_map([&op.row_id], |row| row.get::<_, SqlValue>(0))
10932                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
10933                        .collect::<Result<Vec<_>, _>>()
10934                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
10935                );
10936            }
10937        }
10938        // Delete the complete affected set before restoring any base value, so
10939        // old optimistic unique values cannot collide with the new server rows.
10940        for chunk in keys.chunks(400) {
10941            let holes = vec!["?"; chunk.len()].join(",");
10942            self.conn
10943                .prepare_cached(&format!("DELETE FROM {visible} WHERE {pk} IN ({holes})"))
10944                .and_then(|mut statement| statement.execute(rusqlite::params_from_iter(chunk)))
10945                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
10946        }
10947        let updates = table
10948            .columns
10949            .iter()
10950            .map(|column| column.name.as_str())
10951            .chain(std::iter::once("_syncular_version"))
10952            .chain(table.sealed_columns().iter().copied())
10953            .filter(|name| *name != table.primary_key)
10954            .map(|name| {
10955                let name = quote_ident(name);
10956                format!("{name}=excluded.{name}")
10957            })
10958            .collect::<Vec<_>>()
10959            .join(",");
10960        for chunk in keys.chunks(400) {
10961            let holes = vec!["?"; chunk.len()].join(",");
10962            self.conn.prepare_cached(&format!("INSERT INTO {visible} SELECT * FROM {base} WHERE {pk} IN ({holes}) ON CONFLICT ({pk}) DO UPDATE SET {updates}"))
10963                .and_then(|mut statement| statement.execute(rusqlite::params_from_iter(chunk)))
10964                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
10965        }
10966        Ok(())
10967    }
10968
10969    // -- application-authorized local purge ---------------------------------------
10970
10971    fn compile_local_data_purge(
10972        &self,
10973        input: &LocalDataPurgeInput,
10974    ) -> Result<(Vec<CompiledLocalDataPurgeTarget>, String), String> {
10975        let invalid = |message: String| format!("sync.invalid_request: {message}");
10976        if input.purge_id.is_empty()
10977            || input.purge_id.len() > 128
10978            || !is_local_operation_code_like(&input.purge_id)
10979        {
10980            return Err(invalid(
10981                "local purge purgeId must be a 1–128 character code-like identifier".to_owned(),
10982            ));
10983        }
10984        if input.targets.is_empty() || input.targets.len() > MAX_LOCAL_PURGE_TARGETS {
10985            return Err(invalid(format!(
10986                "local purge needs between 1 and {MAX_LOCAL_PURGE_TARGETS} targets"
10987            )));
10988        }
10989
10990        let mut deduplicated: BTreeMap<
10991            String,
10992            (CompiledLocalDataPurgeTarget, LocalDataPurgeTarget),
10993        > = BTreeMap::new();
10994        for target in &input.targets {
10995            let table = self.schema.table(&target.table).ok_or_else(|| {
10996                invalid(format!(
10997                    "local purge names unknown table {:?}",
10998                    target.table
10999                ))
11000            })?;
11001            if target.selectors.is_empty() || target.selectors.len() > MAX_LOCAL_PURGE_SELECTORS {
11002                return Err(invalid(format!(
11003                    "local purge target {:?} needs between 1 and {MAX_LOCAL_PURGE_SELECTORS} selectors",
11004                    target.table
11005                )));
11006            }
11007            let mut selectors = Vec::with_capacity(target.selectors.len());
11008            let mut canonical_selectors = BTreeMap::new();
11009            for (column_name, raw_values) in &target.selectors {
11010                let Some((column_index, column)) = table
11011                    .columns
11012                    .iter()
11013                    .enumerate()
11014                    .find(|(_, column)| column.name == *column_name)
11015                else {
11016                    return Err(invalid(format!(
11017                        "local purge target {:?} names unknown column {:?}",
11018                        target.table, column_name
11019                    )));
11020                };
11021                let encrypted = table
11022                    .encrypted_columns
11023                    .iter()
11024                    .any(|candidate| candidate.index == column_index);
11025                if column.ty != ColumnType::String || encrypted {
11026                    return Err(invalid(format!(
11027                        "local purge selector {:?}.{:?} must be a plaintext string column",
11028                        target.table, column_name
11029                    )));
11030                }
11031                if raw_values.is_empty() || raw_values.len() > MAX_LOCAL_PURGE_VALUES {
11032                    return Err(invalid(format!(
11033                        "local purge selector {:?}.{:?} needs between 1 and {MAX_LOCAL_PURGE_VALUES} values",
11034                        target.table, column_name
11035                    )));
11036                }
11037                let mut values = raw_values.clone();
11038                values.sort();
11039                values.dedup();
11040                if values.iter().any(|value| {
11041                    value.is_empty()
11042                        || value.len() > MAX_LOCAL_PURGE_VALUE_LENGTH
11043                        || !is_local_operation_code_like(value)
11044                }) {
11045                    return Err(invalid(format!(
11046                        "local purge selector values must be 1–{MAX_LOCAL_PURGE_VALUE_LENGTH} character code-like identifiers"
11047                    )));
11048                }
11049                selectors.push((column_name.clone(), values.clone()));
11050                canonical_selectors.insert(column_name.clone(), values);
11051            }
11052            let canonical = LocalDataPurgeTarget {
11053                table: target.table.clone(),
11054                selectors: canonical_selectors,
11055            };
11056            let key = serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
11057            deduplicated.insert(
11058                key,
11059                (
11060                    CompiledLocalDataPurgeTarget {
11061                        table: target.table.clone(),
11062                        selectors,
11063                    },
11064                    canonical,
11065                ),
11066            );
11067        }
11068        let targets = deduplicated
11069            .values()
11070            .map(|(compiled, _)| compiled.clone())
11071            .collect::<Vec<_>>();
11072        let canonical = deduplicated
11073            .values()
11074            .map(|(_, target)| target.clone())
11075            .collect::<Vec<_>>();
11076        let canonical_plan =
11077            serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
11078        Ok((targets, canonical_plan))
11079    }
11080
11081    fn local_purge_base_row_ids(
11082        &self,
11083        targets: &[CompiledLocalDataPurgeTarget],
11084    ) -> Result<BTreeMap<String, BTreeSet<String>>, String> {
11085        let mut by_table: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
11086        for target in targets {
11087            let table = self
11088                .schema
11089                .table(&target.table)
11090                .ok_or_else(|| format!("sync.invalid_request: unknown table {:?}", target.table))?;
11091            let mut clauses = Vec::with_capacity(target.selectors.len());
11092            let mut params = Vec::new();
11093            for (column, values) in &target.selectors {
11094                clauses.push(format!(
11095                    "{} IN ({})",
11096                    quote_ident(column),
11097                    values.iter().map(|_| "?").collect::<Vec<_>>().join(", ")
11098                ));
11099                params.extend(values.iter().cloned().map(SqlValue::Text));
11100            }
11101            let sql = format!(
11102                "SELECT CAST({} AS TEXT) FROM {} WHERE {}",
11103                quote_ident(&table.primary_key),
11104                base_table(&target.table),
11105                clauses.join(" AND ")
11106            );
11107            let mut statement = self
11108                .conn
11109                .prepare(&sql)
11110                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11111            let rows = statement
11112                .query_map(rusqlite::params_from_iter(params), |row| {
11113                    let value = sql_ref_to_json(&table.columns[table.pk_index], row.get_ref(0)?);
11114                    render_row_id_json(Some(&value))
11115                        .map_err(|message| rusqlite::Error::ToSqlConversionFailure(message.into()))
11116                })
11117                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11118            let ids = by_table.entry(target.table.clone()).or_default();
11119            for row in rows {
11120                ids.insert(
11121                    row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
11122                );
11123            }
11124        }
11125        Ok(by_table)
11126    }
11127
11128    fn local_purge_values_match(
11129        target: &CompiledLocalDataPurgeTarget,
11130        values: &Map<String, Value>,
11131    ) -> bool {
11132        target.selectors.iter().all(|(column, allowed)| {
11133            values
11134                .get(column)
11135                .and_then(Value::as_str)
11136                .is_some_and(|value| allowed.iter().any(|candidate| candidate == value))
11137        })
11138    }
11139
11140    /// Apply one host-authorized local security purge. The host MUST gate the
11141    /// corresponding subscriptions first; this primitive owns local SQLite
11142    /// cleanup and never grants/revokes server authority by itself.
11143    pub fn purge_local_data(
11144        &mut self,
11145        input: &LocalDataPurgeInput,
11146    ) -> Result<LocalDataPurgeResult, String> {
11147        let (targets, canonical_plan) = self.compile_local_data_purge(input)?;
11148        let meta_key = format!("localPurge:{}", input.purge_id);
11149        let applied_plan = self
11150            .conn
11151            .query_row(
11152                "SELECT value FROM _syncular_meta WHERE key = ?1",
11153                rusqlite::params![meta_key],
11154                |row| row.get::<_, String>(0),
11155            )
11156            .optional()
11157            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11158        if let Some(applied_plan) = applied_plan {
11159            if applied_plan != canonical_plan {
11160                return Err(format!(
11161                    "sync.invalid_request: local purge id {:?} was already used with a different plan",
11162                    input.purge_id
11163                ));
11164            }
11165            // A retry after a failed rewrite finishes it.
11166            self.erase_purged_pages()?;
11167            return Ok(LocalDataPurgeResult {
11168                already_applied: true,
11169                purged_rows: 0,
11170                dropped_commits: 0,
11171            });
11172        }
11173
11174        // RFC 0005 D7: the container file cannot be a purge target (its name is
11175        // not in the running schema), so purge drops it as a fixed step, before
11176        // any mirror row is touched and unconditionally.
11177        let _ = self.drop_previous_version();
11178
11179        let rollback = self.capture();
11180        self.begin_observation("syncular_local_purge")?;
11181        let applied = (|| -> Result<(ChangeAccumulator, LocalDataPurgeResult), String> {
11182            let row_ids = self.local_purge_base_row_ids(&targets)?;
11183            let retained_dropped = self.drop_failed_commits(|table, values| {
11184                targets.iter().any(|target| {
11185                    target.table == table && Self::local_purge_values_match(target, values)
11186                })
11187            })?;
11188            let doomed = self
11189                .outbox
11190                .iter()
11191                .filter(|commit| {
11192                    commit.ops.iter().any(|operation| {
11193                        let matching_targets = targets
11194                            .iter()
11195                            .filter(|target| target.table == operation.table)
11196                            .collect::<Vec<_>>();
11197                        if matching_targets.is_empty() {
11198                            return false;
11199                        }
11200                        if row_ids
11201                            .get(&operation.table)
11202                            .is_some_and(|ids| ids.contains(&operation.row_id))
11203                        {
11204                            return true;
11205                        }
11206                        operation.values.as_ref().is_some_and(|values| {
11207                            matching_targets
11208                                .iter()
11209                                .any(|target| Self::local_purge_values_match(target, values))
11210                        })
11211                    })
11212                })
11213                .cloned()
11214                .collect::<Vec<_>>();
11215            let doomed_ids = doomed
11216                .iter()
11217                .map(|commit| commit.client_commit_id.clone())
11218                .collect::<BTreeSet<_>>();
11219            let mut batch = ChangeAccumulator::default();
11220            for commit in &retained_dropped {
11221                batch.conflicts = true;
11222                batch.rejections = true;
11223                batch.outcomes = true;
11224                for operation in &commit.ops {
11225                    batch.table(&operation.table);
11226                }
11227            }
11228            let mut rejections = Vec::new();
11229            // §5.11 purge erasure: the journal keeps a dropped commit's identity and
11230            // codes, never its operations, so no purged value survives in it.
11231            for commit in &doomed {
11232                for operation in &commit.ops {
11233                    batch.table(&operation.table);
11234                    self.overlay_dirty.table(&operation.table);
11235                }
11236                let results = (0..commit.ops.len())
11237                    .map(|op_index| {
11238                        let rejection = RejectionRecord {
11239                            client_commit_id: commit.client_commit_id.clone(),
11240                            op_index: op_index as i32,
11241                            code: "client.local_data_purged".to_owned(),
11242                            message: "the commit was dropped by an application-authorized local data purge".to_owned(),
11243                            retryable: false,
11244                            details: None,
11245                            operation: None,
11246                        };
11247                        rejections.push(rejection.clone());
11248                        CommitOperationOutcome::Error { rejection }
11249                    })
11250                    .collect::<Vec<_>>();
11251                self.persist_commit_outcome(
11252                    &commit.client_commit_id,
11253                    CommitOutcomeStatus::Rejected,
11254                    &results,
11255                    None,
11256                )?;
11257                self.delete_outbox_persisted(&commit.client_commit_id)?;
11258            }
11259            if !doomed.is_empty() {
11260                self.outbox
11261                    .retain(|commit| !doomed_ids.contains(&commit.client_commit_id));
11262                self.rejections.extend(rejections);
11263                self.prune_commit_outcomes()?;
11264                batch.status = true;
11265                batch.rejections = true;
11266                batch.outcomes = true;
11267            }
11268
11269            let mut purged_rows = 0usize;
11270            for (table, ids) in &row_ids {
11271                if ids.is_empty() {
11272                    continue;
11273                }
11274                batch.table(table);
11275                self.overlay_dirty.table(table);
11276                for row_id in ids {
11277                    self.delete_base_row(table, row_id, i64::MAX)?;
11278                    purged_rows += 1;
11279                }
11280            }
11281            // §5.11 purge erasure: a journaled outcome with an operation or
11282            // conflict row the purge reaches keeps its identity, status, ids
11283            // and codes only.
11284            let touches = |table: &str, row_id: &str, values: Option<&Map<String, Value>>| {
11285                targets.iter().any(|target| {
11286                    target.table == table
11287                        && (row_ids.get(table).is_some_and(|ids| ids.contains(row_id))
11288                            || values.is_some_and(|values| {
11289                                Self::local_purge_values_match(target, values)
11290                            }))
11291                })
11292            };
11293            let entries = self
11294                .conn
11295                .prepare("SELECT client_commit_id, results_json, operations_json FROM _syncular_commit_outcomes")
11296                .and_then(|mut statement| {
11297                    statement
11298                        .query_map([], |row| {
11299                            Ok((
11300                                row.get::<_, String>(0)?,
11301                                row.get::<_, String>(1)?,
11302                                row.get::<_, Option<String>>(2)?,
11303                            ))
11304                        })?
11305                        .collect::<Result<Vec<_>, _>>()
11306                })
11307                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11308            for (client_commit_id, results_json, operations_json) in entries {
11309                let corrupt = |_| "sync.local_corrupt: persisted commit outcome is invalid".to_owned();
11310                let mut results: Vec<CommitOperationOutcome> =
11311                    serde_json::from_str(&results_json).map_err(corrupt)?;
11312                let operations: Vec<CommitOperation> = operations_json
11313                    .map(|json| serde_json::from_str(&json))
11314                    .transpose()
11315                    .map_err(corrupt)?
11316                    .unwrap_or_default();
11317                let named = operations
11318                    .iter()
11319                    .chain(results.iter().filter_map(|result| match result {
11320                        CommitOperationOutcome::Conflict { conflict } => conflict.operation.as_ref(),
11321                        CommitOperationOutcome::Error { rejection } => rejection.operation.as_ref(),
11322                        CommitOperationOutcome::Applied { .. } => None,
11323                    }))
11324                    .any(|operation| touches(&operation.table, &operation.row_id, operation.values.as_ref()))
11325                    || results.iter().any(|result| {
11326                        matches!(result, CommitOperationOutcome::Conflict { conflict }
11327                            if touches(&conflict.table, &conflict.row_id, Some(&conflict.server_row)))
11328                    });
11329                if !named {
11330                    continue;
11331                }
11332                for result in &mut results {
11333                    match result {
11334                        CommitOperationOutcome::Conflict { conflict } => {
11335                            conflict.server_row = Map::new();
11336                            conflict.conflict_columns.clear();
11337                            conflict.operation = None;
11338                        }
11339                        CommitOperationOutcome::Error { rejection } => {
11340                            rejection.details = None;
11341                            rejection.operation = None;
11342                        }
11343                        CommitOperationOutcome::Applied { .. } => {}
11344                    }
11345                }
11346                let results_json = serde_json::to_string(&results).map_err(|error| error.to_string())?;
11347                self.conn
11348                    .execute(
11349                        "UPDATE _syncular_commit_outcomes SET results_json = ?1, operations_json = NULL WHERE client_commit_id = ?2",
11350                        rusqlite::params![results_json, client_commit_id],
11351                    )
11352                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11353                batch.outcomes = true;
11354            }
11355            self.rebuild_overlay_if_dirty()?;
11356            self.delete_unreferenced_cached_blobs()?;
11357            self.conn
11358                .execute(
11359                    "INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
11360                    rusqlite::params![meta_key, canonical_plan],
11361                )
11362                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11363            Ok((
11364                batch,
11365                LocalDataPurgeResult {
11366                    already_applied: false,
11367                    purged_rows,
11368                    dropped_commits: doomed.len(),
11369                },
11370            ))
11371        })()
11372        .and_then(|(batch, result)| {
11373            self.finish_observation("syncular_local_purge", batch)
11374                .map(|()| result)
11375        });
11376        let result = match applied {
11377            Ok(result) => result,
11378            Err(error) => {
11379                self.rollback_observation("syncular_local_purge");
11380                self.restore(rollback);
11381                return Err(error);
11382            }
11383        };
11384        self.cancel_sync_round();
11385        self.erase_purged_pages()?;
11386        Ok(result)
11387    }
11388
11389    /// §5.11 purge erasure: SQLite leaves a deleted value in the freed pages of the
11390    /// database file, and the WAL keeps the frames written before the purge,
11391    /// until either is rewritten. VACUUM rebuilds the file from the remaining
11392    /// rows, and the checkpoint truncates the WAL.
11393    fn erase_purged_pages(&self) -> Result<(), String> {
11394        let erased = self
11395            .conn
11396            .execute_batch("PRAGMA secure_delete = ON; VACUUM;")
11397            .and_then(|()| {
11398                self.conn
11399                    .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))
11400            });
11401        let reset = self.conn.execute_batch("PRAGMA secure_delete = OFF");
11402        erased
11403            .and(reset)
11404            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
11405    }
11406
11407    /// Application-authorized recovery of the replicated projection. Unlike a
11408    /// local security purge, this retains the entire outbox, device identity,
11409    /// lease, outcomes, unit registrations, and protected bookkeeping.
11410    /// The projection reset, unit rewind, optimistic replay, and
11411    /// idempotency marker share one savepoint for interruption safety.
11412    pub fn rebootstrap_local_data(
11413        &mut self,
11414        input: &LocalDataRebootstrapInput,
11415    ) -> Result<LocalDataRebootstrapResult, String> {
11416        if input.rebootstrap_id.is_empty()
11417            || input.rebootstrap_id.len() > 128
11418            || !is_local_operation_code_like(&input.rebootstrap_id)
11419        {
11420            return Err(
11421                "sync.invalid_request: local rebootstrap rebootstrapId must be a 1–128 character code-like identifier"
11422                    .to_owned(),
11423            );
11424        }
11425        let meta_key = format!("localRebootstrap:{}", input.rebootstrap_id);
11426        if let Some(persisted) = meta_get(&self.conn, &meta_key).map_err(|_| {
11427            "sync.local_corrupt: persisted local rebootstrap receipt is unreadable".to_owned()
11428        })? {
11429            let (retained_commits, reset_subscriptions) =
11430                decode_local_rebootstrap_receipt(&persisted)?;
11431            return Ok(LocalDataRebootstrapResult {
11432                already_applied: true,
11433                retained_commits,
11434                reset_subscriptions,
11435            });
11436        }
11437        if self.stopped || self.schema_floor.is_some() {
11438            return Err(
11439                "sync.invalid_request: local rebootstrap cannot bypass an active schema-floor stop; update the application first"
11440                    .to_owned(),
11441            );
11442        }
11443
11444        let retained_commits = self.outbox.len();
11445        let reset_subscriptions = self.stored(load_units(&self.conn))?.len();
11446        let receipt = encode_local_rebootstrap_receipt(retained_commits, reset_subscriptions)?;
11447        let rollback = self.capture();
11448        self.begin_observation("syncular_local_rebootstrap")?;
11449        let mut batch = ChangeAccumulator::default();
11450        let applied = (|| -> Result<(), String> {
11451            self.run_schema_reset_observed(&mut batch, false)?;
11452            self.conn
11453                .execute(
11454                    "INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
11455                    rusqlite::params![meta_key, receipt],
11456                )
11457                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11458            self.sync_needed = true;
11459            self.sync_intent_queue.push_back(SyncIntent::Interactive);
11460            batch.status = true;
11461            Ok(())
11462        })()
11463        .and_then(|()| self.finish_observation("syncular_local_rebootstrap", batch));
11464        if let Err(error) = applied {
11465            self.rollback_observation("syncular_local_rebootstrap");
11466            self.restore(rollback);
11467            return Err(error);
11468        }
11469
11470        Ok(LocalDataRebootstrapResult {
11471            already_applied: false,
11472            retained_commits,
11473            reset_subscriptions,
11474        })
11475    }
11476
11477    // -- scope purge + doomed outbox (§3.3) ----------------------------------------
11478
11479    /// Delete base and visible rows owned by the effective scopes
11480    /// ([`local_scope_predicate`]); `Ok(false)` = no local scope-column
11481    /// mapping for a key (the fail-closed case). A SQLite failure is an error.
11482    fn purge_scope_rows(
11483        &mut self,
11484        table_name: &str,
11485        effective: &[(String, Vec<String>)],
11486        owners: &[Vec<(String, Vec<String>)>],
11487    ) -> Result<bool, String> {
11488        let Some(table) = self.schema.table(table_name) else {
11489            return Ok(false);
11490        };
11491        let Ok(predicate) = local_scope_predicate(table, effective, owners) else {
11492            return Ok(false);
11493        };
11494        let Some((predicate, params)) = predicate else {
11495            return Ok(true);
11496        };
11497        self.overlay_dirty.table(table_name);
11498        for source in [base_table(table_name), visible_table(table_name)] {
11499            self.conn
11500                .execute(
11501                    &format!("DELETE FROM {source} WHERE {predicate}"),
11502                    rusqlite::params_from_iter(&params),
11503                )
11504                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11505        }
11506        Ok(true)
11507    }
11508
11509    /// §3.3: drop pending commits whose upserts provably land in the
11510    /// revoked effective scopes and no other owner covers (row ownership)
11511    /// — whole-commit, never per-operation.
11512    fn drop_doomed_outbox(
11513        &mut self,
11514        table: &TableSchema,
11515        effective: &[(String, Vec<String>)],
11516        owners: &[Vec<(String, Vec<String>)>],
11517    ) -> Result<bool, String> {
11518        let doomed: Vec<_> =
11519            self.outbox
11520                .iter()
11521                .filter(|commit| {
11522                    commit.ops.iter().any(|op| {
11523                        op.upsert
11524                            && op.table == table.name
11525                            && op.values.as_ref().is_some_and(|values| {
11526                                owns_local_row(table, effective, owners, values)
11527                            })
11528                    })
11529                })
11530                .cloned()
11531                .collect();
11532        if doomed.is_empty() {
11533            return Ok(false);
11534        }
11535        let mut rejections = Vec::new();
11536        for commit in &doomed {
11537            let results = commit
11538                .ops
11539                .iter()
11540                .enumerate()
11541                .map(|(op_index, operation)| {
11542                    self.overlay_dirty.table(&operation.table);
11543                    let rejection = RejectionRecord {
11544                        client_commit_id: commit.client_commit_id.clone(),
11545                        op_index: op_index as i32,
11546                        code: "sync.scope_revoked".to_owned(),
11547                        message: "the commit was dropped because its effective scope was revoked"
11548                            .to_owned(),
11549                        retryable: false,
11550                        details: None,
11551                        operation: Some(CommitOperation::from(operation)),
11552                    };
11553                    rejections.push(rejection.clone());
11554                    CommitOperationOutcome::Error { rejection }
11555                })
11556                .collect::<Vec<_>>();
11557            self.persist_commit_outcome(
11558                &commit.client_commit_id,
11559                CommitOutcomeStatus::Rejected,
11560                &results,
11561                Some(&commit.ops),
11562            )?;
11563            self.delete_outbox_persisted(&commit.client_commit_id)?;
11564        }
11565        self.prune_commit_outcomes()?;
11566        let doomed_ids = doomed
11567            .iter()
11568            .map(|commit| commit.client_commit_id.as_str())
11569            .collect::<BTreeSet<_>>();
11570        self.outbox
11571            .retain(|commit| !doomed_ids.contains(commit.client_commit_id.as_str()));
11572        self.rejections.extend(rejections);
11573        Ok(true)
11574    }
11575
11576    // -- blobs (§5.9) ----------------------------------------------------------------
11577
11578    /// §5.9.7: hash bytes into the content address, cache them, queue the
11579    /// upload (flushed before the next push, B4). Returns the canonical
11580    /// BlobRef JSON `{blobId, byteLength, mediaType?}` for a `blob_ref`
11581    /// column value.
11582    pub fn upload_blob(
11583        &mut self,
11584        bytes: &[u8],
11585        media_type: Option<String>,
11586        name: Option<String>,
11587    ) -> Result<Value, String> {
11588        let blob_id = blob_id_for(bytes);
11589        let now = self.clock_now_ms();
11590        let tx = self
11591            .conn
11592            .transaction()
11593            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11594        tx.execute(
11595            "INSERT INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES (?,?,?,?,?)
11596             ON CONFLICT(blob_id) DO NOTHING",
11597            rusqlite::params![blob_id, bytes, bytes.len() as i64, media_type, now],
11598        )
11599        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11600        tx.execute(
11601            "INSERT OR IGNORE INTO _syncular_blob_uploads(blob_id, media_type, created_at_ms) VALUES (?,?,?)",
11602            rusqlite::params![blob_id, media_type, now],
11603        )
11604        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11605        tx.commit()
11606            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11607        self.enforce_blob_cache_cap();
11608        let mut obj = Map::new();
11609        obj.insert("blobId".to_owned(), Value::from(blob_id));
11610        obj.insert("byteLength".to_owned(), Value::from(bytes.len() as i64));
11611        if let Some(mt) = media_type {
11612            obj.insert("mediaType".to_owned(), Value::from(mt));
11613        }
11614        if let Some(n) = name {
11615            obj.insert("name".to_owned(), Value::from(n));
11616        }
11617        Ok(Value::Object(obj))
11618    }
11619
11620    /// §5.9.7: resolve an owned blob body without driver encoding. A
11621    /// content-addressed cache hit serves with no fetch (B1); a miss downloads
11622    /// (§5.9.5), verifies the address, caches the bytes, and returns them.
11623    pub fn fetch_blob_bytes(
11624        &mut self,
11625        transport: &mut dyn Transport,
11626        blob_id_or_ref: &str,
11627    ) -> Result<FetchedBlob, (String, String)> {
11628        let simple = |m: String| ("client.failed".to_owned(), m);
11629        // A BlobRef declares the size that caps the download (§5.9.5); a bare
11630        // blobId declares none.
11631        let (blob_id, byte_length) = if blob_id_or_ref.starts_with("sha256:") {
11632            (blob_id_or_ref.to_owned(), None)
11633        } else {
11634            let value: Value = serde_json::from_str(blob_id_or_ref)
11635                .map_err(|_| simple("blob ref is not JSON".to_owned()))?;
11636            (
11637                value
11638                    .get("blobId")
11639                    .and_then(Value::as_str)
11640                    .ok_or_else(|| simple("blob ref has no blobId".to_owned()))?
11641                    .to_owned(),
11642                value.get("byteLength").and_then(Value::as_u64),
11643            )
11644        };
11645        if let Some(cached) = self.get_cached_blob_bytes(&blob_id).map_err(simple)? {
11646            return Ok(cached);
11647        }
11648        if !self.transport_enabled {
11649            return Err(("sync.offline".into(), "transport gate is closed".into()));
11650        }
11651        // §5.9.5: propagate the server's blob.* code (blob.forbidden /
11652        // blob.not_found) verbatim so the harness can assert on it. The
11653        // authorized endpoint serves bytes inline OR (always-issue, presign
11654        // configured) a signed url the client fetches directly — no host auth,
11655        // no fall-through: failure => re-request (the caller's next fetch_blob_bytes).
11656        #[cfg(feature = "bench-internals")]
11657        let download_phase = self.benchmark_phases.start(Phase::BlobDownload);
11658        let bytes = match transport
11659            .blob_download(&blob_id, byte_length)
11660            .map_err(|e| (e.code, e.message))?
11661        {
11662            BlobDownload::Bytes(bytes) => bytes,
11663            BlobDownload::Url {
11664                url,
11665                url_expires_at_ms,
11666            } => {
11667                // §5.9.5: MUST NOT start a fetch at/past expiry.
11668                if url_expires_at_ms.is_some_and(|exp| exp <= self.clock_now_ms()) {
11669                    return Err((
11670                        "sync.segment_expired".to_owned(),
11671                        format!(
11672                            "blob url for {blob_id} expired before fetch — re-request mints a fresh url (§5.9.5)"
11673                        ),
11674                    ));
11675                }
11676                transport
11677                    .fetch_blob_url(&url, byte_length)
11678                    .map_err(|e| (e.code, e.message))?
11679            }
11680        };
11681        #[cfg(feature = "bench-internals")]
11682        drop(download_phase);
11683        #[cfg(feature = "bench-internals")]
11684        let validate_phase = self.benchmark_phases.start(Phase::BlobValidate);
11685        // §5.9.5 inherits §5.1: the declared size caps the body before the
11686        // content address is verified.
11687        if byte_length.is_some_and(|cap| bytes.len() as u64 > cap) {
11688            return Err((
11689                "sync.invalid_request".to_owned(),
11690                "blob exceeds its byteLength".to_owned(),
11691            ));
11692        }
11693        if blob_id_for(&bytes) != blob_id {
11694            return Err(simple(format!(
11695                "blob content address mismatch for {blob_id}"
11696            )));
11697        }
11698        #[cfg(feature = "bench-internals")]
11699        drop(validate_phase);
11700        #[cfg(feature = "bench-internals")]
11701        let insert_phase = self.benchmark_phases.start(Phase::BlobCacheInsert);
11702        let now = self.clock_now_ms();
11703        self.conn
11704            .execute(
11705                "INSERT OR IGNORE INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES (?,?,?,NULL,?)",
11706                rusqlite::params![blob_id, bytes, bytes.len() as i64, now],
11707            )
11708            .map_err(|e| simple(e.to_string()))?;
11709        self.conn
11710            .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))
11711            .map_err(|e| simple(e.to_string()))?;
11712        self.enforce_blob_cache_cap();
11713        #[cfg(feature = "bench-internals")]
11714        drop(insert_phase);
11715        self.get_cached_blob_bytes(&blob_id)
11716            .map_err(simple)?
11717            .ok_or_else(|| simple("blob cache write failed".to_owned()))
11718    }
11719
11720    fn get_cached_blob_bytes(&self, blob_id: &str) -> Result<Option<FetchedBlob>, String> {
11721        #[cfg(feature = "bench-internals")]
11722        let _phase = self.benchmark_phases.start(Phase::BlobCacheRead);
11723        let mut stmt = self
11724            .conn
11725            .prepare("SELECT bytes, byte_length, media_type FROM _syncular_blobs WHERE blob_id = ?")
11726            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11727        let mut rows = stmt
11728            .query(rusqlite::params![blob_id])
11729            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11730        if let Some(row) = rows
11731            .next()
11732            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
11733        {
11734            let bytes: Vec<u8> = row
11735                .get(0)
11736                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11737            let byte_length: i64 = row
11738                .get(1)
11739                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11740            let media_type: Option<String> = row
11741                .get(2)
11742                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11743            return Ok(Some(FetchedBlob {
11744                blob_id: blob_id.to_owned(),
11745                byte_length,
11746                bytes,
11747                media_type,
11748            }));
11749        }
11750        Ok(None)
11751    }
11752
11753    /// §5.9.7 B4: upload every queued blob before push.
11754    fn pending_blob_uploads(&self) -> Result<Vec<crate::round::PendingUpload>, TransportError> {
11755        let pending: Vec<(String, Option<String>)> = {
11756            let mut stmt = self
11757                .conn
11758                .prepare(
11759                    "SELECT blob_id, media_type FROM (
11760                      SELECT blob_id, media_type, created_at_ms FROM _syncular_blob_uploads
11761                      UNION ALL
11762                      SELECT DISTINCT r.blob_id, b.media_type, 9223372036854775807 AS created_at_ms
11763                      FROM _syncular_blob_commit_refs r LEFT JOIN _syncular_blobs b ON b.blob_id = r.blob_id
11764                      WHERE NOT EXISTS (SELECT 1 FROM _syncular_blob_uploads u WHERE u.blob_id = r.blob_id)
11765                    ) ORDER BY created_at_ms, blob_id",
11766                )
11767                .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
11768            let rows = stmt
11769                .query_map([], |row| {
11770                    Ok((row.get::<_, String>(0)?, row.get::<_, Option<String>>(1)?))
11771                })
11772                .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
11773            rows.collect::<Result<_, _>>()
11774                .map_err(|error| match error {
11775                    rusqlite::Error::InvalidColumnType(..)
11776                    | rusqlite::Error::FromSqlConversionFailure(..) => TransportError::new(
11777                        "sync.local_corrupt",
11778                        "Pending blob upload metadata is invalid",
11779                    ),
11780                    _ => TransportError::new("client.failed", error.to_string()),
11781                })?
11782        };
11783        let mut uploads = Vec::new();
11784        for (blob_id, media_type) in pending {
11785            let bytes = (|| {
11786                let stored: Option<(SqlValue, SqlValue)> = self
11787                    .conn
11788                    .query_row(
11789                        "SELECT bytes, byte_length FROM _syncular_blobs WHERE blob_id = ?",
11790                        rusqlite::params![blob_id],
11791                        |row| Ok((row.get(0)?, row.get(1)?)),
11792                    )
11793                    .optional()
11794                    .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
11795                let Some((SqlValue::Blob(bytes), SqlValue::Integer(byte_length))) = stored else {
11796                    return Err(TransportError::new(
11797                        "sync.local_corrupt",
11798                        "Pending blob upload body is missing or corrupt",
11799                    ));
11800                };
11801                if byte_length != bytes.len() as i64 || blob_id_for(&bytes) != blob_id {
11802                    return Err(TransportError::new(
11803                        "sync.local_corrupt",
11804                        "Pending blob upload body is missing or corrupt",
11805                    ));
11806                }
11807                Ok(bytes)
11808            })();
11809            let failed = bytes.is_err();
11810            uploads.push(crate::round::PendingUpload {
11811                id: blob_id,
11812                bytes,
11813                media_type,
11814            });
11815            if failed {
11816                break;
11817            }
11818        }
11819        Ok(uploads)
11820    }
11821
11822    fn visible_blob_ids_sql(&self) -> String {
11823        let mut selects = Vec::new();
11824        for table in &self.schema.tables {
11825            for column in table
11826                .columns
11827                .iter()
11828                .filter(|column| column.ty == ColumnType::BlobRef)
11829            {
11830                let identifier = quote_ident(&column.name);
11831                selects.push(format!(
11832                    "SELECT json_extract({identifier}, '$.blobId') AS blob_id FROM {} \
11833                     WHERE typeof({identifier}) = 'text' AND json_valid({identifier}) \
11834                     AND json_type({identifier}, '$.blobId') = 'text'",
11835                    quote_ident(&table.name),
11836                ));
11837            }
11838        }
11839        if selects.is_empty() {
11840            "SELECT NULL AS blob_id WHERE 0".to_owned()
11841        } else {
11842            selects.join(" UNION ")
11843        }
11844    }
11845
11846    /// §5.9.7 B1 size cap. Visible rows, pending commits, and
11847    /// staged uploads pin bodies directly; no derived refcount is stored.
11848    fn enforce_blob_cache_cap(&self) {
11849        let Some(max_bytes) = self.limits.blob_cache_max_bytes else {
11850            return;
11851        };
11852        let mut total: i64 = self
11853            .conn
11854            .query_row(
11855                "SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs",
11856                [],
11857                |row| row.get(0),
11858            )
11859            .unwrap_or(0);
11860        if total <= max_bytes {
11861            return;
11862        }
11863        let query = format!(
11864            "SELECT blob_id, byte_length FROM _syncular_blobs
11865             WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
11866               AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
11867               AND blob_id NOT IN ({})
11868             ORDER BY created_at_ms ASC, blob_id ASC",
11869            self.visible_blob_ids_sql(),
11870        );
11871        let candidates: Vec<(String, i64)> = {
11872            let Ok(mut stmt) = self.conn.prepare(&query) else {
11873                return;
11874            };
11875            let Ok(rows) = stmt.query_map([], |row| {
11876                Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?))
11877            }) else {
11878                return;
11879            };
11880            rows.filter_map(Result::ok).collect()
11881        };
11882        for (blob_id, byte_length) in candidates {
11883            if total <= max_bytes {
11884                break;
11885            }
11886            let _ = self.conn.execute(
11887                "DELETE FROM _syncular_blobs WHERE blob_id = ?",
11888                rusqlite::params![blob_id],
11889            );
11890            total -= byte_length;
11891        }
11892    }
11893
11894    fn delete_unreferenced_cached_blobs(&mut self) -> Result<(), String> {
11895        #[cfg(feature = "bench-internals")]
11896        let _phase = self.benchmark_phases.start(Phase::BlobRetention);
11897        if !self.schema_has_blobs() {
11898            return Ok(());
11899        }
11900        self.rebuild_overlay_if_dirty()?;
11901        self.conn
11902            .execute(
11903                &format!(
11904                    "DELETE FROM _syncular_blobs
11905                 WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
11906                   AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
11907                   AND blob_id NOT IN ({})",
11908                    self.visible_blob_ids_sql(),
11909                ),
11910                [],
11911            )
11912            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11913        Ok(())
11914    }
11915
11916    // -- local row storage ----------------------------------------------------------
11917
11918    /// The cached per-table primary-key upsert SQL for `full_table` (see
11919    /// the `insert_sql` field: built once, reused per row).
11920    fn insert_row_sql(&self, full_table: &str, table: &crate::schema::TableSchema) -> String {
11921        if let Some(sql) = self.insert_sql.borrow().get(full_table) {
11922            return sql.clone();
11923        }
11924        let mut columns: Vec<String> = table.columns.iter().map(|c| quote_ident(&c.name)).collect();
11925        columns.push(quote_ident("_syncular_version"));
11926        columns.extend(table.sealed_columns().iter().map(|name| quote_ident(name)));
11927        let placeholders: Vec<&str> = columns.iter().map(|_| "?").collect();
11928        let primary_key = quote_ident(&table.primary_key);
11929        let updates = columns
11930            .iter()
11931            .filter(|column| **column != primary_key)
11932            .map(|column| format!("{column}=excluded.{column}"))
11933            .collect::<Vec<_>>()
11934            .join(", ");
11935        // §4.5 version guard: server state lands in the base table only above
11936        // the stored version; the visible overlay takes every write.
11937        let guard = if full_table == base_table(&table.name) {
11938            format!(" WHERE excluded._syncular_version > {full_table}._syncular_version")
11939        } else {
11940            String::new()
11941        };
11942        let sql = format!(
11943            "INSERT INTO {full_table} ({}) VALUES ({}) ON CONFLICT ({primary_key}) DO UPDATE SET {updates}{guard}",
11944            columns.join(", "),
11945            placeholders.join(", ")
11946        );
11947        self.insert_sql
11948            .borrow_mut()
11949            .insert(full_table.to_owned(), sql.clone());
11950        sql
11951    }
11952
11953    /// §4.5: write a server row under the version guard; returns whether it
11954    /// landed. `sealed` is its §5.11 sealed state, `None` for an unsealed row.
11955    fn write_base_row(
11956        &self,
11957        table_name: &str,
11958        row: &Row,
11959        version: i64,
11960        sealed: Option<&SealedRow>,
11961    ) -> Result<bool, String> {
11962        let landed = self.write_row(&base_table(table_name), table_name, row, version, sealed)?;
11963        if landed {
11964            self.overlay_dirty.table(table_name);
11965        }
11966        Ok(landed)
11967    }
11968
11969    fn write_row(
11970        &self,
11971        full_table: &str,
11972        table_name: &str,
11973        row: &Row,
11974        version: i64,
11975        sealed: Option<&SealedRow>,
11976    ) -> Result<bool, String> {
11977        #[cfg(feature = "bench-internals")]
11978        let _phase = self.benchmark_phases.start(Phase::RowWrite);
11979        let table = self
11980            .schema
11981            .table(table_name)
11982            .ok_or_else(|| format!("unknown table {table_name:?}"))?;
11983        let sql = self.insert_row_sql(full_table, table);
11984        let mut stmt = self
11985            .conn
11986            .prepare_cached(&sql)
11987            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11988        let params = row
11989            .iter()
11990            .map(RowParam::Cell)
11991            .chain(std::iter::once(RowParam::Version(version)))
11992            .chain(
11993                [
11994                    sealed.map(|sealed| sealed.key_id.as_str()),
11995                    sealed.map(|sealed| sealed.envelopes.as_str()),
11996                ]
11997                .into_iter()
11998                .take(table.sealed_columns().len())
11999                .map(RowParam::Text),
12000            );
12001        stmt.execute(rusqlite::params_from_iter(params))
12002            .map(|changed| changed > 0)
12003            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
12004    }
12005
12006    /// §4.5: delete a server row whose stored version lies below `version`;
12007    /// returns whether a row left.
12008    fn delete_base_row(
12009        &self,
12010        table_name: &str,
12011        row_id: &str,
12012        version: i64,
12013    ) -> Result<bool, String> {
12014        let table = self
12015            .schema
12016            .table(table_name)
12017            .ok_or_else(|| format!("unknown table {table_name:?}"))?;
12018        let sql = format!(
12019            "DELETE FROM {} WHERE {} AND _syncular_version < ?2",
12020            base_table(table_name),
12021            row_id_predicate(table)
12022        );
12023        let mut stmt = self
12024            .conn
12025            .prepare_cached(&sql)
12026            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12027        let landed = stmt
12028            .execute(rusqlite::params![row_id, version])
12029            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
12030            > 0;
12031        if landed {
12032            self.overlay_dirty.table(table_name);
12033        }
12034        Ok(landed)
12035    }
12036
12037    fn has_acknowledged_rows(&self) -> Result<bool, String> {
12038        self.conn
12039            .query_row(
12040                "SELECT EXISTS(SELECT 1 FROM _syncular_acked_rows)",
12041                [],
12042                |row| row.get(0),
12043            )
12044            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
12045    }
12046
12047    fn acknowledged_operations(
12048        &self,
12049        rows: Option<&HashSet<(&str, &str)>>,
12050        tables: Option<&BTreeSet<&str>>,
12051    ) -> Result<Vec<OutboxOp>, String> {
12052        let keys = rows.map(|rows| serde_json::to_string(&rows).expect("row keys"));
12053        let mut statement = self
12054            .conn
12055            .prepare_cached(if keys.is_some() {
12056                ACKED_ROWS_BY_ROW
12057            } else {
12058                "SELECT op_json,intent_json FROM _syncular_acked_rows WHERE ?1 IS NULL AND (?2 IS NULL OR tbl IN(SELECT value FROM json_each(?2))) ORDER BY commit_seq,idx"
12059            })
12060            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12061        let rows = statement
12062            .query_map(
12063                rusqlite::params![
12064                    keys,
12065                    tables.map(|tables| serde_json::to_string(tables).expect("table keys"))
12066                ],
12067                |row| {
12068                    #[cfg(test)]
12069                    self.acknowledged_replay_count
12070                        .set(self.acknowledged_replay_count.get() + 1);
12071                    Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
12072                },
12073            )
12074            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12075        rows.map(|row| {
12076            let (op, intent) =
12077                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12078            let op: CommitOperation =
12079                serde_json::from_str(&op).map_err(|_| "sync.local_corrupt".to_owned())?;
12080            let table = self
12081                .schema
12082                .table(&op.table)
12083                .ok_or_else(|| "sync.unknown_table".to_owned())?;
12084            let values = if op.op == "upsert" && self.visible_row(table, &op.row_id)?.is_none() {
12085                Some(serde_json::from_str(&intent).map_err(|_| "sync.local_corrupt".to_owned())?)
12086            } else {
12087                op.values.clone()
12088            };
12089            Ok(OutboxOp {
12090                values,
12091                ..op.into()
12092            })
12093        })
12094        .collect()
12095    }
12096
12097    fn drop_acknowledged_rows(
12098        &self,
12099        matches: impl Fn(&str, &Map<String, Value>) -> bool,
12100    ) -> Result<Vec<OutboxCommit>, String> {
12101        let mut statement = self
12102            .conn
12103            .prepare("SELECT commit_id,tbl,intent_json,id FROM _syncular_acked_rows")
12104            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12105        let rows = statement
12106            .query_map([], |row| {
12107                Ok((
12108                    row.get::<_, String>(0)?,
12109                    row.get::<_, String>(1)?,
12110                    row.get::<_, String>(2)?,
12111                    row.get::<_, String>(3)?,
12112                ))
12113            })
12114            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12115        let mut doomed = HashSet::new();
12116        for row in rows {
12117            let (id, table, intent, row_id) =
12118                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12119            let values =
12120                serde_json::from_str(&intent).map_err(|_| "sync.local_corrupt".to_owned())?;
12121            let base_matches = self
12122                .schema
12123                .table(&table)
12124                .map(|schema| self.stored_row(schema, &row_id, true))
12125                .transpose()?
12126                .flatten()
12127                .is_some_and(|(base, _)| {
12128                    let schema = self.schema.table(&table).expect("acknowledged table");
12129                    let values = schema
12130                        .columns
12131                        .iter()
12132                        .zip(base.iter())
12133                        .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
12134                        .collect();
12135                    matches(&table, &values)
12136                });
12137            if matches(&table, &values) || base_matches {
12138                doomed.insert(id);
12139            }
12140        }
12141        let mut dropped = Vec::new();
12142        for id in doomed {
12143            let mut statement = self
12144                .conn
12145                .prepare("SELECT op_json FROM _syncular_acked_rows WHERE commit_id=?1 ORDER BY idx")
12146                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12147            let operations = statement
12148                .query_map([&id], |row| row.get::<_, String>(0))
12149                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12150            let mut ops = Vec::new();
12151            for operation in operations {
12152                let json = operation
12153                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12154                let op: CommitOperation =
12155                    serde_json::from_str(&json).map_err(|_| "sync.local_corrupt".to_owned())?;
12156                self.overlay_dirty.table(&op.table);
12157                ops.push(OutboxOp {
12158                    table: op.table,
12159                    row_id: op.row_id,
12160                    upsert: op.op == "upsert",
12161                    base_version: op.base_version,
12162                    values: op.values,
12163                    insert: op.insert,
12164                });
12165            }
12166            dropped.push(OutboxCommit {
12167                client_commit_id: id.clone(),
12168                ops,
12169            });
12170            self.conn
12171                .execute("DELETE FROM _syncular_acked_rows WHERE commit_id=?1", [id])
12172                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12173        }
12174        Ok(dropped)
12175    }
12176
12177    /// [`Self::rebuild_overlay`], skipped when neither the base tables nor
12178    /// the outbox changed since the last rebuild (the no-op sync round).
12179    fn rebuild_overlay_if_dirty(&mut self) -> Result<(), String> {
12180        if self.overlay_dirty.get() {
12181            self.rebuild_overlay()?;
12182        }
12183        Ok(())
12184    }
12185
12186    /// §7.1: local reads see outbox state applied optimistically — rebuild
12187    /// every visible table as (base server state) + (pending outbox replay
12188    /// on top). Optimistic rows carry version `-1`.
12189    fn rebuild_overlay(&mut self) -> Result<(), String> {
12190        #[cfg(feature = "bench-internals")]
12191        let _phase = self.benchmark_phases.start(Phase::OverlayRebuild);
12192        #[cfg(test)]
12193        self.overlay_rebuild_count
12194            .set(self.overlay_rebuild_count.get() + 1);
12195        let dirty = self.overlay_dirty.snapshot();
12196        let tables: Vec<_> = self
12197            .schema
12198            .tables
12199            .iter()
12200            .filter(|table| {
12201                dirty
12202                    .as_ref()
12203                    .is_none_or(|names| names.is_empty() || names.contains(&table.name))
12204            })
12205            .cloned()
12206            .collect();
12207        let names: BTreeSet<_> = tables.iter().map(|table| table.name.as_str()).collect();
12208        self.begin_observation("syncular_overlay")?;
12209        let rebuilt = (|| {
12210            for table in &tables {
12211                for index in &table.fts_indexes {
12212                    self.drop_fts_triggers(index)?;
12213                }
12214                let visible = visible_table(&table.name);
12215                let base = base_table(&table.name);
12216                self.conn
12217                    .execute_batch(&format!(
12218                        "DELETE FROM {visible}; INSERT INTO {visible} SELECT * FROM {base}"
12219                    ))
12220                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12221            }
12222            self.replay_overlay(&names, None)?;
12223            for table in &tables {
12224                for index in &table.fts_indexes {
12225                    self.rebuild_fts_projection(table, index)?;
12226                    self.create_fts_triggers(table, index)?;
12227                }
12228            }
12229            self.conn
12230                .execute_batch("RELEASE syncular_overlay")
12231                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12232            Ok(())
12233        })();
12234        if rebuilt.is_err() {
12235            self.conn
12236                .execute_batch("ROLLBACK TO syncular_overlay; RELEASE syncular_overlay")
12237                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12238        } else {
12239            self.overlay_dirty.set(false);
12240        }
12241        rebuilt
12242    }
12243
12244    /// §7.1 replay of acknowledged, retained failed and pending intent, in
12245    /// that order, for the operations of `tables` on `rows` (every row for
12246    /// `None`) over their current visible state.
12247    fn replay_overlay(
12248        &self,
12249        tables: &BTreeSet<&str>,
12250        rows: Option<&HashSet<(&str, &str)>>,
12251    ) -> Result<(), String> {
12252        let replays = |op: &&OutboxOp| {
12253            tables.contains(op.table.as_str())
12254                && rows.is_none_or(|rows| rows.contains(&(op.table.as_str(), op.row_id.as_str())))
12255        };
12256        self.apply_outbox_ops(&self.acknowledged_operations(rows, Some(tables))?)?;
12257        for (commit, initial) in self.failed_commits.iter() {
12258            for operation in commit.ops.iter().filter(replays) {
12259                let Some(table) = self.schema.table(&operation.table) else {
12260                    continue;
12261                };
12262                if operation.upsert
12263                    && self.visible_row(table, &operation.row_id)?.is_none()
12264                    && operation.presented_values(table).is_some_and(|values| {
12265                        table
12266                            .columns
12267                            .iter()
12268                            .all(|column| values.contains_key(&column.name))
12269                    })
12270                {
12271                    let key = serde_json::to_string(&(&operation.table, &operation.row_id))
12272                        .expect("row key serialization");
12273                    if let Some(Value::Object(values)) = initial.get(&key) {
12274                        self.apply_outbox_ops(&[OutboxOp {
12275                            values: Some(values.clone()),
12276                            ..operation.clone()
12277                        }])?;
12278                    }
12279                }
12280            }
12281            self.apply_outbox_ops(commit.ops.iter().filter(replays))?;
12282        }
12283        self.apply_outbox_ops(
12284            self.outbox
12285                .iter()
12286                .flat_map(|commit| &commit.ops)
12287                .filter(replays),
12288        )
12289    }
12290
12291    /// §7.1 replay: every operation is applied over the current visible
12292    /// state. An operation that no longer applies (a server row now holds its
12293    /// unique value) stays pending and invisible until a later replay admits
12294    /// it or the server answers its push.
12295    fn apply_outbox_ops<'a>(
12296        &self,
12297        ops: impl IntoIterator<Item = &'a OutboxOp>,
12298    ) -> Result<(), String> {
12299        #[cfg(feature = "bench-internals")]
12300        let mut phase = self.benchmark_phases.start(Phase::PendingReplay);
12301        for op in ops {
12302            #[cfg(feature = "bench-internals")]
12303            if let Some(phase) = &mut phase {
12304                phase.unit();
12305            }
12306            if op.upsert {
12307                if let Some((table, values)) = self
12308                    .schema
12309                    .table(&op.table)
12310                    .filter(|table| table.indexes.iter().any(|index| index.unique))
12311                    .and_then(|table| Some((table, op.presented_values(table)?)))
12312                {
12313                    let mut intended: Map<String, Value> = self
12314                        .visible_row(table, &op.row_id)?
12315                        .map(|(row, _)| {
12316                            table
12317                                .columns
12318                                .iter()
12319                                .zip(&row)
12320                                .map(|(column, value)| {
12321                                    (column.name.clone(), column_value_to_json(value))
12322                                })
12323                                .collect()
12324                        })
12325                        .unwrap_or_default();
12326                    intended.extend(values.into_owned());
12327                    if !self.unique_conflicts(table, &intended, false)?.is_empty() {
12328                        continue;
12329                    }
12330                }
12331            }
12332            self.apply_outbox_op(op)?;
12333        }
12334        Ok(())
12335    }
12336
12337    fn apply_outbox_op(&self, op: &OutboxOp) -> Result<(), String> {
12338        let Some(table) = self.schema.table(&op.table) else {
12339            return Ok(());
12340        };
12341        let visible = visible_table(&table.name);
12342        if !op.upsert {
12343            let sql = format!("DELETE FROM {visible} WHERE {}", row_id_predicate(table));
12344            self.conn
12345                .prepare_cached(&sql)
12346                .and_then(|mut statement| statement.execute(rusqlite::params![op.row_id]))
12347                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12348            return Ok(());
12349        }
12350        // §7.1: an insert-intent operation that lacks a non-nullable column
12351        // cannot encode; it stays invisible until the send-time prepass drops
12352        // it as sync.outbox_incompatible (§7.4.4).
12353        let Some(values) = op.presented_values(table) else {
12354            return Ok(());
12355        };
12356        // §7.1: the overlay applies the operation's PRESENT columns
12357        // over the current local row. A partial operation over an
12358        // absent local row leaves it absent — the server answers it
12359        // with sync.row_deleted or sync.row_missing (§6.2). An
12360        // insert-intent operation presents every column.
12361        let full = table
12362            .columns
12363            .iter()
12364            .all(|column| values.contains_key(&column.name));
12365        let local = self.visible_row(table, &op.row_id)?;
12366        if local.is_none() && !full {
12367            return Ok(());
12368        }
12369        let mut row: Row = Vec::with_capacity(table.columns.len());
12370        for (index, column) in table.columns.iter().enumerate() {
12371            let incoming = match values.get(&column.name) {
12372                Some(raw) => json_to_column_value(
12373                    if raw.get("$bytes").is_some()
12374                        && table
12375                            .encrypted_columns
12376                            .iter()
12377                            .any(|encrypted| encrypted.index == index)
12378                    {
12379                        &table.wire_columns[index]
12380                    } else {
12381                        column
12382                    },
12383                    Some(raw),
12384                ),
12385                None => Ok(local
12386                    .as_ref()
12387                    .and_then(|(values, _)| values.get(index).cloned())
12388                    .flatten()),
12389            };
12390            row.push(incoming.map_err(|_| {
12391                "sync.local_corrupt: persisted optimistic row value is invalid".to_owned()
12392            })?);
12393        }
12394        let version = local.as_ref().map_or(-1, |(_, version)| *version);
12395        self.write_row(&visible, &table.name, &row, version, None)
12396            .map(|_| ())
12397            .map_err(|error| {
12398                if error.starts_with("UNIQUE constraint failed:") {
12399                    "sync.constraint_violation: local write violates a unique constraint".to_owned()
12400                } else {
12401                    error
12402                }
12403            })
12404    }
12405
12406    /// Whether the row is present in the overlay. Existence reads only the
12407    /// key: locked encrypted values stay opaque, and decoding the complete row
12408    /// would reject a valid plain write.
12409    fn visible_row_exists(&self, table: &TableSchema, row_id: &str) -> Result<bool, String> {
12410        self.conn
12411            .query_row(
12412                &format!(
12413                    "SELECT EXISTS(SELECT 1 FROM {} WHERE {})",
12414                    visible_table(&table.name),
12415                    row_id_predicate(table)
12416                ),
12417                rusqlite::params![row_id],
12418                |row| row.get(0),
12419            )
12420            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
12421    }
12422
12423    /// §5.11: a sealed row is read-only; a write could otherwise send NULL
12424    /// over ciphertext this device cannot see.
12425    fn refuse_sealed_row(&self, table: &TableSchema, row_id: &str) -> Result<(), ClientError> {
12426        if !table.has_encrypted_columns() {
12427            return Ok(());
12428        }
12429        let key_id: Option<String> = self
12430            .conn
12431            .query_row(
12432                &format!(
12433                    "SELECT \"_sync_sealed_key\" FROM {} WHERE {}",
12434                    visible_table(&table.name),
12435                    row_id_predicate(table)
12436                ),
12437                rusqlite::params![row_id],
12438                |row| row.get(0),
12439            )
12440            .optional()
12441            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
12442            .flatten();
12443        match key_id {
12444            None => Ok(()),
12445            Some(key_id) => Err(ClientError {
12446                code: "client.encryption_key_missing".to_owned(),
12447                message: "the row is sealed under a key this device does not hold".to_owned(),
12448                details: Some(json!({ "table": table.name, "keyId": key_id })),
12449                retryable: false,
12450            }),
12451        }
12452    }
12453
12454    /// §5.11 reveal: decrypt every sealed base row whose envelope keys the
12455    /// keyring now holds into its columns and clear both reserved columns,
12456    /// one change batch per table. `client.decrypt_failed` fails the caller.
12457    fn reveal_sealed_rows(&mut self) -> Result<(), TransportError> {
12458        let storage = |message: String| TransportError::new("storage.failed", message);
12459        let tables: Vec<TableSchema> = self
12460            .schema
12461            .tables
12462            .iter()
12463            .filter(|table| table.has_encrypted_columns())
12464            .cloned()
12465            .collect();
12466        for table in &tables {
12467            let base = base_table(&table.name);
12468            let pk = quote_ident(&table.primary_key);
12469            let sealed: Vec<(SqlValue, i64, String)> = self
12470                .conn
12471                .prepare_cached(&format!(
12472                    "SELECT {pk}, \"_syncular_version\", \"_sync_sealed\" FROM {base} WHERE \"_sync_sealed_key\" IS NOT NULL"
12473                ))
12474                .and_then(|mut statement| {
12475                    statement
12476                        .query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))?
12477                        .collect()
12478                })
12479                .map_err(|error| storage(Self::sqlite_failure(&self.storage_failure, error)))?;
12480            let mut revealed = Vec::new();
12481            for (key, version, envelopes) in sealed {
12482                if let Some(row) =
12483                    crate::values::reveal_sealed_row(table, &envelopes, &self.encryption)?
12484                {
12485                    revealed.push((key, version, row));
12486                }
12487            }
12488            if revealed.is_empty() {
12489                continue;
12490            }
12491            self.begin_observation("syncular_reveal").map_err(storage)?;
12492            let outcome = (|| {
12493                let assignments = table
12494                    .encrypted_columns
12495                    .iter()
12496                    .map(|enc| format!("{}=?", quote_ident(&table.columns[enc.index].name)))
12497                    .collect::<Vec<_>>()
12498                    .join(",");
12499                let mut statement = self
12500                    .conn
12501                    .prepare_cached(&format!(
12502                        "UPDATE {base} SET {assignments},\"_sync_sealed_key\"=NULL,\"_sync_sealed\"=NULL WHERE {pk}=? AND \"_syncular_version\"=?"
12503                    ))
12504                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12505                for (key, version, row) in &revealed {
12506                    let mut params = table
12507                        .encrypted_columns
12508                        .iter()
12509                        .map(|enc| owned_sql_value(RowParam::Cell(&row[enc.index])))
12510                        .collect::<Result<Vec<_>, _>>()?;
12511                    params.push(key.clone());
12512                    params.push(SqlValue::Integer(*version));
12513                    statement
12514                        .execute(rusqlite::params_from_iter(params))
12515                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12516                }
12517                drop(statement);
12518                self.overlay_dirty.table(&table.name);
12519                self.rebuild_overlay()?;
12520                let mut batch = ChangeAccumulator::default();
12521                batch.table(&table.name);
12522                self.finish_observation("syncular_reveal", batch)
12523            })();
12524            if outcome.is_err() {
12525                self.rollback_observation("syncular_reveal");
12526            }
12527            outcome.map_err(storage)?;
12528        }
12529        Ok(())
12530    }
12531
12532    /// The visible (optimistic) row's values plus version, or `None` when the
12533    /// row is absent from the overlay.
12534    fn visible_row(&self, table: &TableSchema, row_id: &str) -> Result<Option<(Row, i64)>, String> {
12535        self.stored_row(table, row_id, false)
12536    }
12537
12538    fn stored_row(
12539        &self,
12540        table: &TableSchema,
12541        row_id: &str,
12542        base: bool,
12543    ) -> Result<Option<(Row, i64)>, String> {
12544        let sql = format!(
12545            "SELECT * FROM {} WHERE {}",
12546            if base {
12547                base_table(&table.name)
12548            } else {
12549                visible_table(&table.name)
12550            },
12551            row_id_predicate(table)
12552        );
12553        let mut stmt = self
12554            .conn
12555            .prepare_cached(&sql)
12556            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12557        let mut rows = stmt
12558            .query(rusqlite::params![row_id])
12559            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12560        let Some(row) = rows
12561            .next()
12562            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
12563        else {
12564            return Ok(None);
12565        };
12566        let mut values = Vec::with_capacity(table.columns.len());
12567        for (index, column) in table.columns.iter().enumerate() {
12568            let raw = row
12569                .get_ref(index)
12570                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12571            if let rusqlite::types::ValueRef::Blob(bytes) = raw {
12572                if table
12573                    .encrypted_columns
12574                    .iter()
12575                    .any(|encrypted| encrypted.index == index)
12576                {
12577                    values.push(Some(ColumnValue::Bytes(bytes.to_vec())));
12578                    continue;
12579                }
12580            }
12581            let json = sql_ref_to_json(column, raw);
12582            values.push(json_to_column_value(column, Some(&json)).map_err(|_| {
12583                "sync.local_corrupt: persisted local row value is invalid".to_owned()
12584            })?);
12585        }
12586        let version: i64 = row
12587            .get(table.columns.len())
12588            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12589        Ok(Some((values, version)))
12590    }
12591
12592    // -- realtime (§8) ---------------------------------------------------------------
12593
12594    /// §8.8: the configured realtime policy.
12595    #[must_use]
12596    pub fn realtime_policy(&self) -> RealtimePolicy {
12597        self.realtime_policy
12598    }
12599
12600    /// §8.8: the explicit realtime availability state.
12601    #[must_use]
12602    pub fn realtime_state(&self) -> RealtimeState {
12603        self.realtime_state
12604    }
12605
12606    /// §8.8: set the realtime policy. Refused while the socket is connected:
12607    /// a policy change never silently re-routes an ownership state.
12608    pub fn set_realtime_policy(&mut self, policy: RealtimePolicy) -> Result<(), String> {
12609        if self.realtime_state == RealtimeState::Connected && policy != self.realtime_policy {
12610            return Err(
12611                "sync.invalid_request: disconnect realtime before changing the realtime policy"
12612                    .to_owned(),
12613            );
12614        }
12615        self.realtime_policy = policy;
12616        match policy {
12617            RealtimePolicy::Off => self.set_realtime_state(RealtimeState::Disabled, None),
12618            RealtimePolicy::Required | RealtimePolicy::Optional
12619                if self.realtime_state == RealtimeState::Disabled =>
12620            {
12621                self.set_realtime_state(RealtimeState::Disconnected, None);
12622            }
12623            RealtimePolicy::Required | RealtimePolicy::Optional => {}
12624        }
12625        Ok(())
12626    }
12627
12628    /// Record an availability transition, keeping the reason code bounded and
12629    /// code-like (§7.6) and clearing it where it cannot apply.
12630    fn set_realtime_state(&mut self, state: RealtimeState, reason_code: Option<&str>) {
12631        self.realtime_state = state;
12632        self.realtime_reason_code = reason_code.map(Self::diagnostic_code);
12633        if matches!(state, RealtimeState::Connected | RealtimeState::Disabled) {
12634            self.realtime_reason_code = None;
12635            self.realtime_retry_delay_ms = None;
12636        }
12637    }
12638
12639    #[must_use]
12640    fn realtime_connected(&self) -> bool {
12641        self.realtime_state == RealtimeState::Connected
12642    }
12643
12644    pub fn connect_realtime(&mut self, transport: &mut dyn Transport) -> Result<(), String> {
12645        if !self.transport_enabled {
12646            return Err("sync.offline: transport gate is closed".into());
12647        }
12648        if self.realtime_policy == RealtimePolicy::Off {
12649            return Err(
12650                "sync.invalid_request: realtime is off; realtime connect is refused".to_owned(),
12651            );
12652        }
12653        if self.realtime_connected() {
12654            return Ok(());
12655        }
12656        self.realtime_state = RealtimeState::Connecting;
12657        match transport.realtime_connect_for_client(&self.client_id) {
12658            Ok(()) => {
12659                self.set_realtime_state(RealtimeState::Connected, None);
12660                Ok(())
12661            }
12662            Err(error) => {
12663                self.set_realtime_state(RealtimeState::Refused, Some(&error.code));
12664                Err(format!("{}: {}", error.code, error.message))
12665            }
12666        }
12667    }
12668
12669    pub fn disconnect_realtime(&mut self, transport: &mut dyn Transport) {
12670        self.cancel_sync_round();
12671        // A deliberately disconnected or policy-disabled client has nothing
12672        // to release; a `lost` socket still does (the host owns the handle).
12673        if self.realtime_state == RealtimeState::Disconnected
12674            || self.realtime_state == RealtimeState::Disabled
12675        {
12676            return;
12677        }
12678        let _ = transport.realtime_close();
12679        self.set_realtime_state(
12680            if self.realtime_policy == RealtimePolicy::Off {
12681                RealtimeState::Disabled
12682            } else {
12683                RealtimeState::Disconnected
12684            },
12685            None,
12686        );
12687        self.presence.clear(); // §8.6.1: presence is per-connection
12688    }
12689
12690    /// §8.8: the host observed the socket end without disconnecting it (a
12691    /// peer close, an I/O error, a failed send; the native transport reports
12692    /// these as `Inbound::Disconnected`). Releases the transport, moves a
12693    /// `Connected` state to `Lost` with the transport's code as the reason, and schedules
12694    /// one background retry intent so the host reconnects and catches up.
12695    pub fn realtime_lost(&mut self, transport: &mut dyn Transport, error: &TransportError) {
12696        if self.realtime_state != RealtimeState::Connected {
12697            return;
12698        }
12699        self.cancel_sync_round();
12700        let _ = transport.realtime_close();
12701        self.set_realtime_state(RealtimeState::Lost, Some(&error.code));
12702        self.realtime_retry_delay_ms = Some(self.schedule_background_retry());
12703        self.presence.clear();
12704    }
12705
12706    /// §8.6.2: publish (or clear, `doc: None`) this client's presence
12707    /// document for `scope_key`. Requires a live socket; the document is
12708    /// ephemeral (lost on disconnect). Authorization is the connection's
12709    /// registration (§8.6.3) — an unheld key is rejected loudly by the
12710    /// server with `presence.forbidden`.
12711    pub fn set_presence(
12712        &mut self,
12713        transport: &mut dyn Transport,
12714        scope_key: &str,
12715        doc: Option<&Value>,
12716    ) -> Result<(), String> {
12717        if !self.transport_enabled {
12718            return Err("sync.offline: transport gate is closed".into());
12719        }
12720        if !self.realtime_connected() {
12721            return Err("setPresence requires a connected realtime socket (§8.6)".to_string());
12722        }
12723        let text = encode_presence_publish(scope_key, doc);
12724        transport
12725            .realtime_send(&text)
12726            .map_err(|e| format!("{}: {}", e.code, e.message))
12727    }
12728
12729    /// §8.6: the peers currently present on a scope key (ephemeral).
12730    pub fn presence(&self, scope_key: &str) -> Vec<PresencePeer> {
12731        self.presence
12732            .get(scope_key)
12733            .map(|peers| peers.values().cloned().collect())
12734            .unwrap_or_default()
12735    }
12736
12737    /// §8.6 apply an inbound presence fanout to the local map.
12738    fn apply_presence(
12739        &mut self,
12740        scope_key: String,
12741        kind: Option<PresenceKind>,
12742        actor_id: Option<String>,
12743        client_id: Option<String>,
12744        doc: Option<Value>,
12745        error: Option<String>,
12746    ) {
12747        // A refused publish (§8.6.2) changes no peer; the native host surfaces
12748        // it as a `presence` event carrying the server's `error` code.
12749        if error.is_some() {
12750            return;
12751        }
12752        let (Some(kind), Some(actor_id), Some(client_id)) = (kind, actor_id, client_id) else {
12753            return;
12754        };
12755        let peer_key = format!("{actor_id} {client_id}");
12756        match kind {
12757            PresenceKind::Leave => {
12758                if let Some(peers) = self.presence.get_mut(&scope_key) {
12759                    peers.remove(&peer_key);
12760                    if peers.is_empty() {
12761                        self.presence.remove(&scope_key);
12762                    }
12763                }
12764            }
12765            _ => {
12766                let doc = match doc {
12767                    Some(Value::Object(_)) => doc.unwrap(),
12768                    _ => return,
12769                };
12770                self.presence.entry(scope_key).or_default().insert(
12771                    peer_key,
12772                    PresencePeer {
12773                        actor_id,
12774                        client_id,
12775                        doc,
12776                    },
12777                );
12778            }
12779        }
12780    }
12781
12782    /// Inbound JSON control message (§8.1). Unknown events are tolerated.
12783    pub fn on_realtime_text(&mut self, text: &str) {
12784        if self.stopped || !self.transport_enabled || self.security_preflight {
12785            return;
12786        }
12787        match parse_control(text) {
12788            Ok(ControlMessage::Hello { requires_sync, .. }) => {
12789                if requires_sync {
12790                    // §8.1: pull before trusting the socket for continuity.
12791                    self.set_sync_needed(true, true);
12792                }
12793            }
12794            Ok(ControlMessage::Presence {
12795                scope_key,
12796                kind,
12797                actor_id,
12798                client_id,
12799                doc,
12800                error,
12801                ..
12802            }) => {
12803                self.apply_presence(scope_key, kind, actor_id, client_id, doc, error);
12804            }
12805            Ok(ControlMessage::Wake { .. }) => {
12806                // §8.3: any wake-up means "run a pull soon", never data.
12807                self.set_sync_needed(true, true);
12808            }
12809            Ok(ControlMessage::Scopes { .. }) => {
12810                // §8.7: surfaced to the app; no sync-needed, no round.
12811                self.scopes_changed += 1;
12812            }
12813            _ => {}
12814        }
12815    }
12816
12817    /// Inbound binary delta: a complete SSP2 response (§8.2), applied like
12818    /// a pull response per section; an unapplied delta is a wake-up.
12819    /// Mailbox hosts send durable acknowledgements on their I/O executor.
12820    pub fn on_realtime_binary_queued(&mut self, bytes: &[u8]) -> Vec<String> {
12821        let mut transport = crate::round::DownloadResults::default();
12822        self.on_realtime_binary(&mut transport, bytes);
12823        transport.controls
12824    }
12825
12826    /// §8.2: a delta applies exactly like a live section and sets the replica
12827    /// cursor to its `throughSeq`; one at or below the replica cursor is a
12828    /// duplicate. A delta the client does not apply (no live unit, a status
12829    /// other than `active`, a decode or apply failure) is a drop and raises
12830    /// sync-needed.
12831    pub fn on_realtime_binary(&mut self, transport: &mut dyn Transport, bytes: &[u8]) {
12832        if self.stopped || !self.transport_enabled || self.security_preflight {
12833            return;
12834        }
12835        #[cfg(feature = "bench-internals")]
12836        let decode_phase = self.benchmark_phases.start(Phase::ResponseDecode);
12837        let message = match decode_message(bytes) {
12838            Ok(m) if m.msg_kind == MsgKind::Response => m,
12839            _ => {
12840                self.set_sync_needed(true, true);
12841                return;
12842            }
12843        };
12844        #[cfg(feature = "bench-internals")]
12845        drop(decode_phase);
12846        // §8.2: a delta's header means what a pull response's header means. A
12847        // schema floor (§1.6) is recorded, stops syncing and applies nothing.
12848        if let Some(Frame::RespHeader {
12849            required_schema_version: Some(required),
12850            latest_schema_version,
12851            ..
12852        }) = message.frames.first()
12853        {
12854            let floor = SchemaFloor {
12855                required_schema_version: Some(*required),
12856                latest_schema_version: *latest_schema_version,
12857            };
12858            if self.set_schema_floor(Some(floor)).is_err() {
12859                self.set_sync_needed(true, true);
12860            }
12861            return;
12862        }
12863        let through = message.frames.iter().find_map(|frame| match frame {
12864            Frame::LiveEnd { through_seq, .. } => Some(*through_seq),
12865            _ => None,
12866        });
12867        let epoch = self.get_meta(LOG_EPOCH_KEY).ok().flatten();
12868        let mut dropped = !matches!(message.frames.first(), Some(Frame::RespHeader {
12869                reset_required: false,
12870                log_epoch,
12871                ..
12872            }) if Some(log_epoch) == epoch.as_ref());
12873        let mut applied: Option<i64> = None;
12874        dropped |= self.begin_coalesced_apply().is_err();
12875        let mut receivers: Option<Vec<UnitRecord>> = None;
12876        for frame in message.frames {
12877            if dropped {
12878                break;
12879            }
12880            match frame {
12881                Frame::LiveStart { status, .. } => {
12882                    let Ok(cursor) = self.stored(replica_cursor(&self.conn)) else {
12883                        dropped = true;
12884                        break;
12885                    };
12886                    if status != LiveStatus::Active || cursor < 0 {
12887                        dropped = true;
12888                    } else if !through.is_some_and(|through| through <= cursor) {
12889                        // A catching-up unit receives no delta (§8.2).
12890                        receivers = self.stored(load_units(&self.conn)).ok().map(|units| {
12891                            units
12892                                .into_iter()
12893                                .filter(|unit| {
12894                                    unit.status == UnitStatus::Active
12895                                        && unit.echo.is_some()
12896                                        && unit.cursor.is_none()
12897                                })
12898                                .collect()
12899                        });
12900                        dropped = receivers.is_none();
12901                    }
12902                }
12903                Frame::Commit {
12904                    commit_seq,
12905                    tables,
12906                    changes,
12907                    ..
12908                } => {
12909                    let Some(receivers) = receivers.clone() else {
12910                        continue;
12911                    };
12912                    dropped = self
12913                        .apply_live_commit(
12914                            &receivers,
12915                            &tables,
12916                            changes,
12917                            commit_seq,
12918                            AppliedServerCommitSource::Delta,
12919                        )
12920                        .is_err();
12921                }
12922                Frame::LiveEnd { through_seq, .. } if receivers.is_some() => {
12923                    dropped = self
12924                        .set_meta(REPLICA_CURSOR_KEY, &through_seq.to_string())
12925                        .is_err();
12926                    applied = Some(through_seq);
12927                }
12928                Frame::Error { .. } => dropped = true,
12929                _ => {}
12930            }
12931        }
12932        // §1.4 rule 4: the ack follows the COMMIT. A failed COMMIT restored
12933        // the replica cursor, so the delta counts as a drop.
12934        if self.finish_coalesced_apply().is_err() || dropped {
12935            applied = None;
12936            dropped = true;
12937        }
12938        if let Some(cursor) = applied {
12939            // §8.2 ack point: the delta's throughSeq.
12940            let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
12941            let _ = transport.realtime_send(&ack);
12942        }
12943        if dropped {
12944            self.set_sync_needed(true, true);
12945        }
12946    }
12947
12948    /// §8.2 ack point after a pull round on a live connection: the replica
12949    /// cursor while it is at least `0`.
12950    fn ack_after_pull(&mut self, transport: &mut dyn Transport) {
12951        if !self.realtime_connected() {
12952            return;
12953        }
12954        if let Ok(cursor @ 0..) = self.stored(replica_cursor(&self.conn)) {
12955            let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
12956            let _ = transport.realtime_send(&ack);
12957        }
12958    }
12959}
12960
12961#[cfg(test)]
12962mod previous_version_wiring_tests;
12963
12964#[cfg(test)]
12965mod identity_tests;
12966
12967#[cfg(test)]
12968mod insert_intent_tests;
12969
12970#[cfg(test)]
12971mod robustness_tests;
12972
12973#[cfg(test)]
12974mod range_image_tests;