Skip to main content

syncular_client/
client.rs

1//! The Syncular v2 Rust client core (SPEC.md client-behavior contract):
2//! rusqlite local storage, §3.2/§3.3 effective-scope persistence + purge,
3//! §4 pull/cursor/bootstrap (§4.7 resume, §5.6 segment application), §6
4//! push with outbox order, §7 optimistic apply / rollback / replay-on-top,
5//! §2.3 clientCommitId idempotency, §8 realtime client rules, §10 errors.
6//!
7//! Built from `SPEC.md` and the committed `ssp2` codec alone — no
8//! reference to the v1 Rust tree or the v2 TypeScript client.
9
10use crate::{ProgressObserver, ProgressPhase, ProgressState};
11use ssp2::encode_message;
12use std::borrow::Cow;
13#[cfg(test)]
14use std::cell::Cell;
15use std::cell::RefCell;
16use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
17use std::sync::Arc;
18
19use rusqlite::types::{ToSqlOutput, Value as SqlValue, ValueRef};
20use rusqlite::{Connection, OpenFlags, OptionalExtension};
21use serde::{Deserialize, Serialize};
22use serde_json::{json, Map, Value};
23use sha2::{Digest, Sha256};
24use ssp2::decode::WIRE_VERSION;
25use ssp2::model::{
26    FencedRecord, Frame, LiveStatus, MediaType, Message, MsgKind, Op, OpResult, PushStatus,
27    SubStatus,
28};
29use ssp2::primitives::RawJson;
30use ssp2::registration::canonical_scope_json;
31use ssp2::segment::{decode_rows_segment, Column, ColumnType, ColumnValue, Row, RowsSegment};
32use ssp2::{decode_message, encode_presence_publish, parse_control, ControlMessage, PresenceKind};
33
34use crate::api::{
35    AppliedServerCommit, AppliedServerCommitSource, ClientChangeBatch, ClientDiagnosticsHost,
36    ClientDiagnosticsLease, ClientDiagnosticsReplica, ClientDiagnosticsSchema,
37    ClientDiagnosticsSnapshot, ClientDiagnosticsStorage, ClientError, ClientLimits, CommandEffects,
38    CommitDelivery, CommitOperation, CommitOperationOutcome, CommitOutcome, CommitOutcomeQuery,
39    CommitOutcomeResolution, CommitOutcomeStatus, ConflictRecord, CoverageSnapshot,
40    CurrentBaseProof, CurrentBaseProofReason, DiagnosticLastChange, DiagnosticLastRound,
41    DiagnosticQueryFailure, DiagnosticRoundCounters, DiagnosticUnits, EchoRecord, FetchedBlob,
42    LeaseState, LocalDataPurgeInput, LocalDataPurgeResult, LocalDataPurgeTarget,
43    LocalDataRebootstrapInput, LocalDataRebootstrapResult, Mutation, PresencePeer,
44    PreviousVersionStatus, QueryOwner, QueryReadFailure, QueryRow, QuerySnapshot, QueryValue,
45    RejectionDetails, RejectionRecord, ResolveCommitOutcomeInput, RetainedCommitRow,
46    RetainedUniqueConflict, RowState, SchemaFloor, SnapshotRead, SyncIntent, SyncOutcome,
47    SyncReport, SyncStatusSnapshot, TableChange, UnitCatchup, UnitRecord, UnitScopes, UnitStatus,
48    WindowBase, WindowChange, WindowCoverage, WindowState, WindowUnitRef,
49    CLIENT_DIAGNOSTICS_VERSION, MAX_DIAGNOSTIC_QUERY_FAILURES,
50};
51use crate::api::{RealtimePolicy, RealtimeState, REALTIME_UNAVAILABLE_CODE};
52#[cfg(feature = "bench-internals")]
53use crate::bench::{Phase, Recorder};
54use crate::previous_version::{
55    capture_previous_version_from_replica, first_incompatibility,
56    reconcile_previous_version_at_boot, set_local_schema_descriptor,
57    sweep_previous_version_container, PendingCommitAudit, PreviousVersionContextConfig,
58    PreviousVersionLifecycle, PreviousVersionReadSpec, PreviousVersionSnapshot,
59};
60use crate::schema::{parse_schema_json, ClientSchema, FtsIndexSchema, TableSchema};
61use crate::transport::{BlobDownload, SegmentRequest, Transport, TransportError};
62use crate::values::{
63    bytes_to_hex, column_value_to_json, decode_row_bytes, encode_sparse_row_json, full_row_values,
64    is_canonical_bytes_envelope, json_to_column_value, json_to_scope_map, normalize_values_casing,
65    render_row_id_json, scope_map_to_json, SealedRow,
66};
67
68/// §4.2 default: rows and SQLite images, both applied in committed chunks.
69/// Bit 3 (signed URLs, §5.4) is
70/// added per transport capability at request-build time.
71const DEFAULT_ACCEPT: u8 = 0b0111;
72const ACCEPT_INLINE_ROWS: u8 = 1 << 0;
73const ACCEPT_EXTERNAL_ROWS: u8 = 1 << 1;
74const ACCEPT_SQLITE: u8 = 1 << 2;
75const ACCEPT_SIGNED_URLS: u8 = 1 << 3;
76const MAX_DIAGNOSTIC_DOMAINS: usize = 256;
77
78/// §10.1 `recommendedAction` carried by a server HTTP error body.
79fn recommended_action_of(details: &Option<Value>) -> Option<String> {
80    details
81        .as_ref()?
82        .get("recommendedAction")?
83        .as_str()
84        .map(str::to_owned)
85}
86
87/// §7.4.1 persisted local schema-version marker (`_syncular_meta` key).
88pub(crate) const LOCAL_SCHEMA_VERSION_KEY: &str = "localSchemaVersion";
89const LOCAL_REVISION_KEY: &str = "localRevision";
90/// §4.5 replica cursor; absent means `-1`.
91const REPLICA_CURSOR_KEY: &str = "replicaCursor";
92/// Hex of the last pull `LIVE_END.echoDigest` (§4.2); absent means empty.
93const ECHO_DIGEST_KEY: &str = "echoDigest";
94/// Hex of the registration digest of the list the server stores (§4.1).
95const REGISTERED_DIGEST_KEY: &str = "registrationDigest";
96/// §7.4.3: table name → generation, persisted with the declaration evidence.
97const TABLE_GENERATIONS_KEY: &str = "localTableGenerations";
98/// §7.1: the identity of the outbox sequence.
99const OUTBOX_ID_KEY: &str = "outboxId";
100const CLIENT_ID_KEY: &str = "clientId";
101const LEASE_STATE_KEY: &str = "leaseState";
102const SCHEMA_FLOOR_KEY: &str = "schemaFloor";
103pub(crate) const LOG_EPOCH_KEY: &str = "logEpoch";
104/// Persisted fail-closed quarantine marker: present while a security preflight
105/// is pending and `activateSecurity` has yet to run. Storing it in the replica
106/// keeps the gate with the data it protects, so a rebuilt host handle reopening
107/// the same file returns in preflight even though its in-memory state is fresh.
108const SECURITY_PREFLIGHT_PENDING_KEY: &str = "securityPreflightPending";
109const LOCAL_REBOOTSTRAP_RECEIPT_VERSION: u8 = 2;
110const MAX_JS_SAFE_INTEGER: u64 = 9_007_199_254_740_991;
111/// §7.4.4 client-local code: a pending outbox commit cannot re-encode under
112/// the new schema after a bump. Never a wire code (§10.3).
113const OUTBOX_INCOMPATIBLE_CODE: &str = "sync.outbox_incompatible";
114/// §5.11 client-local code: an encode at the push seam resolved no usable key
115/// id or named an unknown key. Never a wire code (§10.3).
116const ENCRYPT_FAILED_CODE: &str = "client.encrypt_failed";
117const INVALID_REQUEST_CODE: &str = "sync.invalid_request:";
118/// §7.4.4: static text for a persisted commit the current codec refuses.
119const LEGACY_VALUE_REFUSAL_MESSAGE: &str =
120    "the persisted commit carries values the current codec refuses";
121/// §7.4.4: bounded reason on every intrinsic value drop.
122const INVALID_STORED_VALUES_REASON: &str = "invalid_stored_values";
123
124#[derive(Debug, Serialize, Deserialize)]
125#[serde(rename_all = "camelCase", deny_unknown_fields)]
126struct PersistedLocalDataRebootstrapReceipt {
127    version: u8,
128    retained_commits: u64,
129    reset_subscriptions: u64,
130}
131
132fn invalid_local_rebootstrap_receipt() -> String {
133    "sync.local_corrupt: persisted local rebootstrap receipt is invalid".to_owned()
134}
135
136fn encode_local_rebootstrap_receipt(
137    retained_commits: usize,
138    reset_subscriptions: usize,
139) -> Result<String, String> {
140    let retained_commits =
141        u64::try_from(retained_commits).map_err(|_| invalid_local_rebootstrap_receipt())?;
142    let reset_subscriptions =
143        u64::try_from(reset_subscriptions).map_err(|_| invalid_local_rebootstrap_receipt())?;
144    if retained_commits > MAX_JS_SAFE_INTEGER || reset_subscriptions > MAX_JS_SAFE_INTEGER {
145        return Err(invalid_local_rebootstrap_receipt());
146    }
147    serde_json::to_string(&PersistedLocalDataRebootstrapReceipt {
148        version: LOCAL_REBOOTSTRAP_RECEIPT_VERSION,
149        retained_commits,
150        reset_subscriptions,
151    })
152    .map_err(|_| invalid_local_rebootstrap_receipt())
153}
154
155fn decode_local_rebootstrap_receipt(value: &str) -> Result<(usize, usize), String> {
156    // Pre-0.15.36 markers proved application but did not retain the receipt.
157    if value == "v1" {
158        return Ok((0, 0));
159    }
160    let receipt: PersistedLocalDataRebootstrapReceipt =
161        serde_json::from_str(value).map_err(|_| invalid_local_rebootstrap_receipt())?;
162    if receipt.version != LOCAL_REBOOTSTRAP_RECEIPT_VERSION
163        || receipt.retained_commits > MAX_JS_SAFE_INTEGER
164        || receipt.reset_subscriptions > MAX_JS_SAFE_INTEGER
165    {
166        return Err(invalid_local_rebootstrap_receipt());
167    }
168    Ok((
169        usize::try_from(receipt.retained_commits)
170            .map_err(|_| invalid_local_rebootstrap_receipt())?,
171        usize::try_from(receipt.reset_subscriptions)
172            .map_err(|_| invalid_local_rebootstrap_receipt())?,
173    ))
174}
175
176#[cfg(test)]
177mod observation_tests;
178
179/// Internal state for hosts that perform network I/O outside the client owner.
180#[doc(hidden)]
181pub struct CurrentBasePreparation {
182    client_commit_id: String,
183    sequence: i64,
184    units: Vec<UnitRecord>,
185    transport_epoch: u64,
186    registration_epoch: u64,
187    security_epoch: u64,
188}
189
190#[cfg(test)]
191mod units_tests;
192
193#[derive(Debug, Clone)]
194struct OutboxOp {
195    upsert: bool,
196    table: String,
197    row_id: String,
198    base_version: Option<i64>,
199    /// Schema-agnostic local form (§0): driver JSON values, encoded with
200    /// the current codec at send time. A key's presence IS the §6.1 sparse
201    /// presence set — both the overlay and the wire payload derive from it.
202    values: Option<Map<String, Value>>,
203    /// §7.1 insert intent: `mutate` recorded this upsert while the row was
204    /// absent from the visible replica. Replay and send-time encoding present
205    /// every current-schema column; a column `values` lacks is NULL.
206    insert: bool,
207}
208
209impl OutboxOp {
210    /// The values this upsert presents under the current `table` (§7.1). An
211    /// insert-intent operation presents every column, a missing one as NULL;
212    /// `None` when a missing column is non-nullable, which leaves the commit
213    /// unencodable (§7.4.4), or when the operation carries no values.
214    fn presented_values(&self, table: &TableSchema) -> Option<Cow<'_, Map<String, Value>>> {
215        let values = self.values.as_ref()?;
216        if !self.insert || table.columns.iter().all(|c| values.contains_key(&c.name)) {
217            return Some(Cow::Borrowed(values));
218        }
219        full_row_values(table, values.clone()).ok().map(Cow::Owned)
220    }
221}
222
223impl From<CommitOperation> for OutboxOp {
224    fn from(operation: CommitOperation) -> Self {
225        Self {
226            upsert: operation.op == "upsert",
227            table: operation.table,
228            row_id: operation.row_id,
229            base_version: operation.base_version,
230            values: operation.values,
231            insert: operation.insert,
232        }
233    }
234}
235
236/// §7.5 static authoring refusal; the table and column travel in details.
237fn invalid_authoring(message: &str, details: Value) -> ClientError {
238    ClientError {
239        code: "sync.invalid_request".to_owned(),
240        message: message.to_owned(),
241        details: Some(details),
242        retryable: false,
243    }
244}
245
246/// §7.1 chained base: `later` repeats `earlier`'s positive base on the same
247/// row and presents only columns `earlier` presents.
248fn chains_on(earlier: &OutboxOp, later: &OutboxOp) -> bool {
249    // Only an upsert carries values, and it always carries its primary key.
250    earlier.row_id == later.row_id
251        && earlier.table == later.table
252        && later.base_version.is_some_and(|base| base > 0)
253        && earlier.base_version == later.base_version
254        && later.values.as_ref().is_some_and(|values| {
255            values.keys().all(|column| {
256                earlier
257                    .values
258                    .as_ref()
259                    .is_some_and(|earlier| earlier.contains_key(column))
260            })
261        })
262}
263
264impl From<&OutboxOp> for CommitOperation {
265    fn from(operation: &OutboxOp) -> Self {
266        Self {
267            table: operation.table.clone(),
268            row_id: operation.row_id.clone(),
269            op: if operation.upsert { "upsert" } else { "delete" }.to_owned(),
270            base_version: operation.base_version,
271            values: operation.values.clone(),
272            insert: operation.insert,
273        }
274    }
275}
276
277#[derive(Debug, Clone)]
278struct OutboxCommit {
279    client_commit_id: String,
280    ops: Vec<OutboxOp>,
281}
282
283/// One commit's `(table, value keys, insert intent)` per upsert operation — the classification
284/// shape the §7.4.4 send-time drop and the RFC 0005 D6 audit share, so the two
285/// can never disagree about which commits cannot re-encode.
286fn commit_audit_operations(commit: &OutboxCommit) -> Vec<(&str, Vec<&str>, bool)> {
287    commit
288        .ops
289        .iter()
290        .filter(|operation| operation.upsert)
291        .map(|operation| {
292            (
293                operation.table.as_str(),
294                operation
295                    .values
296                    .as_ref()
297                    .map(|values| values.keys().map(String::as_str).collect())
298                    .unwrap_or_default(),
299                operation.insert,
300            )
301        })
302        .collect()
303}
304
305type FailedCommit = (OutboxCommit, Map<String, Value>);
306
307#[derive(Debug, Clone)]
308struct CompiledLocalDataPurgeTarget {
309    table: String,
310    selectors: Vec<(String, Vec<String>)>,
311}
312
313struct StoredCommitOutcomeRow {
314    sequence: i64,
315    client_commit_id: String,
316    status: String,
317    recorded_at_ms: i64,
318    results_json: String,
319    operations_json: Option<String>,
320    resolution: String,
321    resolved_at_ms: Option<i64>,
322    replacement_client_commit_id: Option<String>,
323}
324
325impl StoredCommitOutcomeRow {
326    fn from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result<Self> {
327        Ok(Self {
328            sequence: row.get(0)?,
329            client_commit_id: row.get(1)?,
330            status: row.get(2)?,
331            recorded_at_ms: row.get(3)?,
332            results_json: row.get(4)?,
333            operations_json: row.get(5)?,
334            resolution: row.get(6)?,
335            resolved_at_ms: row.get(7)?,
336            replacement_client_commit_id: row.get(8)?,
337        })
338    }
339}
340
341/// Section outcome distinguishing the §5.6 unit-local fail-closed
342/// path from a round-aborting failure (§1.4 rule 5).
343/// §4.4 a unit section's `SUB_START`.
344struct SectionStart {
345    id: String,
346    status: SubStatus,
347    reason_code: String,
348    bootstrap: bool,
349    fenced: Vec<FencedRecord>,
350}
351
352enum SectionError {
353    FailClosed,
354    /// §5.5: the server refused a segment with `sync.forbidden`.
355    Refused,
356    /// §5.3 client memory: the round deferred this image's body to a later
357    /// pull of the same resume token.
358    Deferred,
359    Abort(String, String),
360    Transport(TransportError),
361}
362
363#[derive(Debug)]
364pub(crate) struct RequestMeta {
365    pushed_ids: Vec<String>,
366    /// Hex of the registration digest this pull carried; the server stores
367    /// that list once the round completes (§4.1).
368    registration_digest: String,
369    pub(crate) accept: u8,
370    /// §6.1 splitBatch: outbox commits held back from THIS request because
371    /// the running operation count reached the push cap — the next round
372    /// pushes them (`sync_needed` stays set while any remain).
373    deferred_commits: usize,
374    /// §5.6 applied-segment record per unit: segment id → rows applied at
375    /// the unit's current cursor and token.
376    pub(crate) applied: BTreeMap<String, BTreeMap<String, i64>>,
377    /// §5.3 client memory: (unit, segment id) of the first image whose body
378    /// the round did not fetch because the batch already holds two images.
379    /// That unit's section stops there without persisting `SUB_END`.
380    pub(crate) deferred: BTreeSet<(String, String)>,
381}
382
383#[derive(Default)]
384struct ChangeAccumulator {
385    /// None means table-wide; Some is the exact scoped domain.
386    tables: BTreeMap<String, Option<BTreeSet<String>>>,
387    windows: BTreeMap<String, BTreeSet<String>>,
388    status: bool,
389    conflicts: bool,
390    rejections: bool,
391    outcomes: bool,
392}
393
394impl ChangeAccumulator {
395    fn table(&mut self, table: &str) {
396        self.tables.insert(table.to_owned(), None);
397    }
398
399    fn scope(&mut self, table: &str, key: String) {
400        match self.tables.get_mut(table) {
401            Some(None) => {}
402            Some(Some(keys)) => {
403                keys.insert(key);
404            }
405            None => {
406                self.tables
407                    .insert(table.to_owned(), Some(BTreeSet::from([key])));
408            }
409        }
410    }
411
412    fn window(&mut self, base_key: &str, value: &str) {
413        self.windows
414            .entry(base_key.to_owned())
415            .or_default()
416            .insert(value.to_owned());
417    }
418
419    fn touched(&self) -> bool {
420        !self.tables.is_empty()
421            || !self.windows.is_empty()
422            || self.status
423            || self.conflicts
424            || self.rejections
425            || self.outcomes
426    }
427}
428
429/// §6.1: the server caps total operations per request (reference default
430/// 500) and rejects the whole batch with `sync.too_many_operations`; the
431/// client "splits and retries". Splitting happens at build time: commits are
432/// included IN ORDER until the operation budget is spent, the rest wait for
433/// the next round.
434const PUSH_OPS_PER_REQUEST: usize = 500;
435const MAX_LOCAL_PURGE_TARGETS: usize = 64;
436const MAX_LOCAL_PURGE_SELECTORS: usize = 8;
437const MAX_LOCAL_PURGE_VALUES: usize = 128;
438const MAX_LOCAL_PURGE_VALUE_LENGTH: usize = 256;
439pub const SECURITY_PREFLIGHT_REQUIRED_CODE: &str = "client.security_preflight_required";
440
441/// Dirty visible tables; `None` means a whole-replica recovery/reset.
442struct OverlayDirty(RefCell<Option<BTreeSet<String>>>);
443
444impl OverlayDirty {
445    fn clean() -> Self {
446        Self(RefCell::new(Some(BTreeSet::new())))
447    }
448    fn get(&self) -> bool {
449        self.0
450            .borrow()
451            .as_ref()
452            .is_none_or(|tables| !tables.is_empty())
453    }
454    fn set(&self, dirty: bool) {
455        *self.0.borrow_mut() = if dirty { None } else { Some(BTreeSet::new()) };
456    }
457    fn table(&self, table: &str) {
458        if let Some(tables) = self.0.borrow_mut().as_mut() {
459            tables.insert(table.to_owned());
460        }
461    }
462    fn snapshot(&self) -> Option<BTreeSet<String>> {
463        self.0.borrow().clone()
464    }
465    fn restore(&self, tables: Option<BTreeSet<String>>) {
466        *self.0.borrow_mut() = tables;
467    }
468}
469
470/// A copy-on-write in-memory mirror: a [`Rollback`] shares it, and the first
471/// write after a capture copies it once.
472#[derive(Debug, Clone, Default)]
473struct Shared<T>(Arc<T>);
474
475impl<T> std::ops::Deref for Shared<T> {
476    type Target = T;
477    fn deref(&self) -> &T {
478        &self.0
479    }
480}
481
482impl<T: Clone> std::ops::DerefMut for Shared<T> {
483    fn deref_mut(&mut self) -> &mut T {
484        Arc::make_mut(&mut self.0)
485    }
486}
487
488impl<T> From<T> for Shared<T> {
489    fn from(value: T) -> Self {
490        Self(Arc::new(value))
491    }
492}
493
494/// The in-memory mirrors of durable state, captured before a local
495/// transaction and restored when it rolls back. Queues only grow inside a
496/// transaction, so their lengths suffice.
497#[derive(Debug, Clone)]
498struct Rollback {
499    outbox: Shared<Vec<Arc<OutboxCommit>>>,
500    failed_commits: Shared<Vec<FailedCommit>>,
501    conflicts: Shared<Vec<ConflictRecord>>,
502    rejections: Shared<Vec<RejectionRecord>>,
503    schema_floor: Option<SchemaFloor>,
504    lease_state: Option<LeaseState>,
505    stopped: bool,
506    upgrading: bool,
507    sync_needed: bool,
508    active_round: Option<uuid::Uuid>,
509    overlay_dirty: Option<BTreeSet<String>>,
510    change_queue: usize,
511    sync_intent_queue: usize,
512    last_change: Option<DiagnosticLastChange>,
513}
514
515pub struct SyncClient {
516    #[cfg(feature = "bench-internals")]
517    benchmark_phases: Recorder,
518    conn: Connection,
519    schema: ClientSchema,
520    client_id: String,
521    limits: ClientLimits,
522    outbox: Shared<Vec<Arc<OutboxCommit>>>,
523    failed_commits: Shared<Vec<FailedCommit>>,
524    retain_failed_commits: bool,
525    conflicts: Shared<Vec<ConflictRecord>>,
526    rejections: Shared<Vec<RejectionRecord>>,
527    schema_floor: Option<SchemaFloor>,
528    /// §7.3.5: the opaque auth-lease state (from LEASE frames + lease errors).
529    lease_state: Option<LeaseState>,
530    /// §1.6: the schema-floor response stops syncing until an upgrade.
531    stopped: bool,
532    /// §7.4.5: true while a schema-bump reset + first re-bootstrap is in flight.
533    upgrading: bool,
534    /// §8.4 coalesced sync-needed signal.
535    sync_needed: bool,
536    /// §8.8: how sync rounds treat the realtime binding, and its exact
537    /// availability state with the evidence behind a lost or refused one.
538    realtime_policy: RealtimePolicy,
539    realtime_state: RealtimeState,
540    realtime_reason_code: Option<String>,
541    realtime_retry_delay_ms: Option<u64>,
542    /// §8.6 presence: scopeKey → (`actorId clientId` peer key → peer).
543    presence: HashMap<String, HashMap<String, PresencePeer>>,
544    /// Client clock (epoch ms) for the §5.4 `urlExpiresAtMs` check and the
545    /// §7.8 drive deadline; the host may pin it and move the pinned value
546    /// (conformance runs on a virtual clock).
547    now_ms: Option<std::sync::Arc<std::sync::atomic::AtomicI64>>,
548    /// §5.11 client-side encryption keys (`keyId → key bytes`). Empty ⇒ E2EE
549    /// off. The encrypt/decrypt seam (`values.rs`) is compiled only under the
550    /// `e2ee` feature; without it, a schema with encrypted columns fails loud.
551    encryption: crate::values::EncryptionConfig,
552    /// Fail-closed host bootstrap gate. While set, command hosts permit only
553    /// status/lifecycle inspection and an exact authorized local purge.
554    security_preflight: bool,
555    security_epoch: u64,
556    transport_enabled: bool,
557    transport_epoch: u64,
558    /// Raised by every registration change; a current-base proof spanning
559    /// one reports missing coverage.
560    registration_epoch: u64,
561    /// RFC 0005 D2/D8: the previous-version capture config. `None` (default)
562    /// means the feature is off; the descriptor write and the container orphan
563    /// sweep still happen.
564    previous_version: Option<PreviousVersionContextConfig>,
565    /// Per-table primary-key upsert SQL, built once per (full table name) —
566    /// the row write path runs per row during bootstrap (§5.6), so the SQL
567    /// string (and, via `prepare_cached`, its compiled statement) is reused
568    /// instead of being rebuilt and re-prepared per row. Cleared on a §7.4.3
569    /// schema reset (the column lists may have changed).
570    insert_sql: RefCell<HashMap<String, String>>,
571    /// §7.1: reconcile only tables whose base/outbox changed. Whole-replica
572    /// reset and restart explicitly invalidate every table.
573    overlay_dirty: OverlayDirty,
574    // First SQLite failure of the current operation, before savepoint cleanup.
575    storage_failure: RefCell<Option<QueryReadFailure>>,
576    /// Test-only structural performance signal: response processing must not
577    /// turn a batch of acknowledgements into one full overlay rebuild each.
578    #[cfg(test)]
579    overlay_rebuild_count: Cell<usize>,
580    #[cfg(test)]
581    acknowledged_replay_count: Cell<usize>,
582    /// Test-only structural performance signal: outcome retention is enforced
583    /// once per response rather than once per acknowledgement.
584    #[cfg(test)]
585    outcome_prune_count: Cell<usize>,
586    /// Test-only pinned duration of every §5.3 image chunk, in ms.
587    #[cfg(test)]
588    image_chunk_ms: Cell<Option<f64>>,
589    /// Exact observer-transaction output drained by command/FFI hosts.
590    progress: ProgressObserver,
591    change_queue: VecDeque<ClientChangeBatch>,
592    sync_intent_queue: VecDeque<SyncIntent>,
593    /// Explicit exponential retry policy for transient transport failures.
594    retry_delay_ms: u64,
595    /// §7.7: the background retry the current round scheduled, if any.
596    round_retry_delay_ms: Option<u64>,
597    /// §5.3: a table's next image chunk size, carried from one range image
598    /// of the table to the next.
599    image_chunk_rows: HashMap<String, u32>,
600    /// A round whose response never completed (§1.5): the client keeps its
601    /// code and reports it on every following request until a round receives a
602    /// complete, decodable response. The server already recorded the round, so
603    /// nothing else can tell it the round failed on this side.
604    round_failure: Option<String>,
605    active_round: Option<uuid::Uuid>,
606    last_round: Option<DiagnosticLastRound>,
607    last_change: Option<DiagnosticLastChange>,
608    /// §7.6: latest failed owned snapshot read per owner id, oldest first.
609    query_failures: Vec<DiagnosticQueryFailure>,
610    /// SPEC §1.4 rule 4: the open coalesced apply transaction of a response.
611    coalesced: Option<CoalescedApply>,
612    /// A coalesced apply transaction of the current response failed; every
613    /// later observation of that response fails until it ends.
614    coalesced_failure: Option<String>,
615}
616
617/// SPEC §1.4 rule 4: the frames and blocks of one response share one local
618/// write transaction of at most `COALESCED_APPLY_ROWS` rows. The in-memory
619/// state captured at its `BEGIN` is restored when its `COMMIT` fails.
620struct CoalescedApply {
621    rows: usize,
622    rollback: Rollback,
623}
624
625const COALESCED_APPLY_ROWS: usize = 1024;
626
627pub(crate) fn quote_ident(name: &str) -> String {
628    format!("\"{}\"", name.replace('"', "\"\""))
629}
630/// §7.4.1 `_syncular_meta` helpers. Free functions so RFC 0005's
631/// previous-version module reads and writes the SAME records as the client
632/// instead of growing a parallel meta layer.
633pub(crate) fn meta_get(conn: &Connection, key: &str) -> rusqlite::Result<Option<String>> {
634    conn.query_row(
635        "SELECT value FROM _syncular_meta WHERE key = ?1",
636        rusqlite::params![key],
637        |row| row.get::<_, String>(0),
638    )
639    .optional()
640}
641
642pub(crate) fn meta_set(conn: &Connection, key: &str, value: &str) -> rusqlite::Result<()> {
643    conn.execute(
644        "INSERT OR REPLACE INTO _syncular_meta (key, value) VALUES (?1, ?2)",
645        rusqlite::params![key, value],
646    )
647    .map(|_| ())
648}
649
650pub(crate) fn meta_delete(conn: &Connection, key: &str) -> rusqlite::Result<()> {
651    conn.execute(
652        "DELETE FROM _syncular_meta WHERE key = ?1",
653        rusqlite::params![key],
654    )
655    .map(|_| ())
656}
657
658/// Seek by the existing key while retaining the exact legacy text match.
659fn row_id_predicate(table: &TableSchema) -> String {
660    let key = quote_ident(&table.primary_key);
661    let text_match = format!("CAST({key} AS TEXT) = ?1");
662    match table.columns[table.pk_index].ty {
663        ColumnType::String | ColumnType::Json => {
664            format!("{key} = ?1 AND {text_match}")
665        }
666        // Native columns have no declared SQLite affinity. Match the stored
667        // integer bind type explicitly. Unary + removes expression affinity so
668        // SQLite can seek the typeless key; the text check preserves exact IDs.
669        ColumnType::Integer | ColumnType::Boolean => {
670            format!("{key} = +CAST(?1 AS INTEGER) AND {text_match}")
671        }
672        // REAL-to-text conversion can round distinct values to the same text.
673        // Narrowing those matches by numeric equality would change behavior.
674        _ => text_match,
675    }
676}
677
678/// §3.3/§4.8/§5.6 row ownership: rows whose local scope columns match
679/// `effective` and that no entry of `owners` matches. `owners` holds the
680/// pair scopes of every owner of the table (§3.3 row ownership); a scoped delete never removes a row one of them still covers.
681/// Scope values bind by the column's declared type, as `row_id_predicate`
682/// does. `Ok(None)` matches nothing (an empty map or value list); `Err(())`
683/// is a missing local scope-column mapping (the fail-closed case).
684fn local_scope_predicate(
685    table: &TableSchema,
686    effective: &[(String, Vec<String>)],
687    owners: &[Vec<(String, Vec<String>)>],
688) -> Result<Option<(String, Vec<SqlValue>)>, ()> {
689    if effective.is_empty() || effective.iter().any(|(_, values)| values.is_empty()) {
690        return Ok(None);
691    }
692    let column = |variable: &str| {
693        let name = table.scope_column(variable).ok_or(())?;
694        let column = table
695            .columns
696            .iter()
697            .find(|column| column.name == name)
698            .ok_or(())?;
699        Ok((quote_ident(name), column.ty == ColumnType::Integer))
700    };
701    let mut clauses = Vec::new();
702    let mut params = Vec::new();
703    for (variable, values) in effective {
704        let (key, integer) = column(variable)?;
705        let holes = vec!["?"; values.len()].join(", ");
706        clauses.push(if integer {
707            params.extend(values.iter().cloned().map(SqlValue::Text));
708            let seek = vec!["+CAST(? AS INTEGER)"; values.len()].join(", ");
709            format!("{key} IN ({seek}) AND CAST({key} AS TEXT) IN ({holes})")
710        } else {
711            format!("{key} IN ({holes})")
712        });
713        params.extend(values.iter().cloned().map(SqlValue::Text));
714    }
715    // Owners bind as one JSON parameter per variable set, so the statement
716    // keeps its size at any owner count; owners of one variable bind the
717    // union of their values. An owner holding other values of a variable
718    // `effective` names owns none of its rows and drops out. Values compare
719    // as text, the exact semantics of the effective clauses.
720    let mut groups: BTreeMap<Vec<&str>, Vec<Vec<&[String]>>> = BTreeMap::new();
721    for owner in owners {
722        if owner.is_empty()
723            || owner.iter().any(|(variable, values)| {
724                values.is_empty()
725                    || effective.iter().any(|(name, wanted)| {
726                        name == variable && wanted.iter().all(|value| !values.contains(value))
727                    })
728            })
729        {
730            continue;
731        }
732        let mut owner: Vec<_> = owner.iter().collect();
733        owner.sort_by(|a, b| a.0.cmp(&b.0));
734        groups
735            .entry(
736                owner
737                    .iter()
738                    .map(|(variable, _)| variable.as_str())
739                    .collect(),
740            )
741            .or_default()
742            .push(owner.iter().map(|(_, values)| values.as_slice()).collect());
743    }
744    let within = |variable: &str, list: &str| {
745        let (key, integer) = column(variable)?;
746        let key = if integer {
747            format!("CAST({key} AS TEXT)")
748        } else {
749            key
750        };
751        Ok(format!("{key} IN (SELECT value FROM json_each({list}))"))
752    };
753    for (variables, tuples) in groups {
754        if let [variable] = variables[..] {
755            let values: BTreeSet<&String> = tuples.iter().flatten().copied().flatten().collect();
756            params.push(SqlValue::Text(
757                Value::from_iter(values.into_iter().cloned()).to_string(),
758            ));
759            clauses.push(format!("NOT {}", within(variable, "?")?));
760        } else {
761            params.push(SqlValue::Text(
762                serde_json::to_string(&tuples).expect("owner tuples serialize"),
763            ));
764            let matches = variables
765                .iter()
766                .enumerate()
767                .map(|(index, variable)| within(variable, &format!("o.value,'$[{index}]'")))
768                .collect::<Result<Vec<_>, ()>>()?;
769            clauses.push(format!(
770                "NOT EXISTS (SELECT 1 FROM json_each(?) o WHERE {})",
771                matches.join(" AND ")
772            ));
773        }
774    }
775    Ok(Some((clauses.join(" AND "), params)))
776}
777
778/// [`local_scope_predicate`] over decoded row values (retained rows).
779fn owns_local_row(
780    table: &TableSchema,
781    effective: &[(String, Vec<String>)],
782    owners: &[Vec<(String, Vec<String>)>],
783    values: &Map<String, Value>,
784) -> bool {
785    let matches = |scopes: &[(String, Vec<String>)]| {
786        !scopes.is_empty()
787            && scopes.iter().all(|(variable, allowed)| {
788                table
789                    .scope_column(variable)
790                    .and_then(|column| values.get(column))
791                    .filter(|value| !value.is_null())
792                    .is_some_and(|value| {
793                        allowed.contains(
794                            &value
795                                .as_str()
796                                .map_or_else(|| value.to_string(), str::to_owned),
797                        )
798                    })
799            })
800    };
801    matches(effective) && !owners.iter().any(|owner| matches(owner))
802}
803
804fn is_local_operation_code_like(value: &str) -> bool {
805    let bytes = value.as_bytes();
806    bytes.first().is_some_and(u8::is_ascii_alphanumeric)
807        && bytes
808            .iter()
809            .all(|byte| byte.is_ascii_alphanumeric() || matches!(*byte, b'.' | b'_' | b':' | b'-'))
810}
811
812fn base_table(name: &str) -> String {
813    quote_ident(&format!("_syncular_base_{name}"))
814}
815
816/// §4.8 deferred eviction record: (unit id, table, effective scope map).
817type PendingEvict = (String, String, Vec<(String, Vec<String>)>);
818
819/// §4.8: a stable key for a window base: variable + canonical fixed scopes.
820fn window_base_key(base: &WindowBase) -> String {
821    format!(
822        "{}\0{}",
823        base.variable,
824        ssp2::registration::canonical_unit_json(&scope_pairs(&base.fixed_scopes))
825    )
826}
827
828/// The unit of one window value: the fixed scopes plus the value.
829fn window_unit_scopes(base: &WindowBase, value: &str) -> UnitScopes {
830    let mut scopes = base.fixed_scopes.clone();
831    scopes.insert(base.variable.clone(), value.to_owned());
832    scopes
833}
834
835/// A unit's scope map as the codec and the digests take it.
836fn scope_pairs(scopes: &UnitScopes) -> Vec<(String, String)> {
837    let mut pairs: Vec<(String, String)> = scopes
838        .iter()
839        .map(|(variable, value)| (variable.clone(), value.clone()))
840        .collect();
841    pairs.sort_by(|a, b| a.0.encode_utf16().cmp(b.0.encode_utf16()));
842    pairs
843}
844
845/// §4.1: the unit covers every table declaring all of its variables.
846fn covers(table: &TableSchema, scopes: &UnitScopes) -> bool {
847    scopes
848        .keys()
849        .all(|variable| table.scope_column(variable).is_some())
850}
851
852/// §4.4: the effective scopes of the (unit, table) pair under `echo`: the
853/// scope map as one-element lists plus the fenced lists. `None` when the
854/// fence revoked the pair.
855fn pair_scopes(
856    scopes: &UnitScopes,
857    table: &str,
858    echo: Option<&[EchoRecord]>,
859) -> Option<Vec<(String, Vec<String>)>> {
860    let fenced = echo.and_then(|echo| echo.iter().find(|record| record.table == table));
861    if fenced.is_some_and(|record| record.status == "revoked") {
862        return None;
863    }
864    let mut effective: BTreeMap<String, Vec<String>> = fenced
865        .map(|record| record.scopes.clone())
866        .unwrap_or_default();
867    for (variable, value) in scopes {
868        effective.insert(variable.clone(), vec![value.clone()]);
869    }
870    Some(effective.into_iter().collect())
871}
872
873/// §3.3 row ownership: the pair scopes of every owner of `table` except
874/// `except_id`. An owner is active with a persisted echo and live or
875/// post-bootstrap, or bootstrapping with a segment of the table in its
876/// current bootstrap.
877fn unit_owners(
878    units: &[UnitRecord],
879    table: &TableSchema,
880    except_id: &str,
881) -> Vec<Vec<(String, Vec<String>)>> {
882    units
883        .iter()
884        .filter(|unit| {
885            unit.id != except_id
886                && unit.status == UnitStatus::Active
887                && unit.echo.is_some()
888                && (!unit.bootstrapping() || unit.seeded.contains(&table.name))
889                && covers(table, &unit.scopes)
890        })
891        .filter_map(|unit| pair_scopes(&unit.scopes, &table.name, unit.echo.as_deref()))
892        .collect()
893}
894
895/// §5.5 / §5.6: the unit's scope map, each value a one-element list.
896fn unit_scope_lists(scopes: &UnitScopes) -> Vec<(String, Vec<String>)> {
897    scopes
898        .iter()
899        .map(|(variable, value)| (variable.clone(), vec![value.clone()]))
900        .collect()
901}
902
903/// §7.1 acknowledged intent of the rows `?1` names (`[[table, id], ...]`).
904/// It joins through the (tbl, id) index: a scan per acknowledgement would
905/// make draining a batch quadratic.
906const ACKED_ROWS_BY_ROW: &str = "SELECT a.op_json,a.intent_json FROM json_each(?1) k JOIN _syncular_acked_rows a ON a.tbl=json_extract(k.value,'$[0]') AND a.id=json_extract(k.value,'$[1]') WHERE ?2 IS NULL OR a.tbl IN(SELECT value FROM json_each(?2)) ORDER BY a.commit_seq,a.idx";
907
908const UNIT_COLUMNS: &str =
909    "id, scopes, explicit, status, reason_code, cursor, bootstrap_state, seeded, echo";
910
911/// A unit row; a field that does not parse is a conversion failure, which
912/// both read paths classify as `sync.local_corrupt`.
913fn unit_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result<UnitRecord> {
914    fn json<T: serde::de::DeserializeOwned>(index: usize, raw: &str) -> rusqlite::Result<T> {
915        serde_json::from_str(raw).map_err(|error| {
916            rusqlite::Error::FromSqlConversionFailure(
917                index,
918                rusqlite::types::Type::Text,
919                Box::new(error),
920            )
921        })
922    }
923    Ok(UnitRecord {
924        id: row.get(0)?,
925        scopes: json(1, &row.get::<_, String>(1)?)?,
926        explicit: row.get::<_, i64>(2)? == 1,
927        status: json(
928            3,
929            &serde_json::to_string(&row.get::<_, String>(3)?).expect("text"),
930        )?,
931        reason_code: row.get(4)?,
932        cursor: row.get(5)?,
933        bootstrap_state: row.get(6)?,
934        seeded: json(7, &row.get::<_, String>(7)?)?,
935        echo: row
936            .get::<_, Option<String>>(8)?
937            .map(|echo| json::<Vec<EchoRecord>>(8, &echo))
938            .transpose()?
939            .map(|echo| {
940                if echo
941                    .iter()
942                    .all(|record| record.status == "active" || record.status == "revoked")
943                {
944                    Ok(echo)
945                } else {
946                    Err(rusqlite::Error::FromSqlConversionFailure(
947                        8,
948                        rusqlite::types::Type::Text,
949                        "echo status is neither active nor revoked".into(),
950                    ))
951                }
952            })
953            .transpose()?,
954    })
955}
956
957/// Every unit in registration order (§4.1).
958fn load_units(conn: &Connection) -> rusqlite::Result<Vec<UnitRecord>> {
959    conn.prepare_cached(&format!(
960        "SELECT {UNIT_COLUMNS} FROM _syncular_units ORDER BY rowid"
961    ))?
962    .query_map([], unit_from_row)?
963    .collect()
964}
965
966fn get_unit(conn: &Connection, id: &str) -> rusqlite::Result<Option<UnitRecord>> {
967    conn.query_row(
968        &format!("SELECT {UNIT_COLUMNS} FROM _syncular_units WHERE id = ?1"),
969        [id],
970        unit_from_row,
971    )
972    .optional()
973}
974
975/// Insert or update in place: an update keeps the registration order. The
976/// §5.6 applied-segment record ends when the cursor or the token changes.
977fn save_unit(conn: &Connection, unit: &UnitRecord) -> rusqlite::Result<()> {
978    conn.prepare_cached(&format!(
979        "INSERT INTO _syncular_units({UNIT_COLUMNS}) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9)
980         ON CONFLICT(id) DO UPDATE SET explicit=excluded.explicit, status=excluded.status,
981           reason_code=excluded.reason_code, cursor=excluded.cursor,
982           bootstrap_state=excluded.bootstrap_state, seeded=excluded.seeded, echo=excluded.echo,
983           applied=CASE WHEN cursor IS excluded.cursor
984             AND bootstrap_state IS excluded.bootstrap_state THEN applied END"
985    ))?
986    .execute(rusqlite::params![
987        unit.id,
988        serde_json::to_string(&unit.scopes).expect("scope map serializes"),
989        i64::from(unit.explicit),
990        unit.status.name(),
991        unit.reason_code,
992        unit.cursor,
993        unit.bootstrap_state,
994        serde_json::to_string(&unit.seeded).expect("table list serializes"),
995        unit.echo
996            .as_ref()
997            .map(|echo| serde_json::to_string(echo).expect("echo serializes")),
998    ])
999    .map(|_| ())
1000}
1001
1002/// §5.6 applied-segment record of unit `id`: segment id → rows applied at
1003/// its current cursor and token.
1004fn applied_rows(conn: &Connection, id: &str) -> rusqlite::Result<BTreeMap<String, i64>> {
1005    conn.query_row(
1006        "SELECT applied FROM _syncular_units WHERE id = ?1",
1007        [id],
1008        |row| row.get::<_, Option<String>>(0),
1009    )
1010    .optional()?
1011    .flatten()
1012    .map_or(Ok(BTreeMap::new()), |raw| {
1013        serde_json::from_str(&raw).map_err(|error| {
1014            rusqlite::Error::FromSqlConversionFailure(
1015                0,
1016                rusqlite::types::Type::Text,
1017                Box::new(error),
1018            )
1019        })
1020    })
1021}
1022
1023/// §5.6: `rows` rows of `segment_id` applied at unit `id`'s position. Runs
1024/// in the transaction that commits them.
1025fn record_applied_rows(
1026    conn: &Connection,
1027    id: &str,
1028    segment_id: &str,
1029    rows: i64,
1030) -> rusqlite::Result<()> {
1031    conn.prepare_cached(
1032        "UPDATE _syncular_units SET applied=json_patch(coalesce(applied,'{}'),?1) WHERE id=?2",
1033    )?
1034    .execute(rusqlite::params![
1035        serde_json::json!({ segment_id: rows }).to_string(),
1036        id
1037    ])
1038    .map(|_| ())
1039}
1040
1041/// §4.1 references: every window registry entry `(base, value)` holding a unit.
1042fn window_refs(conn: &Connection, unit_id: &str) -> rusqlite::Result<Vec<(String, String)>> {
1043    conn.prepare_cached("SELECT base, unit FROM _syncular_windows WHERE sub_id = ?1")?
1044        .query_map([unit_id], |row| Ok((row.get(0)?, row.get(1)?)))?
1045        .collect()
1046}
1047
1048/// §7.4.3 reset, §2.1 log reset and local rebootstrap: keep the unit
1049/// registrations and discard the synced state of every unit outside `keep`,
1050/// so the next round sends the list and fresh-bootstraps those units. The
1051/// replica cursor survives only while a kept unit is live. The caller owns
1052/// the transaction.
1053fn reset_units(conn: &Connection, keep: &BTreeSet<String>) -> rusqlite::Result<()> {
1054    conn.execute(
1055        "UPDATE _syncular_units SET status='active', reason_code=NULL, cursor=-1,
1056           bootstrap_state=NULL, seeded='[]', echo=NULL, applied=NULL
1057         WHERE id NOT IN (SELECT value FROM json_each(?1))",
1058        [serde_json::to_string(keep).expect("unit ids serialize")],
1059    )?;
1060    conn.execute_batch(
1061        "DELETE FROM _syncular_meta WHERE key IN ('echoDigest','registrationDigest');
1062         DELETE FROM _syncular_meta WHERE key='replicaCursor'
1063           AND NOT EXISTS (SELECT 1 FROM _syncular_units WHERE cursor IS NULL);",
1064    )
1065}
1066
1067/// §4.5: the replica cursor, `-1` while no unit is live.
1068fn replica_cursor(conn: &Connection) -> rusqlite::Result<i64> {
1069    meta_get(conn, REPLICA_CURSOR_KEY)?.map_or(Ok(-1), |raw| {
1070        raw.parse().map_err(|error| {
1071            rusqlite::Error::FromSqlConversionFailure(
1072                0,
1073                rusqlite::types::Type::Text,
1074                Box::new(error),
1075            )
1076        })
1077    })
1078}
1079
1080fn visible_table(name: &str) -> String {
1081    quote_ident(name)
1082}
1083
1084const FTS_SOURCE_ID_COLUMN: &str = "_syncular_source_id";
1085
1086/// §7.4.3: is a `sqlite_master` table name a synced table (visible or base),
1087/// i.e. NOT one of the durable bookkeeping tables the reset preserves?
1088fn is_synced_table_name(name: &str) -> bool {
1089    if name.starts_with("sqlite_") {
1090        return false;
1091    }
1092    if name.starts_with("_syncular_base_") {
1093        return true; // the base half of a synced table pair
1094    }
1095    // Bookkeeping: outbox, units, meta, blob cache/uploads.
1096    !name.starts_with("_syncular_")
1097}
1098
1099/// `"sha256:" + hex` of the bytes — the content address (§5.9.1).
1100fn blob_id_for(bytes: &[u8]) -> String {
1101    let digest = Sha256::digest(bytes);
1102    format!("sha256:{}", bytes_to_hex(&digest))
1103}
1104
1105/// One [`SyncClient::write_row`] bind parameter, borrowing the row-codec
1106/// value it wraps — strings/JSON/bytes bind as borrowed TEXT/BLOB (no copy
1107/// per row on the §5.6 bootstrap path), scalars bind owned.
1108enum RowParam<'a> {
1109    Cell(&'a Option<ColumnValue>),
1110    Version(i64),
1111    Text(Option<&'a str>),
1112}
1113
1114impl rusqlite::ToSql for RowParam<'_> {
1115    fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
1116        Ok(match self {
1117            RowParam::Version(v) => ToSqlOutput::Owned(SqlValue::Integer(*v)),
1118            RowParam::Text(text) => match text {
1119                Some(text) => ToSqlOutput::Borrowed(ValueRef::Text(text.as_bytes())),
1120                None => ToSqlOutput::Owned(SqlValue::Null),
1121            },
1122            RowParam::Cell(cell) => match cell {
1123                None => ToSqlOutput::Owned(SqlValue::Null),
1124                Some(ColumnValue::String(s)) => ToSqlOutput::Borrowed(ValueRef::Text(s.as_bytes())),
1125                Some(ColumnValue::Integer(i)) => ToSqlOutput::Owned(SqlValue::Integer(*i)),
1126                Some(ColumnValue::Float(f)) => ToSqlOutput::Owned(SqlValue::Real(*f)),
1127                Some(ColumnValue::Boolean(b)) => {
1128                    ToSqlOutput::Owned(SqlValue::Integer(i64::from(*b)))
1129                }
1130                Some(ColumnValue::Json(raw)) | Some(ColumnValue::BlobRef(raw)) => {
1131                    ToSqlOutput::Borrowed(ValueRef::Text(raw.0.as_bytes()))
1132                }
1133                // §5.10: crdt bytes store as BLOB, like bytes.
1134                Some(ColumnValue::Bytes(b)) | Some(ColumnValue::Crdt(b)) => {
1135                    ToSqlOutput::Borrowed(ValueRef::Blob(b))
1136                }
1137            },
1138        })
1139    }
1140}
1141
1142// Retain a typed primary key across the current import chunk without copying row payloads.
1143fn owned_sql_value(param: impl rusqlite::ToSql) -> Result<SqlValue, String> {
1144    match param.to_sql().map_err(|error| error.to_string())? {
1145        ToSqlOutput::Owned(value) => Ok(value),
1146        ToSqlOutput::Borrowed(value) => {
1147            SqlValue::try_from(value).map_err(|error| error.to_string())
1148        }
1149        _ => Err("sync.invalid_request: unsupported SQLite key parameter".into()),
1150    }
1151}
1152
1153/// §5.3 image cell → bind parameter, strict per the declared column type
1154/// (`boolean` from INTEGER 0/1, `json` from its raw TEXT, NULL only when
1155/// nullable). Mismatches are image-producer violations. Returns the cell
1156/// borrowed when the stored representation already matches what the row
1157/// codec would write, or the normalized scalar (boolean → 0/1, float from
1158/// INTEGER → REAL) otherwise — the local write is byte-identical to the
1159/// old convert-then-insert path without allocating per cell.
1160fn image_cell_param<'a>(column: &Column, value: ValueRef<'a>) -> Result<ToSqlOutput<'a>, String> {
1161    use ssp2::segment::ColumnType;
1162    let mismatch = || {
1163        Err(format!(
1164            "image column {:?} holds a value of the wrong type",
1165            column.name
1166        ))
1167    };
1168    match value {
1169        ValueRef::Null => {
1170            if !column.nullable {
1171                return Err(format!(
1172                    "image column {:?} is NULL but not nullable",
1173                    column.name
1174                ));
1175            }
1176            Ok(ToSqlOutput::Owned(SqlValue::Null))
1177        }
1178        ValueRef::Integer(i) => match column.ty {
1179            ColumnType::Integer => Ok(ToSqlOutput::Borrowed(value)),
1180            ColumnType::Boolean => Ok(ToSqlOutput::Owned(SqlValue::Integer(i64::from(i != 0)))),
1181            ColumnType::Float => Ok(ToSqlOutput::Owned(SqlValue::Real(i as f64))),
1182            _ => mismatch(),
1183        },
1184        ValueRef::Real(_) => match column.ty {
1185            ColumnType::Float => Ok(ToSqlOutput::Borrowed(value)),
1186            _ => mismatch(),
1187        },
1188        ValueRef::Text(t) => {
1189            std::str::from_utf8(t)
1190                .map_err(|_| format!("image column {:?} is not UTF-8", column.name))?;
1191            match column.ty {
1192                ColumnType::String | ColumnType::Json | ColumnType::BlobRef => {
1193                    Ok(ToSqlOutput::Borrowed(value))
1194                }
1195                _ => mismatch(),
1196            }
1197        }
1198        ValueRef::Blob(_) => match column.ty {
1199            // §5.10: a crdt column stores its opaque bytes as BLOB, like bytes.
1200            ColumnType::Bytes | ColumnType::Crdt => Ok(ToSqlOutput::Borrowed(value)),
1201            _ => mismatch(),
1202        },
1203    }
1204}
1205
1206pub(crate) fn sql_ref_to_json(column: &Column, value: rusqlite::types::ValueRef<'_>) -> Value {
1207    use rusqlite::types::ValueRef;
1208    match value {
1209        ValueRef::Null => Value::Null,
1210        ValueRef::Integer(i) => match column.ty {
1211            ssp2::segment::ColumnType::Boolean => Value::Bool(i != 0),
1212            ssp2::segment::ColumnType::Float => {
1213                serde_json::Number::from_f64(i as f64).map_or(Value::Null, Value::Number)
1214            }
1215            _ => Value::from(i),
1216        },
1217        ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
1218        ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
1219        ValueRef::Blob(b) => {
1220            let mut map = Map::new();
1221            map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
1222            Value::Object(map)
1223        }
1224    }
1225}
1226
1227/// Bind a driver JSON value form as a rusqlite parameter for [`SyncClient::query`].
1228/// Objects are accepted in lossless `{"$bytes": hex}` and
1229/// `{"$bigint": decimal}` envelope forms.
1230fn json_param_to_sql(value: &Value) -> Result<SqlValue, String> {
1231    Ok(match value {
1232        Value::Null => SqlValue::Null,
1233        Value::Bool(b) => SqlValue::Integer(i64::from(*b)),
1234        Value::Number(n) => {
1235            if let Some(i) = n.as_i64() {
1236                SqlValue::Integer(i)
1237            } else if let Some(f) = n.as_f64() {
1238                SqlValue::Real(f)
1239            } else {
1240                return Err(format!("query param number {n} is out of range"));
1241            }
1242        }
1243        Value::String(s) => SqlValue::Text(s.clone()),
1244        Value::Object(_) => {
1245            if let Some(hex) = value.get("$bytes").and_then(Value::as_str) {
1246                SqlValue::Blob(crate::values::hex_to_bytes(hex)?)
1247            } else if let Some(decimal) = value.get("$bigint").and_then(Value::as_str) {
1248                SqlValue::Integer(decimal.parse::<i64>().map_err(|_| {
1249                    format!("query bigint param {decimal:?} is outside SQLite's i64 range")
1250                })?)
1251            } else {
1252                return Err(
1253                    "query object param must be a {$bytes: hex} or {$bigint: decimal} value"
1254                        .to_owned(),
1255                );
1256            }
1257        }
1258        Value::Array(_) => return Err("query array params are not supported".to_owned()),
1259    })
1260}
1261
1262/// Map a rusqlite value with no schema column to consult (arbitrary query
1263/// output): integers/reals/text pass through by stored affinity, blobs ride
1264/// as `{"$bytes": hex}`. Distinct from [`sql_ref_to_json`], which uses the
1265/// schema column type to recover booleans/floats/json.
1266fn sql_ref_to_json_dynamic(value: rusqlite::types::ValueRef<'_>) -> Value {
1267    use rusqlite::types::ValueRef;
1268    match value {
1269        ValueRef::Null => Value::Null,
1270        ValueRef::Integer(i) => {
1271            // JSON/Tauri IPC cannot represent every SQLite i64 exactly. Keep
1272            // ordinary UI-sized integers ergonomic and envelope only values
1273            // beyond JavaScript's safe range.
1274            const MAX_SAFE_INTEGER: i64 = 9_007_199_254_740_991;
1275            if (-MAX_SAFE_INTEGER..=MAX_SAFE_INTEGER).contains(&i) {
1276                Value::from(i)
1277            } else {
1278                let mut map = Map::new();
1279                map.insert("$bigint".to_owned(), Value::from(i.to_string()));
1280                Value::Object(map)
1281            }
1282        }
1283        ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
1284        ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
1285        ValueRef::Blob(b) => {
1286            let mut map = Map::new();
1287            map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
1288            Value::Object(map)
1289        }
1290    }
1291}
1292
1293/// §7.5: SQLite failures classify by result code into [`QueryReadFailure`].
1294fn query_connection(
1295    conn: &Connection,
1296    sql: &str,
1297    params: &[QueryValue],
1298) -> Result<Vec<QueryRow>, QueryReadFailure> {
1299    let lowered_sql = crate::query_guard::lower_public_query_sql(sql);
1300    let mut stmt = crate::query_guard::prepare_read_only(conn, &lowered_sql)?;
1301    let bound: Vec<SqlValue> = params
1302        .iter()
1303        .map(json_param_to_sql)
1304        .collect::<Result<_, _>>()?;
1305    // §2.4: an unaliased physical version column stays engine-internal, so a
1306    // `SELECT *` row feeds straight back into mutation values. An explicit
1307    // alias (`_sync_version AS v`) returns it as ordinary query data. The
1308    // §5.11 sealed columns follow the same rule.
1309    let columns: Vec<(usize, String)> = stmt
1310        .column_names()
1311        .enumerate()
1312        .filter(|(_, name)| {
1313            !matches!(
1314                *name,
1315                "_syncular_version" | "_sync_sealed_key" | "_sync_sealed"
1316            )
1317        })
1318        .map(|(i, name)| (i, name.to_owned()))
1319        .collect();
1320    let bound_refs: Vec<&dyn rusqlite::ToSql> =
1321        bound.iter().map(|v| v as &dyn rusqlite::ToSql).collect();
1322    let mut sql_rows = stmt.query(bound_refs.as_slice())?;
1323    let mut out = Vec::new();
1324    while let Some(row) = sql_rows.next()? {
1325        let mut record = Map::new();
1326        for (i, name) in &columns {
1327            let value = row.get_ref(*i)?;
1328            record.insert(name.clone(), sql_ref_to_json_dynamic(value));
1329        }
1330        out.push(record);
1331    }
1332    Ok(out)
1333}
1334
1335fn persisted_window_state(
1336    conn: &Connection,
1337    base: &WindowBase,
1338) -> Result<WindowState, QueryReadFailure> {
1339    let mut stmt = conn.prepare(
1340        "SELECT windows.unit, units.status, units.cursor, units.bootstrap_state
1341           FROM _syncular_windows AS windows
1342           LEFT JOIN _syncular_units AS units ON units.id = windows.sub_id
1343          WHERE windows.base = ?1
1344          ORDER BY windows.unit ASC",
1345    )?;
1346    let rows = stmt.query_map(rusqlite::params![window_base_key(base)], |row| {
1347        Ok((
1348            row.get::<_, String>(0)?,
1349            row.get::<_, Option<String>>(1)?,
1350            row.get::<_, Option<i64>>(2)?,
1351            row.get::<_, Option<String>>(3)?,
1352        ))
1353    })?;
1354    let mut units = Vec::new();
1355    let mut pending = Vec::new();
1356    for row in rows {
1357        let (unit, status, cursor, bootstrap_state) = row?;
1358        // §4.8: complete iff registered, active and not bootstrapping.
1359        if status.as_deref() != Some("active")
1360            || cursor.is_some_and(|cursor| cursor < 0 || bootstrap_state.is_some())
1361        {
1362            pending.push(unit.clone());
1363        }
1364        units.push(unit);
1365    }
1366    Ok(WindowState { units, pending })
1367}
1368
1369/// One read-only statement in a §7.5 snapshot read.
1370#[derive(Debug, Clone, Copy)]
1371pub struct SnapshotStatement<'a> {
1372    pub sql: &'a str,
1373    pub params: &'a [Value],
1374}
1375
1376/// A §7.5 atomic read request: read-only statements, window coverage, unit
1377/// catch-up states, and commit deliveries, all resolved from one SQLite read
1378/// transaction and one revision.
1379#[derive(Debug, Clone, Default)]
1380pub struct SnapshotReadRequest<'a> {
1381    pub statements: Vec<SnapshotStatement<'a>>,
1382    pub coverage: &'a [WindowCoverage],
1383    pub units: Vec<String>,
1384    pub commit_ids: Vec<String>,
1385}
1386
1387#[cfg(test)]
1388thread_local! {
1389    /// Test-only rendezvous that runs between the statements of one snapshot
1390    /// read. A test uses it to commit on another connection mid-read and prove
1391    /// the read transaction's snapshot and revision stay stable.
1392    static SNAPSHOT_READ_HOOK: std::cell::RefCell<Option<Box<dyn FnOnce()>>> =
1393        std::cell::RefCell::new(None);
1394}
1395
1396#[cfg(test)]
1397fn run_snapshot_read_hook() {
1398    SNAPSHOT_READ_HOOK.with(|slot| {
1399        if let Some(hook) = slot.borrow_mut().take() {
1400            hook();
1401        }
1402    });
1403}
1404
1405/// §7.5: resolve a request over one SQLite read transaction. The revision,
1406/// every statement's rows, coverage, catch-up states, and deliveries are
1407/// captured under one `SAVEPOINT`; any statement's failure rolls the whole
1408/// read back and releases the connection.
1409fn snapshot_read_connection(
1410    conn: &Connection,
1411    request: &SnapshotReadRequest<'_>,
1412) -> Result<SnapshotRead, QueryReadFailure> {
1413    conn.execute_batch("SAVEPOINT syncular_snapshot_read")?;
1414    let corrupt = |message: &str| QueryReadFailure {
1415        code: Some("sync.local_corrupt"),
1416        sqlite_code: None,
1417        sqlite_message: None,
1418        rollback_failure: None,
1419        message: message.to_owned(),
1420    };
1421    let stored_failure = |error: rusqlite::Error, message: &str| match error {
1422        rusqlite::Error::Utf8Error(..)
1423        | rusqlite::Error::InvalidColumnType(..)
1424        | rusqlite::Error::FromSqlConversionFailure(..)
1425        | rusqlite::Error::IntegralValueOutOfRange(..) => corrupt(message),
1426        _ => QueryReadFailure::from(error),
1427    };
1428    let result: Result<SnapshotRead, QueryReadFailure> = (|| {
1429        // The durable revision as a canonical decimal `u64`; a missing marker
1430        // is the legacy zero, and any other non-canonical value is corrupt.
1431        let revision: Option<String> = conn
1432            .query_row(
1433                "SELECT value FROM _syncular_meta WHERE key = ?1",
1434                rusqlite::params![LOCAL_REVISION_KEY],
1435                |row| row.get(0),
1436            )
1437            .optional()
1438            .map_err(|error| stored_failure(error, "persisted local revision is invalid"))?;
1439        let revision = match revision {
1440            None => "0".to_owned(),
1441            Some(value)
1442                if value == "0"
1443                    || (matches!(value.as_bytes().first(), Some(b'1'..=b'9'))
1444                        && value.bytes().all(|b| b.is_ascii_digit())
1445                        && value.parse::<u64>().is_ok()) =>
1446            {
1447                value
1448            }
1449            Some(_) => return Err(corrupt("persisted local revision is invalid")),
1450        };
1451        let mut queries = Vec::with_capacity(request.statements.len());
1452        for statement in &request.statements {
1453            queries.push(query_connection(conn, statement.sql, statement.params)?);
1454            #[cfg(test)]
1455            run_snapshot_read_hook();
1456        }
1457        let mut pending = Vec::new();
1458        let mut missing = Vec::new();
1459        for requested in request.coverage {
1460            let base_key = window_base_key(&requested.base);
1461            let state = persisted_window_state(conn, &requested.base)?;
1462            for unit in BTreeSet::from_iter(requested.units.iter().cloned()) {
1463                let reference = WindowUnitRef {
1464                    base_key: base_key.clone(),
1465                    unit: unit.clone(),
1466                };
1467                if !state.units.iter().any(|held| held == &unit) {
1468                    missing.push(reference);
1469                } else if state.pending.iter().any(|held| held == &unit) {
1470                    pending.push(reference);
1471                }
1472            }
1473        }
1474        let mut units = Vec::with_capacity(request.units.len());
1475        for id in &request.units {
1476            let exists: bool = conn.query_row(
1477                "SELECT EXISTS(SELECT 1 FROM _syncular_units WHERE id = ?1)",
1478                [id],
1479                |row| row.get(0),
1480            )?;
1481            if !exists {
1482                units.push(UnitCatchup::Unknown { id: id.clone() });
1483                continue;
1484            }
1485            // Every persisted field is required to be well typed; a corrupt
1486            // record fails typed instead of inventing a state.
1487            let unit = get_unit(conn, id)
1488                .map_err(|error| stored_failure(error, "persisted unit state is invalid"))?
1489                .ok_or_else(|| corrupt("persisted unit state is invalid"))?;
1490            let cursor = match unit.cursor {
1491                Some(cursor) => cursor,
1492                None => replica_cursor(conn)
1493                    .map_err(|error| stored_failure(error, "persisted unit state is invalid"))?,
1494            };
1495            if cursor.unsigned_abs() > MAX_JS_SAFE_INTEGER {
1496                return Err(corrupt("persisted unit state is invalid"));
1497            }
1498            let active = unit.status == UnitStatus::Active;
1499            let pending = unit.bootstrapping();
1500            units.push(UnitCatchup::Known {
1501                id: id.clone(),
1502                status: unit.status,
1503                live: unit.cursor.is_none(),
1504                cursor,
1505                has_resume_token: unit.bootstrap_state.is_some(),
1506                bootstrap_complete: active && !pending,
1507                known_pending_pages: active && pending,
1508                reason_code: unit.reason_code,
1509            });
1510        }
1511        let mut deliveries = Vec::with_capacity(request.commit_ids.len());
1512        for id in &request.commit_ids {
1513            let pending: bool = conn.query_row(
1514                "SELECT EXISTS(SELECT 1 FROM _syncular_outbox WHERE commit_id = ?1)",
1515                rusqlite::params![id],
1516                |row| row.get(0),
1517            )?;
1518            if pending {
1519                deliveries.push(CommitDelivery::Pending {
1520                    client_commit_id: id.clone(),
1521                });
1522                continue;
1523            }
1524            let row = conn
1525                .query_row(
1526                    "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
1527                            resolution, resolved_at_ms, replacement_client_commit_id
1528                       FROM _syncular_commit_outcomes WHERE client_commit_id = ?1",
1529                    rusqlite::params![id],
1530                    StoredCommitOutcomeRow::from_row,
1531                )
1532                .optional()
1533                .map_err(|error| stored_failure(error, "persisted commit outcome is invalid"))?;
1534            deliveries.push(match row {
1535                Some(row) => CommitDelivery::Known {
1536                    client_commit_id: id.clone(),
1537                    outcome: SyncClient::outcome_from_row(row)
1538                        .map_err(|_| corrupt("persisted commit outcome is invalid"))?,
1539                },
1540                None => CommitDelivery::Unknown {
1541                    client_commit_id: id.clone(),
1542                },
1543            });
1544        }
1545        Ok(SnapshotRead {
1546            revision,
1547            queries,
1548            coverage: CoverageSnapshot {
1549                complete: pending.is_empty() && missing.is_empty(),
1550                pending,
1551                missing,
1552            },
1553            units,
1554            deliveries,
1555        })
1556    })();
1557    match result {
1558        Ok(read) => {
1559            conn.execute_batch("RELEASE syncular_snapshot_read")?;
1560            Ok(read)
1561        }
1562        Err(mut error) => {
1563            if let Err(rollback) = conn
1564                .execute_batch("ROLLBACK TO syncular_snapshot_read; RELEASE syncular_snapshot_read")
1565            {
1566                error.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
1567            }
1568            Err(error)
1569        }
1570    }
1571}
1572
1573fn snapshot_connection(
1574    conn: &Connection,
1575    sql: &str,
1576    params: &[Value],
1577    coverage: &[WindowCoverage],
1578) -> Result<QuerySnapshot, QueryReadFailure> {
1579    let read = snapshot_read_connection(
1580        conn,
1581        &SnapshotReadRequest {
1582            statements: vec![SnapshotStatement { sql, params }],
1583            coverage,
1584            units: Vec::new(),
1585            commit_ids: Vec::new(),
1586        },
1587    )?;
1588    Ok(QuerySnapshot {
1589        revision: read.revision,
1590        rows: read
1591            .queries
1592            .into_iter()
1593            .next()
1594            .expect("single-query snapshot returns one result"),
1595        coverage: read.coverage,
1596    })
1597}
1598
1599/// A long-lived read-only SQLite sidecar for latency-critical native views.
1600/// Network rounds stay serialized on the mutable core owner, while atomic
1601/// query snapshots use this independent connection and therefore never queue
1602/// behind HTTP/WebSocket latency.
1603pub struct FileQuerySnapshotReader {
1604    path: String,
1605    conn: Option<Connection>,
1606}
1607
1608impl FileQuerySnapshotReader {
1609    #[must_use]
1610    pub fn new(path: impl Into<String>) -> Self {
1611        Self {
1612            path: path.into(),
1613            conn: None,
1614        }
1615    }
1616
1617    fn connection(&mut self) -> Result<&Connection, QueryReadFailure> {
1618        if self.conn.is_none() {
1619            let conn = Connection::open_with_flags(
1620                &self.path,
1621                OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
1622            )
1623            .map_err(QueryReadFailure::from)?;
1624            conn.busy_timeout(std::time::Duration::from_millis(250))
1625                .map_err(QueryReadFailure::from)?;
1626            self.conn = Some(conn);
1627        }
1628        self.conn
1629            .as_ref()
1630            .ok_or_else(|| QueryReadFailure::from("read sidecar connection missing".to_owned()))
1631    }
1632
1633    /// One atomic read over the shared primitive: every statement, coverage,
1634    /// catch-up, and delivery under one `SAVEPOINT` and one revision.
1635    pub fn snapshot_read(
1636        &mut self,
1637        request: &SnapshotReadRequest<'_>,
1638    ) -> Result<SnapshotRead, QueryReadFailure> {
1639        snapshot_read_connection(self.connection()?, request)
1640    }
1641
1642    /// The current durable local revision. A host can skip a read when it is
1643    /// unchanged; a corrupt or unreadable marker fails typed.
1644    pub fn local_revision(&mut self) -> Result<String, QueryReadFailure> {
1645        Ok(self
1646            .snapshot_read(&SnapshotReadRequest::default())?
1647            .revision)
1648    }
1649
1650    /// One unit's persisted catch-up state (see the §7.5 contract).
1651    pub fn unit_catchup(&mut self, id: &str) -> Result<UnitCatchup, QueryReadFailure> {
1652        let read = self.snapshot_read(&SnapshotReadRequest {
1653            units: vec![id.to_owned()],
1654            ..Default::default()
1655        })?;
1656        Ok(read
1657            .units
1658            .into_iter()
1659            .next()
1660            .expect("one requested catch-up"))
1661    }
1662
1663    /// One client commit's delivery status: pending, its retained outcome, or
1664    /// an explicit unknown (never held, or pruned by retention).
1665    pub fn commit_delivery(&mut self, id: &str) -> Result<CommitDelivery, QueryReadFailure> {
1666        let read = self.snapshot_read(&SnapshotReadRequest {
1667            commit_ids: vec![id.to_owned()],
1668            ..Default::default()
1669        })?;
1670        Ok(read
1671            .deliveries
1672            .into_iter()
1673            .next()
1674            .expect("one requested delivery"))
1675    }
1676
1677    /// Hosts report an owned failure, and the first success after it, to the
1678    /// owning core through [`SyncClient::record_query_read`] (§7.6).
1679    pub fn query_snapshot(
1680        &mut self,
1681        sql: &str,
1682        params: &[Value],
1683        coverage: &[WindowCoverage],
1684    ) -> Result<QuerySnapshot, QueryReadFailure> {
1685        snapshot_connection(self.connection()?, sql, params, coverage)
1686    }
1687}
1688
1689fn validate_authority_reads(
1690    schema: &ClientSchema,
1691    declarations: &[crate::api::AuthorityReadDeclaration],
1692) -> Result<(), String> {
1693    let mut seen = BTreeSet::new();
1694    for read in declarations {
1695        let table = schema
1696            .table(&read.table)
1697            .ok_or("client.authority_read_forbidden: authority table is not declared")?;
1698        let plain = |name: &str| {
1699            table.columns.iter().enumerate().any(|(i, c)| {
1700                c.name == name
1701                    && name != "authority_version"
1702                    && !name.starts_with("_sync")
1703                    && !table.encrypted_columns.iter().any(|e| e.index == i)
1704                    && matches!(
1705                        c.ty,
1706                        ssp2::segment::ColumnType::String
1707                            | ssp2::segment::ColumnType::Integer
1708                            | ssp2::segment::ColumnType::Float
1709                            | ssp2::segment::ColumnType::Boolean
1710                            | ssp2::segment::ColumnType::Json
1711                    )
1712            })
1713        };
1714        if !seen.insert(&read.table)
1715            || read.columns.is_empty()
1716            || read.columns.iter().collect::<BTreeSet<_>>().len() != read.columns.len()
1717            || !read.columns.contains(&table.primary_key)
1718            || !read.columns.iter().all(|c| plain(c))
1719            || read.scopes.is_empty()
1720            || !read.scopes.iter().all(|(v, values)| {
1721                table
1722                    .scope_column(v)
1723                    .is_some_and(|c| read.columns.iter().any(|column| column == c) && plain(c))
1724                    && !values.is_empty()
1725                    && values.iter().all(|v| !v.is_empty() && v != "*")
1726            })
1727        {
1728            return Err("client.authority_read_forbidden: authority reads require plain columns, a primary key and nonempty plain scope selectors".into());
1729        }
1730    }
1731    Ok(())
1732}
1733
1734// §7.4.2: every open and recreation checks before any persistent writes.
1735/// A marker read that hit a recognized SQLite storage failure keeps that
1736/// classification (BUSY/LOCKED stay retryable); only an unclassified read
1737/// failure is the invalid-marker `sync.local_corrupt`.
1738fn schema_marker_read_failure(error: rusqlite::Error) -> String {
1739    let failure = QueryReadFailure::from(error);
1740    if failure.code.is_some() {
1741        failure.to_string()
1742    } else {
1743        "sync.local_corrupt: persisted local schema marker is unreadable or invalid".to_owned()
1744    }
1745}
1746
1747/// §7.1: reject an invalid configured client limit before any storage is
1748/// opened or written. Zero would wedge the push queue and an out-of-range
1749/// value would differ per target; the TS core enforces the same bounds.
1750fn validate_client_limits(limits: &crate::api::ClientLimits) -> Result<(), String> {
1751    if limits.outcome_retention_max_entries == Some(0) {
1752        return Err("sync.invalid_request: outcomeRetentionMaxEntries must be positive".to_owned());
1753    }
1754    if [
1755        limits.max_push_commits_per_request,
1756        limits.max_push_operations_per_request,
1757        limits.max_push_request_bytes,
1758    ]
1759    .into_iter()
1760    .flatten()
1761    .any(|value| value == 0 || value > u32::MAX as usize)
1762    {
1763        return Err(
1764            "sync.invalid_request: push limit must be an integer in 1..=4294967295".to_owned(),
1765        );
1766    }
1767    Ok(())
1768}
1769
1770fn read_local_schema_version(conn: &Connection, requested: i32) -> Result<Option<i32>, String> {
1771    let corrupt =
1772        || "sync.local_corrupt: persisted local schema marker is unreadable or invalid".to_owned();
1773    let table_type: Option<String> = conn
1774        .query_row(
1775            "SELECT type FROM sqlite_master WHERE name = '_syncular_meta'",
1776            [],
1777            |row| row.get(0),
1778        )
1779        .optional()
1780        .map_err(schema_marker_read_failure)?;
1781    let Some(table_type) = table_type else {
1782        return Ok(None);
1783    };
1784    if table_type != "table" {
1785        return Err(corrupt());
1786    }
1787    // §7.4.2: a duplicated marker cannot be resolved. `query_row` would take
1788    // whichever row SQLite returns first, and an older one would drive a
1789    // destructive reset over a newer replica.
1790    let mut statement = conn
1791        .prepare("SELECT value FROM _syncular_meta WHERE key = ?1")
1792        .map_err(schema_marker_read_failure)?;
1793    let mut rows = statement
1794        .query([LOCAL_SCHEMA_VERSION_KEY])
1795        .map_err(schema_marker_read_failure)?;
1796    let value: Option<String> = rows
1797        .next()
1798        .map_err(schema_marker_read_failure)?
1799        .map(|row| row.get(0))
1800        .transpose()
1801        .map_err(schema_marker_read_failure)?;
1802    if rows.next().map_err(schema_marker_read_failure)?.is_some() {
1803        return Err(corrupt());
1804    }
1805    let Some(value) = value else {
1806        let has_descriptor = conn
1807            .query_row(
1808                "SELECT 1 FROM _syncular_meta WHERE key = ?1 LIMIT 1",
1809                [crate::previous_version::LOCAL_SCHEMA_DESCRIPTOR_KEY],
1810                |_| Ok(()),
1811            )
1812            .optional()
1813            .map_err(schema_marker_read_failure)?
1814            .is_some();
1815        if has_descriptor {
1816            return Err(corrupt());
1817        }
1818        return Ok(None);
1819    };
1820    if !matches!(value.as_bytes().first(), Some(b'1'..=b'9'))
1821        || !value.bytes().all(|b| b.is_ascii_digit())
1822    {
1823        return Err(corrupt());
1824    }
1825    let version = value.parse::<i32>().map_err(|_| corrupt())?;
1826    if version > requested {
1827        return Err(
1828            "client.schema_downgrade: persisted local schema is newer than the requested schema"
1829                .to_owned(),
1830        );
1831    }
1832    Ok(Some(version))
1833}
1834
1835/// A file replica on a browser worker must recover a hot journal before any
1836/// statement reads it. `wasm_vfs` replaces the installed OPFS VFS's reserved
1837/// lock callback, so install it before the connection opens its file.
1838#[cfg(all(target_arch = "wasm32", target_os = "unknown"))]
1839fn prepare_wasm_replica(path: &str) -> Result<(), String> {
1840    if path.is_empty() || path == ":memory:" {
1841        return Ok(());
1842    }
1843    crate::wasm_vfs::disable_foreign_lock_check()
1844}
1845
1846#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))]
1847fn prepare_wasm_replica(_path: &str) -> Result<(), String> {
1848    Ok(())
1849}
1850
1851impl SyncClient {
1852    pub fn new_with_identity(
1853        client_id: Option<String>,
1854        schema_json: &Value,
1855        limits: ClientLimits,
1856    ) -> Result<Self, String> {
1857        let resolved = client_id.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
1858        let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
1859        Self::with_connection(resolved, schema_json, limits, conn)
1860    }
1861
1862    pub fn new(
1863        client_id: String,
1864        schema_json: &Value,
1865        limits: ClientLimits,
1866    ) -> Result<Self, String> {
1867        let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
1868        Self::with_connection(client_id, schema_json, limits, conn)
1869    }
1870
1871    /// Build a client backed by an on-disk SQLite database at `path` — the
1872    /// seam a native host (Tauri plugin, FFI file-DB variant) uses to persist
1873    /// across process restarts. `create_tables` runs `IF NOT EXISTS`, so
1874    /// re-opening the same file reuses the persisted rows. Keeps rusqlite out
1875    /// of the command router's dependency set (the router only holds a path).
1876    pub fn open_path(
1877        client_id: String,
1878        schema_json: &Value,
1879        limits: ClientLimits,
1880        path: &str,
1881    ) -> Result<Self, String> {
1882        validate_client_limits(&limits)?;
1883        parse_schema_json(schema_json)?;
1884        prepare_wasm_replica(path)?;
1885        let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
1886        Self::with_connection(client_id, schema_json, limits, conn)
1887    }
1888
1889    /// Open a file replica and resolve its client id. Native targets require
1890    /// WAL for the independent snapshot reader; `wasm32-unknown-unknown`
1891    /// accepts the sqlite-wasm worker journal instead.
1892    pub fn open_path_with_identity(
1893        client_id: Option<String>,
1894        schema_json: &Value,
1895        limits: ClientLimits,
1896        path: &str,
1897    ) -> Result<Self, String> {
1898        Self::open_path_with_identity_journal(
1899            client_id,
1900            schema_json,
1901            limits,
1902            path,
1903            !cfg!(all(target_arch = "wasm32", target_os = "unknown")),
1904        )
1905    }
1906
1907    fn open_path_with_identity_journal(
1908        client_id: Option<String>,
1909        schema_json: &Value,
1910        limits: ClientLimits,
1911        path: &str,
1912        require_wal: bool,
1913    ) -> Result<Self, String> {
1914        validate_client_limits(&limits)?;
1915        // §7.4.2: an invalid requested version is refused before the replica is
1916        // even created, so a rejected constructor leaves no file behind.
1917        parse_schema_json(schema_json)?;
1918        prepare_wasm_replica(path)?;
1919        let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
1920        // A file-backed native client uses an independent read connection for
1921        // latency-critical snapshots. WAL is SQLite's intended reader/writer
1922        // concurrency mode: a view read never holds a rollback-journal lock
1923        // that delays the client's next commit. The busy timeout must precede
1924        // this replica's own startup transaction; WAL is applied only after
1925        // startup succeeded, because a refused open must leave the replica's
1926        // journal mode and contents untouched (§7.4.2).
1927        conn.busy_timeout(std::time::Duration::from_millis(250))
1928            .map_err(|error| format!("configure db {path:?} busy timeout: {error}"))?;
1929        // Durability is the caller's contract here, not the VFS default: a
1930        // rollback-journal commit needs FULL before it writes the database
1931        // pages, or a killed worker can leave a half-written image behind.
1932        // WAL below inherits the same setting.
1933        conn.pragma_update(None, "synchronous", "FULL")
1934            .map_err(|error| format!("configure db {path:?} synchronous: {error}"))?;
1935        let client = Self::with_connection_identity(client_id, schema_json, limits, conn)?;
1936        // A browser worker's sqlite-wasm VFS has no WAL mode, and the wasm
1937        // host has no independent snapshot reader. The WAL requirement exists
1938        // for native file replicas, where it is the reader/writer boundary.
1939        if require_wal {
1940            let journal_mode: String = client
1941                .conn
1942                .pragma_update_and_check(None, "journal_mode", "WAL", |row| row.get(0))
1943                .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
1944            if journal_mode != "wal" {
1945                return Err(
1946                    "sync.invalid_request: file-backed client requires WAL journal mode".to_owned(),
1947                );
1948            }
1949        }
1950        Ok(client)
1951    }
1952
1953    /// Delete a file replica at `path`, including its `-journal`, `-wal`, and
1954    /// `-shm` sidecars, through the VFS that owns it. Use this after
1955    /// `client.storage_corrupt`: an OPFS SAH pool stores those files under
1956    /// opaque names, so deleting the pool directory behind SQLite bypasses the
1957    /// pool's slot accounting.
1958    ///
1959    /// Close the client and every [`FileQuerySnapshotReader`] on `path` first;
1960    /// a VFS cannot delete a file another handle still owns. A file that does
1961    /// not exist counts as deleted, so a cleanly closed replica without
1962    /// sidecars resets. The next open
1963    /// creates a fresh replica (new client id) and re-bootstraps from the
1964    /// server. A memory replica has nothing to delete and fails with
1965    /// `sync.invalid_request`.
1966    pub fn reset_replica(path: &str) -> Result<(), String> {
1967        if path.is_empty() || path == ":memory:" {
1968            return Err("sync.invalid_request: a memory replica has nothing to reset".to_owned());
1969        }
1970        // SAFETY: a null VFS name asks SQLite for its process-wide default VFS,
1971        // the same one `Connection::open(path)` above selects. `xDelete` is
1972        // that VFS's own function pointer, and `c_name` outlives the call.
1973        let vfs = unsafe { rusqlite::ffi::sqlite3_vfs_find(std::ptr::null()) };
1974        if vfs.is_null() {
1975            return Err("client.storage_io: no SQLite VFS is registered".to_owned());
1976        }
1977        let delete = unsafe { (*vfs).xDelete }
1978            .ok_or_else(|| "client.storage_io: the SQLite VFS cannot delete files".to_owned())?;
1979        for suffix in ["-wal", "-shm", "-journal", ""] {
1980            let file_name = format!("{path}{suffix}");
1981            let c_name = std::ffi::CString::new(file_name.as_str()).map_err(|_| {
1982                format!("sync.invalid_request: replica path {file_name:?} contains a NUL byte")
1983            })?;
1984            // SAFETY: `vfs` and its `xDelete` pointer come from the live
1985            // SQLite library; `c_name` is a valid NUL-terminated path.
1986            let code = unsafe { delete(vfs, c_name.as_ptr(), 1) };
1987            // A cleanly closed WAL replica has no sidecars; a missing file is
1988            // already deleted.
1989            if code != rusqlite::ffi::SQLITE_OK && code != rusqlite::ffi::SQLITE_IOERR_DELETE_NOENT
1990            {
1991                return Err(format!(
1992                    "client.storage_io: deleting replica file {file_name:?} failed with SQLite code {code}"
1993                ));
1994            }
1995        }
1996        Ok(())
1997    }
1998
1999    #[must_use]
2000    pub fn client_id(&self) -> &str {
2001        &self.client_id
2002    }
2003
2004    /// Rotate the caller's client id (§1.5) and return the new one. Use it
2005    /// after a `sync.invalid_client_id` refusal (`recommendedAction:
2006    /// resetClientId`): the next round carries the new id. A live realtime
2007    /// socket is closed so the next connection registers with the new id.
2008    /// The replica and its outbox stay in place; wipe them yourself when the
2009    /// stored data belongs to the account that signed out.
2010    pub fn reset_client_id(&mut self, transport: &mut dyn Transport) -> Result<String, String> {
2011        self.disconnect_realtime(transport);
2012        let client_id = uuid::Uuid::new_v4().to_string();
2013        self.set_meta(CLIENT_ID_KEY, &client_id)?;
2014        self.client_id.clone_from(&client_id);
2015        self.round_failure = None;
2016        Ok(client_id)
2017    }
2018
2019    /// Build a client over a caller-supplied rusqlite connection — the seam a
2020    /// native host (Tauri plugin, FFI file-DB variant) uses to back the core
2021    /// with an on-disk database (`Connection::open(path)`) rather than the
2022    /// default `:memory:`. The connection MUST be fresh (no pre-existing
2023    /// syncular tables); `create_tables` runs `IF NOT EXISTS`, so re-opening
2024    /// the same file across process restarts reuses the persisted rows.
2025    pub fn with_connection(
2026        client_id: String,
2027        schema_json: &Value,
2028        limits: ClientLimits,
2029        conn: Connection,
2030    ) -> Result<Self, String> {
2031        Self::with_connection_identity(Some(client_id), schema_json, limits, conn)
2032    }
2033
2034    /// [`Self::with_connection`] for `open_path_with_identity`: no requested id
2035    /// means a durable replica's persisted `clientId` wins over a fresh one.
2036    fn with_connection_identity(
2037        client_id: Option<String>,
2038        schema_json: &Value,
2039        limits: ClientLimits,
2040        conn: Connection,
2041    ) -> Result<Self, String> {
2042        validate_client_limits(&limits)?;
2043        let schema = parse_schema_json(schema_json)?;
2044        validate_authority_reads(&schema, &limits.authority_reads)?;
2045        // RFC 0005 D8: resolve the previous-version config before the opening
2046        // reset runs — the capture happens inside it. Bad bounds fail loud.
2047        let previous_version = match limits.previous_version_context {
2048            Some(config) => {
2049                config.validate()?;
2050                Some(config)
2051            }
2052            None => None,
2053        };
2054        let mut client = SyncClient {
2055            #[cfg(feature = "bench-internals")]
2056            benchmark_phases: Recorder::default(),
2057            conn,
2058            schema,
2059            client_id: client_id
2060                .clone()
2061                .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()),
2062            limits,
2063            outbox: Shared::default(),
2064            failed_commits: Shared::default(),
2065            retain_failed_commits: false,
2066            conflicts: Shared::default(),
2067            rejections: Shared::default(),
2068            schema_floor: None,
2069            lease_state: None,
2070            stopped: false,
2071            upgrading: false,
2072            sync_needed: false,
2073            realtime_policy: RealtimePolicy::Optional,
2074            realtime_state: RealtimeState::Disconnected,
2075            realtime_reason_code: None,
2076            realtime_retry_delay_ms: None,
2077            presence: HashMap::new(),
2078            now_ms: None,
2079            encryption: crate::values::EncryptionConfig::default(),
2080            security_preflight: false,
2081            security_epoch: 0,
2082            transport_enabled: true,
2083            transport_epoch: 0,
2084            registration_epoch: 0,
2085            previous_version,
2086            insert_sql: RefCell::new(HashMap::new()),
2087            overlay_dirty: OverlayDirty::clean(),
2088            storage_failure: RefCell::new(None),
2089            #[cfg(test)]
2090            overlay_rebuild_count: Cell::new(0),
2091            #[cfg(test)]
2092            acknowledged_replay_count: Cell::new(0),
2093            #[cfg(test)]
2094            outcome_prune_count: Cell::new(0),
2095            #[cfg(test)]
2096            image_chunk_ms: Cell::new(None),
2097            progress: ProgressObserver::default(),
2098            change_queue: VecDeque::new(),
2099            sync_intent_queue: VecDeque::new(),
2100            retry_delay_ms: 250,
2101            round_retry_delay_ms: None,
2102            image_chunk_rows: HashMap::new(),
2103            round_failure: None,
2104            active_round: None,
2105            last_round: None,
2106            last_change: None,
2107            query_failures: Vec::new(),
2108            coalesced: None,
2109            coalesced_failure: None,
2110        };
2111        // The row write path leans on the prepared-statement cache (two
2112        // insert statements per synced table, plus the bookkeeping
2113        // statements); size it so a multi-table schema never thrashes.
2114        client
2115            .conn
2116            .set_prepared_statement_cache_capacity(64.max(client.schema.tables.len() * 4));
2117        // §7.4.2: one write transaction (SQLite `BEGIN IMMEDIATE`, a reserved
2118        // writer) covers the marker read, the identity check, and every
2119        // bookkeeping/schema write, so a concurrent open that upgraded the
2120        // replica cannot interleave between the guard and the writes, and a
2121        // refusal leaves the replica unchanged.
2122        client
2123            .conn
2124            .execute_batch("BEGIN IMMEDIATE")
2125            .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
2126        let startup = (|| -> Result<(), String> {
2127            // The marker guard runs inside the write transaction, so the version
2128            // it validates is the version every write below commits under.
2129            let marker = read_local_schema_version(&client.conn, client.schema.version)?;
2130            // New app indexes may reference columns that only exist after a
2131            // version-bump reset.
2132            client.create_bookkeeping_tables()?;
2133            match client.get_meta(CLIENT_ID_KEY)? {
2134                Some(existing) => match client_id {
2135                    Some(requested) if existing != requested => {
2136                        return Err(format!(
2137                            "client.identity_mismatch: this database belongs to {existing:?}; refusing to rebind it to {requested:?}"
2138                        ))
2139                    }
2140                    _ => client.client_id = existing,
2141                },
2142                None => client.set_meta(CLIENT_ID_KEY, &client.client_id)?,
2143            }
2144            client.restore_persisted_state()?;
2145            client.failed_commits = client.load_failed_commits()?.into();
2146            match marker {
2147                None => {
2148                    client.create_synced_tables()?;
2149                    client.persist_schema_version()?;
2150                    client.save_unit_scope_schema()?;
2151                }
2152                Some(version) if version == client.schema.version => {
2153                    client.create_synced_tables()?;
2154                    // RFC 0005 D1: the same-version open backfills the descriptor
2155                    // for a database first opened by an unaware binary, so the
2156                    // NEXT bump can capture without needing a schema bump first.
2157                    set_local_schema_descriptor(&client.conn, &client.schema)
2158                        .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
2159                    client.save_unit_scope_schema()?;
2160                }
2161                Some(_) => client.run_schema_reset()?,
2162            }
2163            // RFC 0005 D9: the reset sweep cannot see a container whose capture
2164            // committed in its OWN file while the replica's savepoint never
2165            // released — a crash at that seam leaves a container beside a matching
2166            // OLD marker, and the next open at that version runs no reset. Discard
2167            // it before anything can read it. No-op when no container exists.
2168            client.reconcile_previous_version_at_boot()?;
2169            client.clear_satisfied_persisted_schema_floor()?;
2170            client.prune_unknown_units(false)?;
2171            // §7.1/§7.4.4: recover intrinsically codec-invalid persisted values at
2172            // the full startup/reset reconciliation boundary, inside this open
2173            // transaction and after the schema and identity guards, so a commit
2174            // an earlier version accepted is dropped with durable rejection
2175            // evidence before any replay. A table the current schema removed
2176            // keeps the §7.4.4 send-time classification, and an unresolved
2177            // encryption key stays with the send seam because hosts configure
2178            // keys after open.
2179            client.drop_unencodable_outbox(false, None)?;
2180            if marker == Some(client.schema.version)
2181                && (!client.outbox.is_empty()
2182                    || !client.failed_commits.is_empty()
2183                    || client.has_acknowledged_rows()?)
2184            {
2185                // Reconstruct the visible optimistic overlay from the durable
2186                // base plus outbox instead of trusting a process-interrupted
2187                // mirror.
2188                client.overlay_dirty.set(true);
2189                client.rebuild_overlay()?;
2190            }
2191            // Every persisted active unit needs one catch-up pull on
2192            // open: realtime only covers changes after connection, while an
2193            // idempotent setWindow correctly creates no fresh command effect.
2194            // Pending outbox work has the same restart requirement. The core
2195            // owns this intent so native hosts never poll or require an
2196            // application-issued sync().
2197            client.enqueue_startup_sync_if_needed()?;
2198            Ok(())
2199        })();
2200        match startup {
2201            Ok(()) => match client.conn.execute_batch("COMMIT") {
2202                // §4.8: an eviction a crash interrupted finishes before any
2203                // read, with or without a transport; a replica reopened in
2204                // preflight finishes it at activation.
2205                Ok(()) => {
2206                    if !client.security_preflight {
2207                        client.drain_pending_evictions()?;
2208                    }
2209                    Ok(client)
2210                }
2211                Err(error) => {
2212                    let failure = Self::sqlite_failure(&client.storage_failure, error);
2213                    if let Err(rollback) = client.conn.execute_batch("ROLLBACK") {
2214                        // Keep a failed rollback's typed storage classification.
2215                        let _ = Self::sqlite_failure(&client.storage_failure, rollback);
2216                    }
2217                    Err(failure)
2218                }
2219            },
2220            Err(error) => {
2221                if let Err(rollback) = client.conn.execute_batch("ROLLBACK") {
2222                    // Keep a failed rollback's typed storage classification.
2223                    let _ = Self::sqlite_failure(&client.storage_failure, rollback);
2224                }
2225                Err(error)
2226            }
2227        }
2228    }
2229
2230    /// Pin the client clock (epoch ms) — the §5.4 expiry check runs
2231    /// against this instead of system time (conformance virtual clock).
2232    pub fn set_now_ms(&mut self, now_ms: i64) {
2233        self.now_ms = Some(std::sync::Arc::new(now_ms.into()));
2234    }
2235
2236    /// The pinned clock, shared: a host that stores into it moves the client
2237    /// clock while a call runs (a virtual clock that advances per request).
2238    pub fn pinned_clock(&self) -> Option<std::sync::Arc<std::sync::atomic::AtomicI64>> {
2239        self.now_ms.clone()
2240    }
2241
2242    /// §5.11: install the client-side encryption keys (`keyId → key bytes`).
2243    /// The command router parses these from the `create` command's
2244    /// `encryption` config (keys as `{$bytes: hex}`).
2245    pub fn set_encryption(&mut self, encryption: crate::values::EncryptionConfig) {
2246        self.encryption = encryption;
2247    }
2248
2249    #[must_use]
2250    pub fn security_lifecycle(&self) -> &'static str {
2251        if self.security_preflight {
2252            "preflight"
2253        } else {
2254            "active"
2255        }
2256    }
2257
2258    #[must_use]
2259    pub fn security_preflight(&self) -> bool {
2260        self.security_preflight
2261    }
2262
2263    /// Host-owned, non-persisted network gate. Local operations remain available.
2264    pub fn set_transport_enabled(&mut self, transport: &mut dyn Transport, enabled: bool) {
2265        if self.transport_enabled != enabled {
2266            self.transport_enabled = enabled;
2267            self.transport_epoch = self.transport_epoch.wrapping_add(1);
2268            self.sync_intent_queue.clear();
2269            self.sync_intent_queue
2270                .push_back(if enabled && !self.security_preflight {
2271                    SyncIntent::Interactive
2272                } else {
2273                    SyncIntent::None
2274                });
2275        }
2276        // A captured reply keeps its normal apply/revocation semantics.
2277        if !enabled && self.active_round.is_none() {
2278            self.disconnect_realtime(transport);
2279        }
2280    }
2281
2282    pub fn transport_enabled(&self) -> bool {
2283        self.transport_enabled
2284    }
2285
2286    /// Quarantine this replica: enter the fail-closed gate, release all
2287    /// core-owned key material, and record the gate in the database so
2288    /// reopening re-enters preflight until `activate_security` clears it.
2289    ///
2290    /// Use [`SyncClient::seal_security_on_teardown`] for the shutdown barrier,
2291    /// which must NOT leave that durable mark.
2292    pub fn begin_security_preflight(&mut self) -> Result<(), String> {
2293        let entering = !self.security_preflight;
2294        self.seal_security_on_teardown();
2295        // Persist the quarantine so it survives handle teardown and restart:
2296        // reopening this replica re-enters preflight until activation clears it.
2297        self.set_meta(SECURITY_PREFLIGHT_PENDING_KEY, "1")?;
2298        if entering {
2299            self.emit_status_batch()?;
2300        }
2301        Ok(())
2302    }
2303
2304    /// One change batch carrying the status snapshot (§5.11 lifecycle).
2305    fn emit_status_batch(&mut self) -> Result<(), String> {
2306        self.begin_observation("syncular_status")?;
2307        let batch = ChangeAccumulator {
2308            status: true,
2309            ..ChangeAccumulator::default()
2310        };
2311        self.finish_observation("syncular_status", batch)
2312            .inspect_err(|_| self.rollback_observation("syncular_status"))
2313    }
2314
2315    /// Release core-owned key material as a host tears the client down,
2316    /// WITHOUT recording a quarantine.
2317    ///
2318    /// Shutting an activated client down is not a quarantine event. Persisting
2319    /// the gate here marks every cleanly closed replica as pending, and the
2320    /// next plain `create` is then refused permanently: the reopen path
2321    /// restores the flag from the marker and the create guard rejects it. The
2322    /// in-memory flag still closes the gate for anything still holding this
2323    /// instance, which is all a teardown barrier needs.
2324    pub fn seal_security_on_teardown(&mut self) {
2325        self.cancel_sync_round();
2326        self.security_preflight = true;
2327        self.security_epoch = self.security_epoch.wrapping_add(1);
2328        self.encryption = crate::values::EncryptionConfig::default();
2329        self.sync_intent_queue.clear();
2330    }
2331
2332    /// Install the post-authentication keyring and release the host loop.
2333    pub fn activate_security(
2334        &mut self,
2335        encryption: crate::values::EncryptionConfig,
2336    ) -> Result<(), String> {
2337        if !self.security_preflight {
2338            return Err(
2339                "sync.invalid_request: activateSecurity requires security preflight".to_owned(),
2340            );
2341        }
2342        // §4.8: an interrupted eviction finishes before protected reads open.
2343        self.drain_pending_evictions()?;
2344        self.enqueue_startup_sync_if_needed()?;
2345        self.delete_meta(SECURITY_PREFLIGHT_PENDING_KEY)?;
2346        self.encryption = encryption;
2347        self.security_preflight = false;
2348        // §5.11: rows sealed under a key this keyring now holds open at once.
2349        self.reveal_sealed_rows()
2350            .map_err(|error| format!("{}: {}", error.code, error.message))?;
2351        self.emit_status_batch()
2352    }
2353
2354    fn clock_now_ms(&self) -> i64 {
2355        self.now_ms
2356            .as_ref()
2357            .map_or_else(crate::wall_clock_ms, |clock| {
2358                clock.load(std::sync::atomic::Ordering::Relaxed)
2359            })
2360    }
2361
2362    fn create_bookkeeping_tables(&self) -> Result<(), String> {
2363        let exists = |name: &str| {
2364            self.conn
2365                .query_row(
2366                    "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?1)",
2367                    [name],
2368                    |row| row.get::<_, bool>(0),
2369                )
2370                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2371        };
2372        let outbox_existed = exists("_syncular_outbox")?;
2373        // Durable client bookkeeping (outbox + units + meta).
2374        self.conn
2375            .execute_batch(
2376                "CREATE TABLE IF NOT EXISTS _syncular_outbox (
2377                   seq INTEGER PRIMARY KEY AUTOINCREMENT,
2378                   commit_id TEXT NOT NULL UNIQUE, ops_json TEXT NOT NULL);
2379                 CREATE TABLE IF NOT EXISTS _syncular_row_deliveries (
2380                   tbl TEXT NOT NULL, id TEXT NOT NULL, commit_seq INTEGER NOT NULL, PRIMARY KEY(tbl,id));
2381                 CREATE TABLE IF NOT EXISTS _syncular_acked_rows (
2382                   commit_id TEXT NOT NULL, idx INTEGER NOT NULL, tbl TEXT NOT NULL, id TEXT NOT NULL,
2383                   commit_seq INTEGER NOT NULL, op_json TEXT NOT NULL, intent_json TEXT,
2384                   PRIMARY KEY(commit_id, idx));
2385                 CREATE INDEX IF NOT EXISTS _syncular_acked_rows_row
2386                   ON _syncular_acked_rows(tbl, id);
2387                 CREATE TABLE IF NOT EXISTS _syncular_failed_commits (
2388                   seq INTEGER PRIMARY KEY AUTOINCREMENT, client_commit_id TEXT NOT NULL UNIQUE,
2389                   operations_json TEXT NOT NULL, initial_json TEXT NOT NULL);
2390                 CREATE TABLE IF NOT EXISTS _syncular_commit_outcomes (
2391                   seq INTEGER PRIMARY KEY AUTOINCREMENT,
2392                   client_commit_id TEXT NOT NULL UNIQUE,
2393                   status TEXT NOT NULL CHECK(status IN ('applied', 'cached', 'conflict', 'rejected')),
2394                   recorded_at_ms INTEGER NOT NULL,
2395                   results_json TEXT NOT NULL,
2396                   operations_json TEXT,
2397                   resolution TEXT NOT NULL DEFAULT 'active'
2398                     CHECK(resolution IN ('active', 'resolved_keep_server', 'superseded', 'dismissed')),
2399                   resolved_at_ms INTEGER,
2400                   replacement_client_commit_id TEXT);
2401                 CREATE INDEX IF NOT EXISTS _syncular_commit_outcomes_resolution_seq
2402                   ON _syncular_commit_outcomes(resolution, seq);
2403                 CREATE TABLE IF NOT EXISTS _syncular_units (
2404                   id TEXT PRIMARY KEY, scopes TEXT NOT NULL,
2405                   explicit INTEGER NOT NULL DEFAULT 0,
2406                   status TEXT NOT NULL DEFAULT 'active', reason_code TEXT,
2407                   cursor INTEGER DEFAULT -1, bootstrap_state TEXT,
2408                   seeded TEXT NOT NULL DEFAULT '[]', echo TEXT, applied TEXT);
2409                 CREATE TABLE IF NOT EXISTS _syncular_meta (
2410                   key TEXT PRIMARY KEY, value TEXT NOT NULL);
2411                 CREATE TABLE IF NOT EXISTS _syncular_windows (
2412                   base TEXT NOT NULL, unit TEXT NOT NULL, sub_id TEXT NOT NULL,
2413                   PRIMARY KEY (base, unit));
2414                 CREATE INDEX IF NOT EXISTS _syncular_windows_unit ON _syncular_windows(sub_id);
2415                 CREATE TABLE IF NOT EXISTS _syncular_pending_evictions (
2416                   unit_id TEXT NOT NULL, tbl TEXT NOT NULL,
2417                   effective_scopes TEXT NOT NULL, PRIMARY KEY (unit_id, tbl));
2418                 CREATE TABLE IF NOT EXISTS _syncular_blob_commit_refs(
2419                   commit_id TEXT NOT NULL, blob_id TEXT NOT NULL,
2420                   PRIMARY KEY(commit_id, blob_id));
2421                 CREATE INDEX IF NOT EXISTS _syncular_blob_commit_refs_body
2422                   ON _syncular_blob_commit_refs(blob_id);",
2423            )
2424            .map_err(|e| e.to_string())?;
2425        // §5.6 applied-segment record, added after the units table shipped.
2426        if !self
2427            .conn
2428            .prepare("SELECT 1 FROM pragma_table_info('_syncular_units') WHERE name = 'applied'")
2429            .and_then(|mut stmt| stmt.exists([]))
2430            .map_err(|e| e.to_string())?
2431        {
2432            self.conn
2433                .execute_batch("ALTER TABLE _syncular_units ADD COLUMN applied TEXT")
2434                .map_err(|e| e.to_string())?;
2435        }
2436        // Migrate an outcome journal created before failed aggregate
2437        // envelopes were retained. Historical rows intentionally stay NULL.
2438        let _ = self
2439            .conn
2440            .execute_batch("ALTER TABLE _syncular_commit_outcomes ADD COLUMN operations_json TEXT");
2441        if self.get_meta(LOCAL_REVISION_KEY)?.is_none() {
2442            self.set_meta(LOCAL_REVISION_KEY, "0")?;
2443        }
2444        // §7.1: `outboxId` names the outbox sequence. A new outbox table starts
2445        // a new sequence, so it replaces the id in the same transaction.
2446        if !outbox_existed || self.get_meta(OUTBOX_ID_KEY)?.is_none() {
2447            self.set_meta(OUTBOX_ID_KEY, &uuid::Uuid::new_v4().to_string())?;
2448        }
2449        // §9: a replica written under wire version 3 keeps its outbox, journal,
2450        // rows and pending evictions and registers its units fresh. Its
2451        // subscriptions, cursors, echoes, resume tokens and window entries go;
2452        // units left from an earlier version 4 life restart.
2453        if exists("_syncular_window_pending_evict")? {
2454            let invalid: bool = self
2455                .conn
2456                .query_row(
2457                    "SELECT EXISTS(SELECT 1 FROM _syncular_window_pending_evict
2458                       WHERE NOT json_valid(effective_scopes)
2459                          OR json_type(effective_scopes) <> 'object'
2460                          OR EXISTS(SELECT 1 FROM json_each(effective_scopes)
2461                               WHERE type <> 'array'
2462                                  OR EXISTS(SELECT 1 FROM json_each(value) AS v
2463                                       WHERE v.type <> 'text')))",
2464                    [],
2465                    |row| row.get(0),
2466                )
2467                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
2468            if invalid {
2469                return Err(
2470                    "sync.local_corrupt: a version 3 pending eviction cannot be upgraded"
2471                        .to_owned(),
2472                );
2473            }
2474            self.conn
2475                .execute_batch(
2476                    "INSERT OR REPLACE INTO _syncular_pending_evictions
2477                       SELECT sub_id, tbl, effective_scopes FROM _syncular_window_pending_evict;
2478                     DROP TABLE _syncular_window_pending_evict;
2479                     DROP TABLE IF EXISTS _syncular_subscriptions;
2480                     DELETE FROM _syncular_windows;",
2481                )
2482                .and_then(|()| reset_units(&self.conn, &BTreeSet::new()))
2483                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
2484        }
2485        // §5.9.7 blob cache. Mutable upload state stays off the body row so
2486        // completing a large upload does not rewrite its BLOB record.
2487        if self.schema_has_blobs() {
2488            if exists("_syncular_blobs")? {
2489                let mut statement = self
2490                    .conn
2491                    .prepare("PRAGMA table_info(_syncular_blobs)")
2492                    .map_err(|e| e.to_string())?;
2493                let mut columns = statement
2494                    .query_map([], |row| row.get::<_, String>(1))
2495                    .map_err(|e| e.to_string())?
2496                    .collect::<Result<Vec<_>, _>>()
2497                    .map_err(|e| e.to_string())?;
2498                columns.sort();
2499                let mut expected = vec![
2500                    "blob_id",
2501                    "byte_length",
2502                    "bytes",
2503                    "created_at_ms",
2504                    "media_type",
2505                ];
2506                expected.sort_unstable();
2507                if columns != expected {
2508                    return Err(
2509                        "sync.schema_mismatch: Local blob schema is incompatible".to_owned()
2510                    );
2511                }
2512            } else {
2513                self.conn
2514                    .execute_batch(
2515                        "CREATE TABLE _syncular_blobs(
2516                           blob_id TEXT PRIMARY KEY,
2517                           bytes BLOB NOT NULL,
2518                           byte_length INTEGER NOT NULL,
2519                           media_type TEXT,
2520                           created_at_ms INTEGER NOT NULL);",
2521                    )
2522                    .map_err(|e| e.to_string())?;
2523            }
2524            self.conn
2525                .execute_batch(
2526                    "CREATE TABLE IF NOT EXISTS _syncular_blob_uploads(
2527                       blob_id TEXT PRIMARY KEY,
2528                       media_type TEXT,
2529                       created_at_ms INTEGER NOT NULL);",
2530                )
2531                .map_err(|e| e.to_string())?;
2532        }
2533        Ok(())
2534    }
2535
2536    /// True iff any synced table declares a `blob_ref` column (§5.9).
2537    fn schema_has_blobs(&self) -> bool {
2538        self.schema
2539            .tables
2540            .iter()
2541            .any(|t| t.columns.iter().any(|c| c.ty == ColumnType::BlobRef))
2542    }
2543
2544    // -- meta (§7.4.1 marker, bookkeeping) ------------------------------------
2545
2546    fn get_meta(&self, key: &str) -> Result<Option<String>, String> {
2547        meta_get(&self.conn, key)
2548            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2549    }
2550
2551    // The caller holds the startup/reset transaction. Either both schema
2552    // records persist or the enclosing operation fails and rolls back.
2553    fn persist_schema_version(&self) -> Result<(), String> {
2554        self.conn
2555            .execute(
2556                "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
2557                rusqlite::params![LOCAL_SCHEMA_VERSION_KEY, self.schema.version.to_string()],
2558            )
2559            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
2560        set_local_schema_descriptor(&self.conn, &self.schema)
2561            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2562    }
2563
2564    pub(crate) fn set_meta(&self, key: &str, value: &str) -> Result<(), String> {
2565        meta_set(&self.conn, key, value)
2566            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2567    }
2568
2569    fn delete_meta(&self, key: &str) -> Result<(), String> {
2570        meta_delete(&self.conn, key)
2571            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2572    }
2573
2574    /// §7.1: the durable outbox in FIFO order, in schema-agnostic local form.
2575    fn load_outbox(&self) -> Result<Vec<Arc<OutboxCommit>>, String> {
2576        let mut stmt = self
2577            .conn
2578            .prepare("SELECT commit_id, ops_json FROM _syncular_outbox ORDER BY seq ASC")
2579            .map_err(|error| error.to_string())?;
2580        let rows = stmt
2581            .query_map([], |row| {
2582                Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
2583            })
2584            .map_err(|error| error.to_string())?;
2585        let mut commits = Vec::new();
2586        for row in rows {
2587            let (client_commit_id, raw) = row.map_err(|error| error.to_string())?;
2588            let entries: Vec<Value> = serde_json::from_str(&raw).map_err(|error| {
2589                format!("invalid persisted outbox {client_commit_id:?}: {error}")
2590            })?;
2591            let mut ops = Vec::with_capacity(entries.len());
2592            for entry in entries {
2593                let op = entry.get("op").and_then(Value::as_str).unwrap_or("delete");
2594                ops.push(OutboxOp {
2595                    upsert: op == "upsert",
2596                    table: entry
2597                        .get("table")
2598                        .and_then(Value::as_str)
2599                        .ok_or_else(|| "persisted outbox operation missing table".to_owned())?
2600                        .to_owned(),
2601                    row_id: entry
2602                        .get("rowId")
2603                        .and_then(Value::as_str)
2604                        .ok_or_else(|| "persisted outbox operation missing rowId".to_owned())?
2605                        .to_owned(),
2606                    base_version: entry.get("baseVersion").and_then(Value::as_i64),
2607                    // A legacy entry's `changedFields` key is ignored:
2608                    // the values map is the presence set (§6.1).
2609                    values: entry.get("values").and_then(Value::as_object).cloned(),
2610                    insert: entry.get("insert") == Some(&Value::Bool(true)),
2611                });
2612            }
2613            commits.push(Arc::new(OutboxCommit {
2614                client_commit_id,
2615                ops,
2616            }));
2617        }
2618        Ok(commits)
2619    }
2620
2621    fn restore_persisted_state(&mut self) -> Result<(), String> {
2622        self.outbox = self.load_outbox()?.into();
2623
2624        self.prune_commit_outcomes()?;
2625        let active = self.commit_outcomes(CommitOutcomeQuery {
2626            active_only: true,
2627            ..CommitOutcomeQuery::default()
2628        })?;
2629        self.conflicts = active
2630            .iter()
2631            .flat_map(|outcome| outcome.results.iter())
2632            .filter_map(|result| match result {
2633                CommitOperationOutcome::Conflict { conflict } => Some(conflict.clone()),
2634                _ => None,
2635            })
2636            .collect::<Vec<_>>()
2637            .into();
2638        self.rejections = active
2639            .iter()
2640            .flat_map(|outcome| outcome.results.iter())
2641            .filter_map(|result| match result {
2642                CommitOperationOutcome::Error { rejection } => Some(rejection.clone()),
2643                _ => None,
2644            })
2645            .collect::<Vec<_>>()
2646            .into();
2647
2648        self.lease_state = self
2649            .get_meta(LEASE_STATE_KEY)?
2650            .map(|raw| serde_json::from_str(&raw))
2651            .transpose()
2652            .map_err(|error| format!("invalid persisted lease state: {error}"))?;
2653        self.schema_floor = self
2654            .get_meta(SCHEMA_FLOOR_KEY)?
2655            .map(|raw| serde_json::from_str(&raw))
2656            .transpose()
2657            .map_err(|error| format!("invalid persisted schema floor: {error}"))?;
2658        self.stopped = self.schema_floor.is_some();
2659        // A replica left in unactivated preflight reopens gated: the quarantine
2660        // decision persists with the data across handle teardown and restart.
2661        if self.get_meta(SECURITY_PREFLIGHT_PENDING_KEY)?.as_deref() == Some("1") {
2662            self.security_preflight = true;
2663        }
2664        Ok(())
2665    }
2666
2667    fn save_unit_scope_schema(&self) -> Result<(), String> {
2668        let declarations: BTreeMap<_, BTreeMap<_, _>> = self
2669            .schema
2670            .tables
2671            .iter()
2672            .map(|table| {
2673                (
2674                    table.name.clone(),
2675                    table
2676                        .scope_variables
2677                        .iter()
2678                        .map(|scope| {
2679                            (
2680                                scope.variable.clone(),
2681                                [scope.prefix.clone(), scope.column.clone()],
2682                            )
2683                        })
2684                        .collect(),
2685                )
2686            })
2687            .collect();
2688        self.set_meta(
2689            "subscriptionScopeSchema",
2690            &serde_json::to_string(&declarations).expect("scope schema serialization"),
2691        )?;
2692        let generations: BTreeMap<_, _> = self
2693            .schema
2694            .tables
2695            .iter()
2696            .filter_map(|table| Some((&table.name, table.generation.as_ref()?)))
2697            .collect();
2698        self.set_meta(
2699            TABLE_GENERATIONS_KEY,
2700            &serde_json::to_string(&generations).expect("generations serialize"),
2701        )
2702    }
2703
2704    /// Drop units that cover no table of the running schema, with their window
2705    /// registry rows and pending evictions; never translate scopes. At a bump
2706    /// (§7.4.3) a unit also needs every variable unchanged in prefix and column
2707    /// on every table it covered before.
2708    fn prune_unknown_units(&mut self, bump: bool) -> Result<(), String> {
2709        let previous: Option<BTreeMap<String, BTreeMap<String, [String; 2]>>> = if bump {
2710            self.get_meta("subscriptionScopeSchema")?
2711                .map(|raw| serde_json::from_str(&raw))
2712                .transpose()
2713                .map_err(|_| "sync.local_corrupt: invalid persisted scope schema".to_owned())?
2714        } else {
2715            None
2716        };
2717        // A table keeps the variables' meaning when each has the same prefix
2718        // and column as before. An old binary stored no declaration evidence;
2719        // matching names cannot prove unchanged meaning, so the app registers
2720        // again.
2721        let unchanged = |name: &str, variables: &[&String]| {
2722            let (Some(old), Some(table)) = (
2723                previous.as_ref().and_then(|previous| previous.get(name)),
2724                self.schema.table(name),
2725            ) else {
2726                return false;
2727            };
2728            variables.iter().all(|variable| {
2729                table
2730                    .scope_variables
2731                    .iter()
2732                    .find(|scope| &scope.variable == *variable)
2733                    .is_some_and(|scope| {
2734                        old.get(*variable) == Some(&[scope.prefix.clone(), scope.column.clone()])
2735                    })
2736            })
2737        };
2738        let compatible = |scopes: &UnitScopes| {
2739            if !self.schema.tables.iter().any(|table| covers(table, scopes)) {
2740                return false;
2741            }
2742            if !bump {
2743                return true;
2744            }
2745            let Some(previous) = &previous else {
2746                return false;
2747            };
2748            previous.iter().all(|(name, old)| {
2749                !scopes.keys().all(|variable| old.contains_key(variable))
2750                    || unchanged(name, &scopes.keys().collect::<Vec<_>>())
2751            })
2752        };
2753        let mut stale: BTreeSet<String> = self
2754            .stored(load_units(&self.conn))?
2755            .into_iter()
2756            .filter(|unit| !compatible(&unit.scopes))
2757            .map(|unit| unit.id)
2758            .collect();
2759        // A shrunk window's deferred eviction outlives its unit row and keeps
2760        // its per-table scopes. A table the schema no longer has cannot be
2761        // evicted, and at a bump a table whose scope declarations changed must
2762        // not be evicted by the outdated rule.
2763        for (id, table, effective) in self.load_pending_evictions()? {
2764            if self.schema.table(&table).is_none()
2765                || (bump
2766                    && !unchanged(
2767                        &table,
2768                        &effective
2769                            .iter()
2770                            .map(|(variable, _)| variable)
2771                            .collect::<Vec<_>>(),
2772                    ))
2773            {
2774                stale.insert(id);
2775            }
2776        }
2777        for id in &stale {
2778            for sql in [
2779                "DELETE FROM _syncular_windows WHERE sub_id = ?1",
2780                "DELETE FROM _syncular_pending_evictions WHERE unit_id = ?1",
2781                "DELETE FROM _syncular_units WHERE id = ?1",
2782            ] {
2783                self.conn
2784                    .execute(sql, rusqlite::params![id])
2785                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
2786            }
2787        }
2788        Ok(())
2789    }
2790
2791    pub fn local_revision(&self) -> Result<u64, String> {
2792        Ok(self
2793            .get_meta(LOCAL_REVISION_KEY)?
2794            .and_then(|value| value.parse().ok())
2795            .unwrap_or(0))
2796    }
2797
2798    /// §7.5: resolve a snapshot read on the mutable owner connection through
2799    /// the same primitive the read-only sidecar uses, so the command surface
2800    /// and the file sidecar return one revision, rows, coverage, catch-up, and
2801    /// delivery shape. The structured [`QueryReadFailure`] is preserved; an
2802    /// `owner` records its failure (and its next success) in diagnostics.
2803    pub fn snapshot_read(
2804        &mut self,
2805        request: &SnapshotReadRequest<'_>,
2806        owner: Option<QueryOwner<'_>>,
2807    ) -> Result<SnapshotRead, QueryReadFailure> {
2808        let result = snapshot_read_connection(&self.conn, request);
2809        if let Some(owner) = owner {
2810            self.record_query_read(owner, result.as_ref().err());
2811        }
2812        result
2813    }
2814
2815    #[must_use]
2816    pub fn status_snapshot(&self) -> SyncStatusSnapshot {
2817        SyncStatusSnapshot {
2818            current_schema_version: self.schema.version,
2819            outbox: self.outbox.len(),
2820            upgrading: self.upgrading,
2821            lease_state: self.lease_state.clone(),
2822            schema_floor: self.schema_floor.clone(),
2823            sync_needed: self.sync_needed,
2824            previous_version_context: self.previous_version_status(),
2825            security_lifecycle: self.security_lifecycle(),
2826        }
2827    }
2828
2829    /// RFC 0005 A2: container presence for a host that must act on it (Diego's
2830    /// rollback precondition), read without creating the file.
2831    fn previous_version_status(&self) -> PreviousVersionStatus {
2832        let Some(replica_path) = self.previous_version_replica_path() else {
2833            return PreviousVersionStatus {
2834                present: false,
2835                created_at_ms: None,
2836            };
2837        };
2838        match crate::previous_version::read_previous_version_record(&replica_path) {
2839            Ok(Some(record)) => PreviousVersionStatus {
2840                present: true,
2841                created_at_ms: Some(record.created_at_ms),
2842            },
2843            _ => PreviousVersionStatus {
2844                present: std::path::Path::new(
2845                    &crate::previous_version::previous_version_container_path(&replica_path),
2846                )
2847                .exists(),
2848                created_at_ms: None,
2849            },
2850        }
2851    }
2852
2853    /// RFC 0005 D7: read captured rows from the previous local schema. `state`
2854    /// is always `previousVersion`; the container never makes
2855    /// `query_snapshot()` coverage complete. Lease stop/expiry, scope
2856    /// revocation, coverage completion and the TTL DISCARD the container and
2857    /// close the read with their own reason.
2858    pub fn previous_version_snapshot(
2859        &self,
2860        spec: &PreviousVersionReadSpec,
2861    ) -> Result<PreviousVersionSnapshot, String> {
2862        let now_ms = self.clock_now_ms();
2863        crate::previous_version::previous_version_snapshot(
2864            &self.conn,
2865            self.previous_version_replica_path().as_deref(),
2866            self.previous_version.as_ref(),
2867            PreviousVersionLifecycle {
2868                lease_inactive: self.previous_version_lease_inactive(now_ms),
2869                scope_revoked: self
2870                    .stored(load_units(&self.conn))?
2871                    .iter()
2872                    .any(|unit| unit.status == UnitStatus::Revoked),
2873                coverage_complete: self.previous_version_coverage_complete()?,
2874            },
2875            now_ms,
2876            self.schema.version,
2877            spec,
2878        )
2879    }
2880
2881    /// RFC 0005 D6: the advisory pre-reset compatibility audit.
2882    pub fn previous_version_audit(
2883        &self,
2884    ) -> Result<Option<crate::previous_version::PreviousVersionAudit>, String> {
2885        crate::previous_version::stored_previous_version_audit(&self.conn)
2886            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2887    }
2888
2889    /// RFC 0005 A2/D9: the executable downgrade step. Drop the container file
2890    /// and both metadata records, and report whether anything was present.
2891    /// Idempotent: absence is a successful no-op.
2892    pub fn previous_version_discard(
2893        &self,
2894    ) -> Result<crate::previous_version::PreviousVersionDiscardOutcome, String> {
2895        self.drop_previous_version()
2896    }
2897
2898    fn drop_previous_version(
2899        &self,
2900    ) -> Result<crate::previous_version::PreviousVersionDiscardOutcome, String> {
2901        let Some(replica_path) = self.previous_version_replica_path() else {
2902            return Ok(crate::previous_version::PreviousVersionDiscardOutcome {
2903                present: false,
2904                discarded: false,
2905            });
2906        };
2907        crate::previous_version::discard_previous_version(&self.conn, &replica_path)
2908    }
2909
2910    /// RFC 0005 D7: `leaseState.errorCode` or an expired lease.
2911    fn previous_version_lease_inactive(&self, now_ms: i64) -> bool {
2912        let Some(lease) = &self.lease_state else {
2913            return false;
2914        };
2915        if lease.error_code.is_some() {
2916            return true;
2917        }
2918        lease.expires_at_ms.is_some_and(|expires| expires <= now_ms)
2919    }
2920
2921    /// RFC 0005 D7: §7.4.5 replacement coverage — no active unit is
2922    /// bootstrapping. An empty active set is not completion (mirrors the TS
2923    /// predicate exactly).
2924    fn previous_version_coverage_complete(&self) -> Result<bool, String> {
2925        let units = self.stored(load_units(&self.conn))?;
2926        let mut active = units
2927            .iter()
2928            .filter(|unit| unit.status == UnitStatus::Active)
2929            .peekable();
2930        Ok(active.peek().is_some() && !active.any(UnitRecord::bootstrapping))
2931    }
2932
2933    /// RFC 0005 D7: lifetime triggers evaluated once per sync round, after the
2934    /// apply. A container that no longer has a reason to exist is discarded so
2935    /// the rows do not outlive the window they were kept for.
2936    fn previous_version_lifetime_check(&self) {
2937        let Some(config) = self.previous_version.filter(|config| config.enabled) else {
2938            return;
2939        };
2940        let now_ms = self.clock_now_ms();
2941        if self.previous_version_lease_inactive(now_ms) {
2942            let _ = self.drop_previous_version();
2943            return;
2944        }
2945        let Some(replica_path) = self.previous_version_replica_path() else {
2946            return;
2947        };
2948        let Ok(Some(record)) = crate::previous_version::read_previous_version_record(&replica_path)
2949        else {
2950            return;
2951        };
2952        let Ok(units) = load_units(&self.conn) else {
2953            return;
2954        };
2955        if units.iter().any(|unit| unit.status == UnitStatus::Revoked) {
2956            let _ = self.drop_previous_version();
2957            return;
2958        }
2959        if now_ms - record.created_at_ms > config.max_age_ms {
2960            let _ = self.drop_previous_version();
2961            return;
2962        }
2963        if self.previous_version_coverage_complete() == Ok(true) {
2964            let _ = self.drop_previous_version();
2965        }
2966    }
2967
2968    pub fn diagnostics_snapshot(&self) -> Result<ClientDiagnosticsSnapshot, String> {
2969        let registered = self.stored(load_units(&self.conn))?;
2970        // Both meta values in one statement: the FFI host recomputes
2971        // diagnostics after every command.
2972        let (local_revision, replica_cursor): (Option<String>, Option<String>) = self.stored(
2973            self.conn
2974                .prepare_cached(
2975                    "SELECT (SELECT value FROM _syncular_meta WHERE key = ?1),
2976                            (SELECT value FROM _syncular_meta WHERE key = ?2)",
2977                )
2978                .and_then(|mut statement| {
2979                    statement.query_row([LOCAL_REVISION_KEY, REPLICA_CURSOR_KEY], |row| {
2980                        Ok((row.get(0)?, row.get(1)?))
2981                    })
2982                }),
2983        )?;
2984        let mut units = DiagnosticUnits::default();
2985        for unit in &registered {
2986            *match unit.status {
2987                UnitStatus::Revoked => &mut units.revoked,
2988                UnitStatus::Failed => &mut units.failed,
2989                UnitStatus::Active if !unit.bootstrapping() => &mut units.complete,
2990                UnitStatus::Active
2991                    if unit.reason_code.as_deref() == Some("sync.cursor_expired") =>
2992                {
2993                    &mut units.reset
2994                }
2995                UnitStatus::Active => &mut units.bootstrapping,
2996            } += 1;
2997        }
2998        units.reason_codes = registered
2999            .iter()
3000            .filter_map(|unit| unit.reason_code.as_deref().map(Self::diagnostic_code))
3001            .collect::<BTreeSet<_>>()
3002            .into_iter()
3003            .collect();
3004        let captured_at_ms = self.clock_now_ms();
3005        let lease = if let Some(error_code) = self
3006            .lease_state
3007            .as_ref()
3008            .and_then(|state| state.error_code.clone())
3009        {
3010            ClientDiagnosticsLease {
3011                state: "stopped".to_owned(),
3012                expires_at_ms: self
3013                    .lease_state
3014                    .as_ref()
3015                    .and_then(|state| state.expires_at_ms),
3016                error_code: Some(Self::diagnostic_code(&error_code)),
3017            }
3018        } else if let Some(expires_at_ms) = self
3019            .lease_state
3020            .as_ref()
3021            .and_then(|state| state.expires_at_ms)
3022        {
3023            ClientDiagnosticsLease {
3024                state: if expires_at_ms <= captured_at_ms {
3025                    "expired".to_owned()
3026                } else {
3027                    "active".to_owned()
3028                },
3029                expires_at_ms: Some(expires_at_ms),
3030                error_code: None,
3031            }
3032        } else {
3033            ClientDiagnosticsLease {
3034                state: "none".to_owned(),
3035                expires_at_ms: None,
3036                error_code: None,
3037            }
3038        };
3039        let connectivity = match self.last_round.as_ref() {
3040            _ if !self.transport_enabled && self.active_round.is_none() => "offline",
3041            Some(round) if round.status == "succeeded" => "online",
3042            Some(round)
3043                if round.status == "failed"
3044                    && matches!(
3045                        round.cause_kind,
3046                        Some(
3047                            crate::TransportCauseKind::Network | crate::TransportCauseKind::Connect
3048                        )
3049                    ) =>
3050            {
3051                "offline"
3052            }
3053            Some(round) if round.status == "failed" && round.http_status.is_some() => "online",
3054            _ => "unknown",
3055        };
3056        Ok(ClientDiagnosticsSnapshot {
3057            version: CLIENT_DIAGNOSTICS_VERSION,
3058            captured_at_ms,
3059            host: ClientDiagnosticsHost {
3060                kind: "direct".to_owned(),
3061                role: "single".to_owned(),
3062                connectivity: connectivity.to_owned(),
3063                realtime: self.realtime_state,
3064                realtime_policy: self.realtime_policy,
3065                realtime_reason_code: self.realtime_reason_code.clone(),
3066                realtime_retry_delay_ms: self.realtime_retry_delay_ms,
3067            },
3068            security_lifecycle: self.security_lifecycle().to_owned(),
3069            schema: ClientDiagnosticsSchema {
3070                current_version: self.schema.version,
3071                upgrading: self.upgrading,
3072                required_version: self
3073                    .schema_floor
3074                    .as_ref()
3075                    .and_then(|floor| floor.required_schema_version),
3076                latest_version: self
3077                    .schema_floor
3078                    .as_ref()
3079                    .and_then(|floor| floor.latest_schema_version),
3080            },
3081            replica: ClientDiagnosticsReplica {
3082                local_revision: local_revision
3083                    .and_then(|value| value.parse::<u64>().ok())
3084                    .unwrap_or(0)
3085                    .to_string(),
3086                sync_needed: self.sync_needed,
3087                pending_outbox: self.outbox.len(),
3088            },
3089            lease,
3090            active_units: registered
3091                .iter()
3092                .filter(|unit| unit.status == UnitStatus::Active)
3093                .count(),
3094            units,
3095            replica_cursor: replica_cursor.map_or(Ok(-1), |raw| {
3096                raw.parse()
3097                    .map_err(|_| "sync.local_corrupt: persisted unit state is invalid".to_owned())
3098            })?,
3099            last_round: self.last_round.clone(),
3100            last_change: self.last_change.clone(),
3101            storage: self.diagnostics_storage(),
3102            query_failures: self.query_failures.clone(),
3103        })
3104    }
3105
3106    fn diagnostics_storage(&self) -> ClientDiagnosticsStorage {
3107        let read = || -> Result<ClientDiagnosticsStorage, rusqlite::Error> {
3108            let page_count: i64 = self
3109                .conn
3110                .query_row("PRAGMA page_count", [], |row| row.get(0))?;
3111            let page_size: i64 = self
3112                .conn
3113                .query_row("PRAGMA page_size", [], |row| row.get(0))?;
3114            // One cached aggregate, since the FFI host recomputes diagnostics
3115            // after every command; every invocation still reads current storage.
3116            let (outbox_bytes, outcome_entries, outcome_bytes): (i64, i64, i64) = self
3117                .conn
3118                .prepare_cached(
3119                    "SELECT (SELECT COALESCE(SUM(LENGTH(ops_json)), 0) FROM _syncular_outbox),
3120                            COUNT(*), COALESCE(SUM(LENGTH(results_json) + COALESCE(LENGTH(operations_json), 0)), 0)
3121                       FROM _syncular_commit_outcomes",
3122                )?
3123                .query_row([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))?;
3124            let blob_bytes = if self.schema_has_blobs() {
3125                self.conn
3126                    .prepare_cached("SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs")?
3127                    .query_row([], |row| row.get(0))?
3128            } else {
3129                0
3130            };
3131            let pressure = self
3132                .limits
3133                .blob_cache_max_bytes
3134                .is_some_and(|limit| blob_bytes > limit);
3135            Ok(ClientDiagnosticsStorage {
3136                status: if pressure { "pressure" } else { "healthy" }.to_owned(),
3137                database_bytes_approx: Some(page_count.saturating_mul(page_size).max(0)),
3138                pending_outbox_bytes_approx: Some(outbox_bytes.max(0)),
3139                retained_outcome_bytes_approx: Some(outcome_bytes.max(0)),
3140                retained_outcome_entries: Some(outcome_entries.max(0)),
3141                blob_cache_bytes_approx: Some(blob_bytes.max(0)),
3142                pressure_reason_code: pressure.then(|| "client.blob_cache_over_limit".to_owned()),
3143            })
3144        };
3145        read().unwrap_or_else(|_| ClientDiagnosticsStorage {
3146            status: "unreadable".to_owned(),
3147            database_bytes_approx: None,
3148            pending_outbox_bytes_approx: None,
3149            retained_outcome_bytes_approx: None,
3150            retained_outcome_entries: None,
3151            blob_cache_bytes_approx: None,
3152            pressure_reason_code: None,
3153        })
3154    }
3155
3156    pub fn drain_change_batches(&mut self) -> Vec<ClientChangeBatch> {
3157        self.change_queue.drain(..).collect()
3158    }
3159
3160    pub fn drain_sync_intents(&mut self) -> Vec<SyncIntent> {
3161        let intents = self.sync_intent_queue.drain(..).collect();
3162        if self.transport_enabled {
3163            intents
3164        } else {
3165            vec![SyncIntent::None]
3166        }
3167    }
3168
3169    fn schedule_background_retry(&mut self) -> u64 {
3170        let delay_ms = self.retry_delay_ms;
3171        self.sync_intent_queue
3172            .push_back(SyncIntent::Background { delay_ms });
3173        self.retry_delay_ms = (self.retry_delay_ms * 2).min(30_000);
3174        self.round_retry_delay_ms = Some(delay_ms);
3175        delay_ms
3176    }
3177
3178    fn reset_background_retry(&mut self) {
3179        self.retry_delay_ms = 250;
3180    }
3181
3182    fn retryable_transport_code(code: &str) -> bool {
3183        code == "transport.failed"
3184            || code == "transport.timeout"
3185            || code == "transport.unavailable"
3186            || code == "sync.transport_failed"
3187    }
3188
3189    /// §1.8: the transport reported a refused or unreachable endpoint. The
3190    /// native transport says `connect` (DNS failure, connection refused,
3191    /// reset, or aborted); a browser fetch transport says `network`. Repeating
3192    /// the request on a backoff repairs neither, so no retry is scheduled.
3193    fn unreachable_transport(details: &Option<serde_json::Value>) -> bool {
3194        matches!(
3195            details
3196                .as_ref()
3197                .and_then(|details| details.get("causeKind"))
3198                .and_then(serde_json::Value::as_str),
3199            Some("connect" | "network")
3200        )
3201    }
3202
3203    /// §1.5: a segment fetch that fails inside an otherwise complete round is
3204    /// invisible to the server (the round response arrived whole), so its code
3205    /// rides the following requests like a socket-round failure. A code the
3206    /// frame grammar rejects is not reportable and the previous code stays.
3207    fn segment_fetch_failed(&mut self, error: TransportError) -> SectionError {
3208        if ssp2::model::valid_round_failure_code(&error.code) {
3209            self.round_failure = Some(error.code.clone());
3210        }
3211        SectionError::Transport(error)
3212    }
3213
3214    /// A failed round worth a background retry: a transport failure, or a
3215    /// §10.2 code the catalog marks retryable (clients may hardcode the
3216    /// catalog metadata, §10.1). Matches the TypeScript core, which retries
3217    /// on the server's `retryable` flag.
3218    fn retryable_failure_code(code: &str) -> bool {
3219        Self::retryable_transport_code(code)
3220            || matches!(
3221                code,
3222                "sync.auth_required"
3223                    | "sync.auth_lease_required"
3224                    | "sync.auth_lease_revoked"
3225                    | "sync.internal_error"
3226                    | "sync.idempotency_cache_miss"
3227                    | "sync.schema_not_ready"
3228                    | "sync.segment_expired"
3229                    | "sync.rate_limited"
3230                    | "sync.websocket_connection_limit"
3231                    | "sync.idle_timeout"
3232            )
3233    }
3234
3235    fn diagnostic_code(code: &str) -> String {
3236        let valid = !code.is_empty()
3237            && code.len() <= 96
3238            && code.contains('.')
3239            && code
3240                .bytes()
3241                .next()
3242                .is_some_and(|byte| byte.is_ascii_lowercase())
3243            && code.bytes().all(|byte| {
3244                byte.is_ascii_lowercase()
3245                    || byte.is_ascii_digit()
3246                    || matches!(byte, b'.' | b'_' | b'-')
3247            });
3248        if valid {
3249            code.to_owned()
3250        } else {
3251            "client.unknown_failure".to_owned()
3252        }
3253    }
3254
3255    fn set_sync_needed(&mut self, value: bool, interactive: bool) {
3256        if self.sync_needed != value {
3257            if self.begin_observation("syncular_status").is_ok() {
3258                self.sync_needed = value;
3259                let batch = ChangeAccumulator {
3260                    status: true,
3261                    ..ChangeAccumulator::default()
3262                };
3263                if self.finish_observation("syncular_status", batch).is_err() {
3264                    self.rollback_observation("syncular_status");
3265                }
3266            } else {
3267                self.sync_needed = value;
3268            }
3269        }
3270        if value && interactive {
3271            self.sync_intent_queue.push_back(SyncIntent::Interactive);
3272        }
3273    }
3274
3275    /// A unit-store result: a row that does not parse is `sync.local_corrupt`;
3276    /// every other SQLite error keeps its storage classification.
3277    fn stored<T>(&self, result: rusqlite::Result<T>) -> Result<T, String> {
3278        result.map_err(|error| match error {
3279            rusqlite::Error::FromSqlConversionFailure(..) => {
3280                "sync.local_corrupt: persisted unit state is invalid".to_owned()
3281            }
3282            error => Self::sqlite_failure(&self.storage_failure, error),
3283        })
3284    }
3285
3286    fn sqlite_failure(
3287        storage_failure: &RefCell<Option<QueryReadFailure>>,
3288        error: rusqlite::Error,
3289    ) -> String {
3290        let message = error.to_string();
3291        let failure = QueryReadFailure::from(error);
3292        let code = failure.code;
3293        if code.is_some() && storage_failure.borrow().is_none() {
3294            *storage_failure.borrow_mut() = Some(failure);
3295        }
3296        match code {
3297            Some(code) => format!("{code}: {message}"),
3298            None => message,
3299        }
3300    }
3301
3302    fn begin_observation(&self, name: &str) -> Result<(), String> {
3303        if let Some(failure) = &self.coalesced_failure {
3304            return Err(failure.clone());
3305        }
3306        // SQLite rolls a whole transaction back on some I/O and capacity
3307        // errors; later frames must not commit on their own after that.
3308        if self.coalesced.is_some() && self.conn.is_autocommit() {
3309            return Err("the coalesced apply transaction was rolled back".to_owned());
3310        }
3311        self.conn
3312            .execute_batch(&format!("SAVEPOINT {name}"))
3313            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
3314    }
3315
3316    /// SPEC §1.4 rule 4: open a coalesced apply transaction unless a caller
3317    /// already owns one.
3318    fn begin_coalesced_apply(&mut self) -> Result<(), String> {
3319        if self.coalesced.is_some() || !self.conn.is_autocommit() {
3320            return Ok(());
3321        }
3322        self.conn
3323            .execute_batch("BEGIN")
3324            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3325        self.coalesced = Some(CoalescedApply {
3326            rows: 0,
3327            rollback: self.capture(),
3328        });
3329        Ok(())
3330    }
3331
3332    fn capture(&self) -> Rollback {
3333        Rollback {
3334            outbox: self.outbox.clone(),
3335            failed_commits: self.failed_commits.clone(),
3336            conflicts: self.conflicts.clone(),
3337            rejections: self.rejections.clone(),
3338            schema_floor: self.schema_floor.clone(),
3339            lease_state: self.lease_state.clone(),
3340            stopped: self.stopped,
3341            upgrading: self.upgrading,
3342            sync_needed: self.sync_needed,
3343            active_round: self.active_round,
3344            overlay_dirty: self.overlay_dirty.snapshot(),
3345            change_queue: self.change_queue.len(),
3346            sync_intent_queue: self.sync_intent_queue.len(),
3347            last_change: self.last_change.clone(),
3348        }
3349    }
3350
3351    fn restore(&mut self, rollback: Rollback) {
3352        self.outbox = rollback.outbox;
3353        self.failed_commits = rollback.failed_commits;
3354        self.conflicts = rollback.conflicts;
3355        self.rejections = rollback.rejections;
3356        self.schema_floor = rollback.schema_floor;
3357        self.lease_state = rollback.lease_state;
3358        self.stopped = rollback.stopped;
3359        self.upgrading = rollback.upgrading;
3360        self.sync_needed = rollback.sync_needed;
3361        self.active_round = rollback.active_round;
3362        self.overlay_dirty.restore(rollback.overlay_dirty);
3363        self.change_queue.truncate(rollback.change_queue);
3364        self.sync_intent_queue.truncate(rollback.sync_intent_queue);
3365        self.last_change = rollback.last_change;
3366    }
3367
3368    /// Account the rows of the next frame or block. A unit that would take the
3369    /// coalesced transaction past the bound commits the earlier units first.
3370    fn coalesce_unit(&mut self, rows: usize) -> Result<(), String> {
3371        let Some(open) = &self.coalesced else {
3372            return Ok(());
3373        };
3374        if open.rows > 0 && open.rows + rows > COALESCED_APPLY_ROWS {
3375            self.commit_coalesced_apply()?;
3376            self.begin_coalesced_apply()?;
3377        }
3378        if let Some(open) = &mut self.coalesced {
3379            open.rows += rows;
3380        }
3381        Ok(())
3382    }
3383
3384    /// COMMIT the coalesced transaction. A failed COMMIT rolls every coalesced
3385    /// frame back, restores the in-memory state of its `BEGIN`, drops the
3386    /// change batches and sync intents queued since, and fails the rest of the
3387    /// response.
3388    fn commit_coalesced_apply(&mut self) -> Result<(), String> {
3389        let Some(open) = self.coalesced.take() else {
3390            return Ok(());
3391        };
3392        let committed = if self.conn.is_autocommit() {
3393            Err("the coalesced apply transaction was rolled back".to_owned())
3394        } else {
3395            self.conn
3396                .execute_batch("COMMIT")
3397                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
3398        };
3399        let Err(error) = committed else {
3400            return Ok(());
3401        };
3402        if !self.conn.is_autocommit() {
3403            if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
3404                let _ = Self::sqlite_failure(&self.storage_failure, rollback);
3405            }
3406        }
3407        self.restore(open.rollback);
3408        self.coalesced_failure = Some(error.clone());
3409        Err(error)
3410    }
3411
3412    /// End the response's coalescing: COMMIT what is open and clear a failure.
3413    fn finish_coalesced_apply(&mut self) -> Result<(), String> {
3414        let committed = self.commit_coalesced_apply();
3415        match self.coalesced_failure.take() {
3416            Some(failure) => Err(failure),
3417            None => committed,
3418        }
3419    }
3420
3421    fn rollback_observation(&self, name: &str) {
3422        if let Err(error) = self
3423            .conn
3424            .execute_batch(&format!("ROLLBACK TO {name}; RELEASE {name}"))
3425        {
3426            if let Some(failure) = self.storage_failure.borrow_mut().as_mut() {
3427                if failure.rollback_failure.is_none() {
3428                    failure.rollback_failure = Some(Box::new(QueryReadFailure::from(error)));
3429                }
3430            }
3431        }
3432    }
3433
3434    fn finish_observation(
3435        &mut self,
3436        name: &str,
3437        mut batch: ChangeAccumulator,
3438    ) -> Result<(), String> {
3439        #[cfg(feature = "bench-internals")]
3440        let _phase = self.benchmark_phases.start(Phase::ObservationCommit);
3441        if self.failed_commits.iter().any(|(commit, _)| {
3442            commit
3443                .ops
3444                .iter()
3445                .any(|operation| batch.tables.contains_key(&operation.table))
3446        }) {
3447            batch.outcomes = true;
3448        }
3449        if !batch.touched() {
3450            self.conn
3451                .execute_batch(&format!("RELEASE {name}"))
3452                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3453            return Ok(());
3454        }
3455        let revision = self
3456            .local_revision()?
3457            .checked_add(1)
3458            .ok_or_else(|| "local revision exhausted u64".to_owned())?;
3459        self.conn
3460            .execute(
3461                "INSERT OR REPLACE INTO _syncular_meta(key, value) VALUES (?1, ?2)",
3462                rusqlite::params![LOCAL_REVISION_KEY, revision.to_string()],
3463            )
3464            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3465        let status = batch.status.then(|| self.status_snapshot());
3466        let event = ClientChangeBatch {
3467            revision: revision.to_string(),
3468            tables: batch
3469                .tables
3470                .into_iter()
3471                .map(|(table, scope_keys)| TableChange {
3472                    table,
3473                    scope_keys: scope_keys.map(|keys| keys.into_iter().collect()),
3474                })
3475                .collect(),
3476            windows: batch
3477                .windows
3478                .into_iter()
3479                .map(|(base_key, units)| WindowChange {
3480                    base_key,
3481                    units: units.into_iter().collect(),
3482                })
3483                .collect(),
3484            status,
3485            conflicts_changed: batch.conflicts,
3486            rejections_changed: batch.rejections,
3487            outcomes_changed: batch.outcomes,
3488            applied_commits: Vec::new(),
3489        };
3490        self.conn
3491            .execute_batch(&format!("RELEASE {name}"))
3492            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3493        let mut diagnostic_tables = event
3494            .tables
3495            .iter()
3496            .map(|entry| entry.table.clone())
3497            .collect::<BTreeSet<_>>()
3498            .into_iter()
3499            .collect::<Vec<_>>();
3500        // §7.6: the selected variables of the changed window bases.
3501        let mut diagnostic_windows = event
3502            .windows
3503            .iter()
3504            .map(|entry| {
3505                entry
3506                    .base_key
3507                    .split('\0')
3508                    .next()
3509                    .unwrap_or_default()
3510                    .to_owned()
3511            })
3512            .collect::<BTreeSet<_>>()
3513            .into_iter()
3514            .collect::<Vec<_>>();
3515        let domains_truncated = diagnostic_tables.len() > MAX_DIAGNOSTIC_DOMAINS
3516            || diagnostic_windows.len() > MAX_DIAGNOSTIC_DOMAINS;
3517        diagnostic_tables.truncate(MAX_DIAGNOSTIC_DOMAINS);
3518        diagnostic_windows.truncate(MAX_DIAGNOSTIC_DOMAINS);
3519        self.last_change = Some(DiagnosticLastChange {
3520            revision: event.revision.clone(),
3521            recorded_at_ms: self.clock_now_ms(),
3522            tables: diagnostic_tables,
3523            windows: diagnostic_windows,
3524            domains_truncated,
3525            status_changed: event.status.is_some(),
3526            conflicts_changed: event.conflicts_changed,
3527            rejections_changed: event.rejections_changed,
3528            outcomes_changed: event.outcomes_changed,
3529        });
3530        self.change_queue.push_back(event);
3531        Ok(())
3532    }
3533
3534    fn record_scope_map(
3535        &self,
3536        batch: &mut ChangeAccumulator,
3537        table_name: &str,
3538        scopes: &[(String, Vec<String>)],
3539    ) {
3540        let Some(table) = self.schema.table(table_name) else {
3541            return;
3542        };
3543        // A bulk scoped delete whose map leaves a declared variable unnamed
3544        // cannot list the deleted rows' keys for it: the change is table-wide.
3545        if table.scope_variables.iter().any(|scope| {
3546            !scopes
3547                .iter()
3548                .any(|(variable, _)| *variable == scope.variable)
3549        }) {
3550            batch.table(table_name);
3551            return;
3552        }
3553        for (variable, values) in scopes {
3554            let Some(scope) = table
3555                .scope_variables
3556                .iter()
3557                .find(|scope| &scope.variable == variable)
3558            else {
3559                continue;
3560            };
3561            for value in values {
3562                batch.scope(table_name, format!("{}:{value}", scope.prefix));
3563            }
3564        }
3565    }
3566
3567    /// Record a row's current scope keys from the base or visible table.
3568    fn record_row_scopes(
3569        &self,
3570        batch: &mut ChangeAccumulator,
3571        table_name: &str,
3572        row_id: &str,
3573        base: bool,
3574    ) -> bool {
3575        let Some(table) = self.schema.table(table_name) else {
3576            return false;
3577        };
3578        if table.scope_variables.is_empty() {
3579            return false;
3580        }
3581        let columns = table
3582            .scope_variables
3583            .iter()
3584            .map(|scope| quote_ident(&scope.column))
3585            .collect::<Vec<_>>()
3586            .join(", ");
3587        let full_table = if base {
3588            base_table(table_name)
3589        } else {
3590            visible_table(table_name)
3591        };
3592        let sql = format!(
3593            "SELECT {columns} FROM {full_table} WHERE {} LIMIT 1",
3594            row_id_predicate(table)
3595        );
3596        let Ok(mut stmt) = self.conn.prepare_cached(&sql) else {
3597            return false;
3598        };
3599        let values = stmt.query_row(rusqlite::params![row_id], |row| {
3600            let mut values = Vec::with_capacity(table.scope_variables.len());
3601            for index in 0..table.scope_variables.len() {
3602                values.push(row.get::<_, Option<String>>(index)?);
3603            }
3604            Ok(values)
3605        });
3606        let Ok(values) = values else {
3607            return false;
3608        };
3609        let mut recorded = false;
3610        for (scope, value) in table.scope_variables.iter().zip(values) {
3611            if let Some(value) = value {
3612                batch.scope(table_name, format!("{}:{value}", scope.prefix));
3613                recorded = true;
3614            }
3615        }
3616        recorded
3617    }
3618
3619    fn record_commit_changes(
3620        &self,
3621        batch: &mut ChangeAccumulator,
3622        tables: &[String],
3623        changes: &[ssp2::model::Change],
3624    ) {
3625        #[cfg(feature = "bench-internals")]
3626        let _phase = self.benchmark_phases.start(Phase::ObservationPrepare);
3627        for change in changes {
3628            let Some(table_name) = tables.get(change.table_index as usize) else {
3629                continue;
3630            };
3631            let mut precise = self.record_row_scopes(batch, table_name, &change.row_id, true);
3632            if let Some(table) = self.schema.table(table_name) {
3633                for (variable, value) in &change.scopes {
3634                    if let Some(scope) = table
3635                        .scope_variables
3636                        .iter()
3637                        .find(|scope| &scope.variable == variable)
3638                    {
3639                        batch.scope(table_name, format!("{}:{value}", scope.prefix));
3640                        precise = true;
3641                    }
3642                }
3643            }
3644            if !precise {
3645                batch.table(table_name);
3646            }
3647        }
3648    }
3649
3650    /// Whether a base or visible row matches the scope predicate; `false`
3651    /// without a local scope-column mapping. A SQLite failure is an error.
3652    fn scoped_rows_exist(
3653        &self,
3654        table_name: &str,
3655        effective: &[(String, Vec<String>)],
3656        owners: &[Vec<(String, Vec<String>)>],
3657    ) -> Result<bool, String> {
3658        let Some(table) = self.schema.table(table_name) else {
3659            return Ok(false);
3660        };
3661        let Ok(Some((predicate, params))) = local_scope_predicate(table, effective, owners) else {
3662            return Ok(false);
3663        };
3664        for source in [base_table(table_name), visible_table(table_name)] {
3665            if self
3666                .conn
3667                .query_row(
3668                    &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {predicate})"),
3669                    rusqlite::params_from_iter(&params),
3670                    |row| row.get::<_, bool>(0),
3671                )
3672                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
3673            {
3674                return Ok(true);
3675            }
3676        }
3677        Ok(false)
3678    }
3679
3680    /// §5.6 first-page clear of `unit_id`'s pair: delete its rows except
3681    /// those another owner covers (§3.3), failing closed without a local
3682    /// scope-column mapping. Owners are read only when a row matches, inside
3683    /// the caller's transaction. Returns whether rows matched.
3684    fn clear_scope(
3685        &mut self,
3686        table: &TableSchema,
3687        effective: &[(String, Vec<String>)],
3688        unit_id: &str,
3689    ) -> Result<bool, SectionError> {
3690        let storage = |message| SectionError::Abort("storage.failed".to_owned(), message);
3691        let matched = self
3692            .scoped_rows_exist(&table.name, effective, &[])
3693            .map_err(storage)?;
3694        let owners = if matched {
3695            unit_owners(
3696                &self.stored(load_units(&self.conn)).map_err(storage)?,
3697                table,
3698                unit_id,
3699            )
3700        } else {
3701            Vec::new()
3702        };
3703        let changed = matched
3704            && (owners.is_empty()
3705                || self
3706                    .scoped_rows_exist(&table.name, effective, &owners)
3707                    .map_err(storage)?);
3708        if !self
3709            .purge_scope_rows(&table.name, effective, &owners)
3710            .map_err(storage)?
3711        {
3712            return Err(SectionError::FailClosed);
3713        }
3714        Ok(changed)
3715    }
3716
3717    /// §7.4.5: true while a schema-bump reset + first re-bootstrap runs.
3718    pub fn upgrading(&self) -> bool {
3719        self.upgrading
3720    }
3721
3722    /// §7.4.2 "app ships new code": swap to a NEW generated schema while
3723    /// keeping this client's local database (identity, outbox, tables). The
3724    /// §7.4.1 marker check then fires the wipe/re-bootstrap flow when the
3725    /// version increases. Recreation runs the same marker guard as opening
3726    /// a durable replica, holds the guard and every write it authorizes under
3727    /// one write transaction, and preserves the current client (compiled
3728    /// schema, outbox, tables) on refusal.
3729    pub fn recreate_with_schema(&mut self, schema_json: &Value) -> Result<(), String> {
3730        let new_schema = parse_schema_json(schema_json)?;
3731        validate_authority_reads(&new_schema, &self.limits.authority_reads)?;
3732        self.conn
3733            .execute_batch("BEGIN IMMEDIATE")
3734            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3735        let prior_schema = std::mem::replace(&mut self.schema, new_schema);
3736        // §7.4.3/§7.4.4: a reset inside this transaction mutates in-memory
3737        // state (dropped outbox commits, rejections, upgrade and floor flags,
3738        // overlay dirtiness) and appends change/intent batches. A COMMIT that
3739        // fails after the reset succeeded rolls the database back, so every
3740        // one of those fields is captured here and restored with it.
3741        let rollback = self.capture();
3742        let guarded = (|| -> Result<(), String> {
3743            // §7.4.2: the marker is re-read under the write lock the caller
3744            // holds, so a replica another process upgraded meanwhile refuses
3745            // this recreation instead of having its marker and tables
3746            // downgraded.
3747            let marker = read_local_schema_version(&self.conn, self.schema.version)?;
3748            self.create_bookkeeping_tables()?;
3749            match marker {
3750                Some(version) if version < self.schema.version => self.run_schema_reset()?,
3751                None => {
3752                    self.create_synced_tables()?;
3753                    self.persist_schema_version()?;
3754                }
3755                _ => {}
3756            }
3757            // §7.4.2: recreation IS the boot for the in-memory core, so reload the
3758            // durable outbox the same-version path is about to replay. Only the
3759            // outbox is reloaded: the recovery below reads it, and the caller's
3760            // rollback snapshot already covers this field.
3761            self.outbox = self.load_outbox()?.into();
3762            // §7.1/§7.4.4: recover intrinsically codec-invalid persisted values at
3763            // this guarded boundary once the schema and marker agree. A bump went
3764            // through `run_schema_reset`, which performs the same recovery after
3765            // it creates the new physical tables and before its final replay.
3766            self.drop_unencodable_outbox(false, None)?;
3767            if marker == Some(self.schema.version)
3768                && (!self.outbox.is_empty()
3769                    || !self.failed_commits.is_empty()
3770                    || self.has_acknowledged_rows()?)
3771            {
3772                self.overlay_dirty.set(true);
3773                self.rebuild_overlay()?;
3774            }
3775            self.prune_unknown_units(false)?;
3776            self.save_unit_scope_schema()?;
3777            // The conformance recreate is the in-memory equivalent of reopening
3778            // a durable client. Apply the same startup catch-up contract even
3779            // when the schema itself did not change.
3780            self.enqueue_startup_sync_if_needed()?;
3781            Ok(())
3782        })();
3783        let outcome = match guarded {
3784            Ok(()) => self
3785                .conn
3786                .execute_batch("COMMIT")
3787                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error)),
3788            Err(error) => Err(error),
3789        };
3790        if let Err(error) = outcome {
3791            if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
3792                // Keep a failed rollback's typed storage classification.
3793                let _ = Self::sqlite_failure(&self.storage_failure, rollback);
3794            }
3795            self.schema = prior_schema;
3796            self.restore(rollback);
3797            // Prepared insert statements were derived from the reverted schema.
3798            self.insert_sql.borrow_mut().clear();
3799            return Err(error);
3800        }
3801        if !self.security_preflight {
3802            self.drain_pending_evictions()?;
3803        }
3804        Ok(())
3805    }
3806
3807    fn enqueue_startup_sync_if_needed(&mut self) -> Result<(), String> {
3808        let startup_work = !self.stopped
3809            && (!self.outbox.is_empty()
3810                || self.has_acknowledged_rows()?
3811                || !self.load_pending_evictions()?.is_empty()
3812                || self
3813                    .stored(load_units(&self.conn))?
3814                    .iter()
3815                    .any(|unit| unit.status == UnitStatus::Active));
3816        if startup_work {
3817            self.sync_needed = true;
3818            self.sync_intent_queue.push_back(SyncIntent::Interactive);
3819        }
3820        Ok(())
3821    }
3822
3823    /// A native client persists schema-floor stops across process restarts.
3824    /// Once the running app already satisfies that floor, the persisted stop
3825    /// is only stale evidence from an older server round (for example, the
3826    /// server was restarted after catching up to an app that was ahead).
3827    /// Clear it and let the normal startup pull re-negotiate. If the server is
3828    /// still incompatible it will return the floor again in that first round.
3829    fn clear_satisfied_persisted_schema_floor(&mut self) -> Result<(), String> {
3830        let satisfied = self
3831            .schema_floor
3832            .as_ref()
3833            .and_then(|floor| floor.required_schema_version)
3834            .is_some_and(|required| self.schema.version >= required);
3835        if !satisfied {
3836            return Ok(());
3837        }
3838        self.delete_meta(SCHEMA_FLOOR_KEY)?;
3839        self.schema_floor = None;
3840        self.stopped = false;
3841        Ok(())
3842    }
3843
3844    /// RFC 0005 D1: discard an orphan or stale container at boot. The reset
3845    /// path already swept it; this covers a same-version or fresh-install open,
3846    /// which runs no reset and therefore no sweep — plus the D7 TTL, which only
3847    /// an aware binary can apply.
3848    fn reconcile_previous_version_at_boot(&self) -> Result<(), String> {
3849        let Some(replica_path) = self.previous_version_replica_path() else {
3850            return Ok(());
3851        };
3852        let max_age_ms = self
3853            .previous_version
3854            .filter(|config| config.enabled)
3855            .map(|config| config.max_age_ms);
3856        reconcile_previous_version_at_boot(
3857            &self.conn,
3858            &replica_path,
3859            self.schema.version,
3860            max_age_ms,
3861            self.clock_now_ms(),
3862        )
3863    }
3864
3865    /// RFC 0005 D3: the sibling container path, or `None` when the replica has
3866    /// no file — an in-memory replica returns an empty path, and a `file:` URI
3867    /// has no plain sibling path — so the feature resolves as off there.
3868    fn previous_version_replica_path(&self) -> Option<String> {
3869        let path = self.conn.path()?;
3870        if path.is_empty() || path.starts_with("file:") {
3871            return None;
3872        }
3873        Some(path.to_owned())
3874    }
3875
3876    /// §7.4.3 reset: whole-database local reset EXCEPT the outbox, clientId,
3877    /// and leaseState. Drops/recreates every synced table from the new
3878    /// schema, resets unit sync state (keeping registrations), clears
3879    /// the schema-floor stop state, rewrites the marker, drops outbox commits
3880    /// that cannot re-encode (§7.4.4), and replays the survivors on top.
3881    fn run_schema_reset(&mut self) -> Result<(), String> {
3882        self.begin_observation("syncular_schema_reset")?;
3883        let rollback = self.capture();
3884        let mut batch = ChangeAccumulator::default();
3885        let result = self
3886            .run_schema_reset_observed(&mut batch, true)
3887            .and_then(|()| self.finish_observation("syncular_schema_reset", batch));
3888        if let Err(error) = result {
3889            self.rollback_observation("syncular_schema_reset");
3890            self.restore(rollback);
3891            return Err(error);
3892        }
3893        Ok(())
3894    }
3895
3896    fn run_log_epoch_reset(&mut self, log_epoch: &str) -> Result<Vec<String>, String> {
3897        if self.get_meta(LOG_EPOCH_KEY)?.is_none() {
3898            self.begin_observation("syncular_epoch_acquisition")?;
3899            let rollback = self.capture();
3900            if let Err(error) = self.conn.execute(
3901                "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
3902                rusqlite::params![LOG_EPOCH_KEY, log_epoch],
3903            ) {
3904                self.rollback_observation("syncular_epoch_acquisition");
3905                return Err(Self::sqlite_failure(&self.storage_failure, error));
3906            }
3907            self.sync_needed = true;
3908            let batch = ChangeAccumulator {
3909                status: true,
3910                ..ChangeAccumulator::default()
3911            };
3912            if let Err(error) = self.finish_observation("syncular_epoch_acquisition", batch) {
3913                self.rollback_observation("syncular_epoch_acquisition");
3914                self.restore(rollback);
3915                return Err(error);
3916            }
3917            self.sync_intent_queue.push_back(SyncIntent::Interactive);
3918            return Ok(Vec::new());
3919        }
3920        let resets = self
3921            .stored(load_units(&self.conn))?
3922            .into_iter()
3923            .map(|unit| unit.id)
3924            .collect();
3925        self.begin_observation("syncular_log_epoch_reset")?;
3926        let rollback = self.capture();
3927        let mut batch = ChangeAccumulator::default();
3928        let result = self
3929            .run_schema_reset_observed(&mut batch, false)
3930            .and_then(|()| {
3931                self.conn
3932                    .execute(
3933                        "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
3934                        rusqlite::params![LOG_EPOCH_KEY, log_epoch],
3935                    )
3936                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3937                self.sync_needed = true;
3938                batch.status = true;
3939                self.finish_observation("syncular_log_epoch_reset", batch)
3940            });
3941        if let Err(error) = result {
3942            self.rollback_observation("syncular_log_epoch_reset");
3943            self.restore(rollback);
3944            return Err(error);
3945        }
3946        self.sync_intent_queue.push_back(SyncIntent::Interactive);
3947        Ok(resets)
3948    }
3949
3950    fn run_schema_reset_observed(
3951        &mut self,
3952        batch: &mut ChangeAccumulator,
3953        capture: bool,
3954    ) -> Result<(), String> {
3955        // §7.4.2: the reset is the only path that wipes local tables for a
3956        // version bump, and it rewrites the marker last. Re-read the marker
3957        // under this reset's transaction: a replica another process upgraded
3958        // meanwhile refuses here instead of being reset to this client's
3959        // older schema.
3960        read_local_schema_version(&self.conn, self.schema.version)?;
3961        // §7.4.3: a schema bump keeps every table whose generation equals the
3962        // persisted one; the log-epoch reset keeps nothing. The previous
3963        // declaration evidence decides which units cover only kept tables.
3964        let mut kept = BTreeSet::new();
3965        let mut previous_declarations: Option<BTreeMap<String, BTreeMap<String, Value>>> = None;
3966        if capture {
3967            let persisted: BTreeMap<String, String> = self
3968                .get_meta(TABLE_GENERATIONS_KEY)?
3969                .map(|raw| serde_json::from_str(&raw))
3970                .transpose()
3971                .map_err(|_| "sync.local_corrupt: invalid persisted table generations".to_owned())?
3972                .unwrap_or_default();
3973            kept = self
3974                .schema
3975                .tables
3976                .iter()
3977                .filter(|table| {
3978                    table.generation.is_some()
3979                        && table.generation.as_ref() == persisted.get(&table.name)
3980                })
3981                .map(|table| table.name.clone())
3982                .collect();
3983            previous_declarations = self
3984                .get_meta("subscriptionScopeSchema")?
3985                .map(|raw| serde_json::from_str(&raw))
3986                .transpose()
3987                .map_err(|_| "sync.local_corrupt: invalid persisted scope schema".to_owned())?;
3988        }
3989        // Kept tables keep both halves, their FTS projections with the FTS5
3990        // shadow tables and id mapping, and their row-level bookkeeping.
3991        let mut kept_objects = BTreeSet::new();
3992        for table in self
3993            .schema
3994            .tables
3995            .iter()
3996            .filter(|table| kept.contains(&table.name))
3997        {
3998            kept_objects.insert(table.name.clone());
3999            kept_objects.insert(format!("_syncular_base_{}", table.name));
4000            for index in &table.fts_indexes {
4001                kept_objects.insert(index.name.clone());
4002                for shadow in ["data", "idx", "content", "docsize", "config"] {
4003                    kept_objects.insert(format!("{}_{shadow}", index.name));
4004                }
4005            }
4006        }
4007        let kept_json = serde_json::to_string(&kept).expect("table names serialize");
4008        for sql in [
4009            "DELETE FROM _syncular_acked_rows WHERE tbl NOT IN (SELECT value FROM json_each(?1))",
4010            "DELETE FROM _syncular_row_deliveries WHERE tbl NOT IN (SELECT value FROM json_each(?1))",
4011        ] {
4012            self.conn
4013                .execute(sql, [&kept_json])
4014                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4015        }
4016        self.cancel_sync_round();
4017        self.upgrading = true;
4018        batch.status = true;
4019        // RFC 0005 D6: classify the pending outbox against the NEW compiled
4020        // schema BEFORE the wipe, and record the advisory audit. This pass
4021        // drops nothing, so the audit sees the original pending intent; the
4022        // §7.4.4 send-time drop and the intrinsic value recovery later in this
4023        // reset are the paths that remove a commit. The classification is
4024        // shared with the send-time drop so the two cannot disagree about what
4025        // is incompatible.
4026        if self.previous_version.is_some_and(|config| config.enabled) {
4027            let previous_version = self
4028                .get_meta(LOCAL_SCHEMA_VERSION_KEY)?
4029                .and_then(|value| value.parse::<i32>().ok())
4030                .unwrap_or(self.schema.version);
4031            let pending = self
4032                .outbox
4033                .iter()
4034                .map(|commit| PendingCommitAudit {
4035                    commit_id: &commit.client_commit_id,
4036                    operations: commit_audit_operations(commit),
4037                })
4038                .collect::<Vec<_>>();
4039            crate::previous_version::write_previous_version_audit(
4040                &self.conn,
4041                &crate::previous_version::build_previous_version_audit(
4042                    &self.schema,
4043                    &pending,
4044                    previous_version,
4045                    self.schema.version,
4046                    self.clock_now_ms(),
4047                ),
4048            )?;
4049        }
4050        // RFC 0005 D5: the container lives in its own FILE, so the reset can
4051        // never reach it. Sweep first (unconditional — also on the log-epoch
4052        // reset, which captures nothing), then capture BEFORE the wipe drops
4053        // any row. A refused capture records its reason and leaves no file.
4054        if let Some(replica_path) = self.previous_version_replica_path() {
4055            sweep_previous_version_container(&self.conn, &replica_path)?;
4056            if capture {
4057                if let Some(config) = self.previous_version.filter(|config| config.enabled) {
4058                    let previous_version = self
4059                        .get_meta(LOCAL_SCHEMA_VERSION_KEY)?
4060                        .and_then(|value| value.parse::<i32>().ok())
4061                        .unwrap_or(self.schema.version);
4062                    capture_previous_version_from_replica(
4063                        &self.conn,
4064                        &replica_path,
4065                        previous_version,
4066                        self.schema.version,
4067                        &config,
4068                        self.clock_now_ms(),
4069                    )?;
4070                }
4071            }
4072        }
4073        for table in self
4074            .schema
4075            .tables
4076            .iter()
4077            .filter(|table| !kept.contains(&table.name))
4078        {
4079            batch.table(&table.name);
4080        }
4081        for (base_key, value) in self.load_window_registry()? {
4082            batch.window(&base_key, &value);
4083        }
4084        if capture {
4085            self.prune_unknown_units(true)?;
4086        }
4087        // The per-table insert SQL is derived from the OLD column lists.
4088        self.insert_sql.borrow_mut().clear();
4089        self.overlay_dirty.set(true);
4090        // Drop every synced table (base + visible) that currently exists —
4091        // discovered from sqlite_master so a bump that adds/removes tables is
4092        // handled. Bookkeeping tables (`_syncular_outbox/_units/_meta`
4093        // and the blob cache) are preserved; base tables are `_syncular_base_*`
4094        // so they are matched explicitly, not by the bookkeeping filter.
4095        // Drop virtual tables first so SQLite removes every FTS shadow table
4096        // atomically instead of the generic discovery tearing it apart.
4097        let virtual_tables: Vec<String> = {
4098            let mut stmt = self
4099                .conn
4100                .prepare(
4101                    "SELECT name FROM sqlite_master WHERE type = 'table' AND sql LIKE 'CREATE VIRTUAL TABLE%'",
4102                )
4103                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4104            let rows = stmt
4105                .query_map([], |row| row.get::<_, String>(0))
4106                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4107            rows.filter_map(Result::ok)
4108                .filter(|name| is_synced_table_name(name) && !kept_objects.contains(name))
4109                .collect()
4110        };
4111        for name in virtual_tables {
4112            self.conn
4113                .execute_batch(&format!(
4114                    "DROP TABLE IF EXISTS {}",
4115                    quote_ident(&format!("_syncular_fts_{name}"))
4116                ))
4117                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4118            self.conn
4119                .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
4120                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4121        }
4122        let existing: Vec<String> = {
4123            let mut stmt = self
4124                .conn
4125                .prepare("SELECT name FROM sqlite_master WHERE type = 'table'")
4126                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4127            let rows = stmt
4128                .query_map([], |row| row.get::<_, String>(0))
4129                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4130            rows.filter_map(Result::ok)
4131                .filter(|name| is_synced_table_name(name) && !kept_objects.contains(name))
4132                .collect()
4133        };
4134        for name in &existing {
4135            if let Some(table) = name.strip_prefix("_syncular_base_") {
4136                batch.table(table);
4137            } else if !name.starts_with("_syncular_") {
4138                batch.table(name);
4139            }
4140        }
4141        for name in existing {
4142            self.conn
4143                .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
4144                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4145        }
4146        // Recreate the synced tables from the NEW schema.
4147        self.create_synced_tables()?;
4148        // Reset the sync state of every unit that covers a table the reset
4149        // did not keep, under the previous declarations or the new schema,
4150        // keeping the registration.
4151        let keep_units = match &previous_declarations {
4152            Some(previous) if !kept.is_empty() => self
4153                .stored(load_units(&self.conn))?
4154                .into_iter()
4155                .filter(|unit| {
4156                    previous.iter().all(|(name, old)| {
4157                        kept.contains(name) || !unit.scopes.keys().all(|v| old.contains_key(v))
4158                    }) && self
4159                        .schema
4160                        .tables
4161                        .iter()
4162                        .all(|table| kept.contains(&table.name) || !covers(table, &unit.scopes))
4163                })
4164                .map(|unit| unit.id)
4165                .collect(),
4166            _ => BTreeSet::new(),
4167        };
4168        reset_units(&self.conn, &keep_units)
4169            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4170        // The stop state is over: this client now ships a servable schema.
4171        self.stopped = false;
4172        self.schema_floor = None;
4173        self.delete_meta(SCHEMA_FLOOR_KEY)?;
4174        // Rewrite the marker LAST so a crash mid-reset re-runs the reset.
4175        self.persist_schema_version()?;
4176        self.save_unit_scope_schema()?;
4177        // §7.1/§7.4.4: the previous-version capture and audit above saw the
4178        // original pending intent; the synced tables now carry the new schema, so
4179        // recover intrinsically codec-invalid persisted values here, before the
4180        // final replay. A pending upsert that names a column the new schema
4181        // removed stays pending for the send-time `drop_incompatible_outbox`
4182        // prepass. The reset wrapper snapshots and restores the outbox,
4183        // rejections, and change queues, so a failed commit rolls this recovery
4184        // back with the rest of the reset.
4185        // §7.4.4: the enclosing reset observation owns the publication, so the
4186        // recovery contributes its rejection, outcome, and table domains to it.
4187        self.drop_unencodable_outbox(false, Some(batch))?;
4188        // Re-apply the surviving outbox optimistically over the empty tables.
4189        self.rebuild_overlay()?;
4190        Ok(())
4191    }
4192
4193    /// §7.4.4 send-time prepass: a persisted upsert whose values reference a
4194    /// column the current schema lacks (or a removed table) cannot be encoded
4195    /// (`commit_audit_operations` inspects upserts, so a value-free delete stays
4196    /// encodable and is validated by the server), nor can an insert-intent
4197    /// upsert that lacks a non-nullable column (§7.1, the audit's
4198    /// `missing-required-column`). Drop the commit, undo
4199    /// its optimistic projection, and raise a client-local
4200    /// `sync.outbox_incompatible` rejection.
4201    fn drop_incompatible_outbox(&mut self) -> Result<bool, String> {
4202        let incompatible = self
4203            .outbox
4204            .iter()
4205            .filter(|commit| {
4206                first_incompatibility(&self.schema, &commit_audit_operations(commit)).is_some()
4207            })
4208            .cloned()
4209            .collect::<Vec<_>>();
4210        if incompatible.is_empty() {
4211            return Ok(false);
4212        }
4213        self.drop_outbox_commits(
4214            &incompatible,
4215            OUTBOX_INCOMPATIBLE_CODE,
4216            "the persisted commit cannot encode under the current schema",
4217        )?;
4218        Ok(true)
4219    }
4220
4221    /// §7.4.4 incompatible-commit path, also taken by a refused blob upload
4222    /// (§5.9.7) with the server's code: reject every operation of `commits`,
4223    /// remove them from the outbox, and undo their optimistic rows.
4224    fn drop_outbox_commits(
4225        &mut self,
4226        incompatible: &[Arc<OutboxCommit>],
4227        code: &str,
4228        message: &str,
4229    ) -> Result<(), String> {
4230        // §7.4.4: one observation covers the durable outcome writes, the outbox
4231        // removal, the in-memory drop, and the overlay rebuild, so a storage
4232        // fault leaves the replica and the client unchanged and publishes no
4233        // change; a successful drop publishes exactly one batch carrying the
4234        // rejections, outcomes, status, and the tables whose projection changed.
4235        let rollback = self.capture();
4236        let owns_transaction = self.conn.is_autocommit();
4237        self.begin_observation("syncular_outbox_drop")?;
4238        let mut batch = ChangeAccumulator::default();
4239        let dropped = (|| -> Result<(), String> {
4240            let mut rejections = Vec::new();
4241            for commit in incompatible {
4242                let results = commit
4243                    .ops
4244                    .iter()
4245                    .enumerate()
4246                    .map(|(op_index, operation)| {
4247                        let rejection = RejectionRecord {
4248                            client_commit_id: commit.client_commit_id.clone(),
4249                            op_index: op_index as i32,
4250                            code: code.to_owned(),
4251                            message: message.to_owned(),
4252                            retryable: false,
4253                            details: None,
4254                            operation: Some(CommitOperation::from(operation)),
4255                        };
4256                        rejections.push(rejection.clone());
4257                        CommitOperationOutcome::Error { rejection }
4258                    })
4259                    .collect::<Vec<_>>();
4260                self.persist_commit_outcome(
4261                    &commit.client_commit_id,
4262                    CommitOutcomeStatus::Rejected,
4263                    &results,
4264                    Some(&commit.ops),
4265                )?;
4266                self.delete_outbox_persisted(&commit.client_commit_id)?;
4267                // A removed table has no local projection to invalidate; every
4268                // surviving table the dropped commit touched does.
4269                for operation in &commit.ops {
4270                    if self.schema.table(&operation.table).is_some() {
4271                        batch.table(&operation.table);
4272                    }
4273                }
4274            }
4275            self.prune_commit_outcomes()?;
4276            let incompatible_ids = incompatible
4277                .iter()
4278                .map(|commit| commit.client_commit_id.as_str())
4279                .collect::<BTreeSet<_>>();
4280            self.outbox
4281                .retain(|commit| !incompatible_ids.contains(commit.client_commit_id.as_str()));
4282            self.rejections.extend(rejections);
4283            // The dropped commits' purely-optimistic rows are undone: the
4284            // visible projection is re-derived from the surviving base + outbox.
4285            self.overlay_dirty.set(true);
4286            self.rebuild_overlay()?;
4287            batch.status = true;
4288            batch.rejections = true;
4289            batch.outcomes = true;
4290            Ok(())
4291        })();
4292        let result = dropped.and_then(|()| self.finish_observation("syncular_outbox_drop", batch));
4293        if let Err(error) = result {
4294            // An outer RELEASE is the COMMIT. Roll back that transaction
4295            // directly: retrying RELEASE after ROLLBACK TO still needs the
4296            // reader's lock. A nested observation only rolls back its savepoint.
4297            if owns_transaction {
4298                if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
4299                    let mut failure = self.storage_failure.borrow_mut();
4300                    let failure =
4301                        failure.get_or_insert_with(|| QueryReadFailure::from(error.clone()));
4302                    failure.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
4303                }
4304            } else {
4305                self.rollback_observation("syncular_outbox_drop");
4306            }
4307            self.restore(rollback);
4308            return Err(error);
4309        }
4310        Ok(())
4311    }
4312
4313    /// §7.4.3: (re)create the base + visible table pair for every synced
4314    /// table in the CURRENT schema (idempotent — `IF NOT EXISTS`).
4315    fn create_synced_tables(&self) -> Result<(), String> {
4316        for table in &self.schema.tables {
4317            // The base half + the visible half form the synced-table pair. An
4318            // index name is global in SQLite, so the base half's indexes are
4319            // name-prefixed (`_syncular_base_<index>`) to stay distinct.
4320            for (full, index_prefix) in [
4321                (base_table(&table.name), "_syncular_base_"),
4322                (visible_table(&table.name), ""),
4323            ] {
4324                let mut cols: Vec<String> =
4325                    table.columns.iter().map(|c| quote_ident(&c.name)).collect();
4326                cols.push("\"_syncular_version\" INTEGER NOT NULL".to_owned());
4327                cols.extend(
4328                    table
4329                        .sealed_columns()
4330                        .iter()
4331                        .map(|name| format!("{} TEXT", quote_ident(name))),
4332                );
4333                let sql = format!(
4334                    "CREATE TABLE IF NOT EXISTS {full} ({} , PRIMARY KEY ({}))",
4335                    cols.join(", "),
4336                    quote_ident(&table.primary_key)
4337                );
4338                self.conn
4339                    .execute(&sql, [])
4340                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4341                // §5.11: a replica created before sealed apply gains the
4342                // reserved columns in place, on both halves in the same order,
4343                // so data, indexes and the FTS projection stay.
4344                for name in table.sealed_columns() {
4345                    let present: bool = self
4346                        .conn
4347                        .query_row(
4348                            "SELECT EXISTS(SELECT 1 FROM pragma_table_info(?1) WHERE name = ?2)",
4349                            rusqlite::params![format!("{index_prefix}{}", table.name), name],
4350                            |row| row.get(0),
4351                        )
4352                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4353                    if !present {
4354                        self.conn
4355                            .execute(
4356                                &format!(
4357                                    "ALTER TABLE {full} ADD COLUMN {} TEXT",
4358                                    quote_ident(name)
4359                                ),
4360                                [],
4361                            )
4362                            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4363                    }
4364                }
4365                // The reveal at every round start reads the sealed rows of the
4366                // base half through this partial index.
4367                if table.has_encrypted_columns() && !index_prefix.is_empty() {
4368                    self.conn
4369                        .execute(
4370                            &format!(
4371                                "CREATE INDEX IF NOT EXISTS {} ON {full} (\"_sync_sealed_key\") WHERE \"_sync_sealed_key\" IS NOT NULL",
4372                                quote_ident(&format!("_syncular_sealed_{}", table.name))
4373                            ),
4374                            [],
4375                        )
4376                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4377                }
4378                // Local secondary indexes (CREATE INDEX subset). Created on
4379                // both halves so mirror reads hit an index on either. Runs on
4380                // both the initial create and the §7.4.3 reset recreate path.
4381                for index in &table.indexes {
4382                    let unique = if index.unique { "UNIQUE " } else { "" };
4383                    let index_name = quote_ident(&format!("{index_prefix}{}", index.name));
4384                    let cols_sql = index
4385                        .columns
4386                        .iter()
4387                        .map(|c| quote_ident(c))
4388                        .collect::<Vec<_>>()
4389                        .join(", ");
4390                    let index_sql = format!(
4391                        "CREATE {unique}INDEX IF NOT EXISTS {index_name} ON {full} ({cols_sql})"
4392                    );
4393                    self.conn
4394                        .execute(&index_sql, [])
4395                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4396                }
4397            }
4398        }
4399        // FTS exists only on the visible half. It is a local search
4400        // projection, never a synced/base table or a wire-schema member.
4401        for table in &self.schema.tables {
4402            for index in &table.fts_indexes {
4403                self.create_fts_projection(table, index)?;
4404            }
4405        }
4406        Ok(())
4407    }
4408
4409    fn fts_projection_exists(&self, index: &FtsIndexSchema) -> Result<bool, String> {
4410        let count: i64 = self
4411            .conn
4412            .query_row(
4413                "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?1",
4414                rusqlite::params![index.name],
4415                |row| row.get(0),
4416            )
4417            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4418        Ok(count > 0)
4419    }
4420
4421    fn drop_fts_triggers(&self, index: &FtsIndexSchema) -> Result<(), String> {
4422        for suffix in ["ai", "ad", "au"] {
4423            self.conn
4424                .execute(
4425                    &format!(
4426                        "DROP TRIGGER IF EXISTS {}",
4427                        quote_ident(&format!("{}_{suffix}", index.name))
4428                    ),
4429                    [],
4430                )
4431                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4432        }
4433        Ok(())
4434    }
4435
4436    fn create_fts_triggers(
4437        &self,
4438        table: &TableSchema,
4439        index: &FtsIndexSchema,
4440    ) -> Result<(), String> {
4441        let fts = quote_ident(&index.name);
4442        let source = visible_table(&table.name);
4443        let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
4444        let pk = quote_ident(&table.primary_key);
4445        let projection_columns = std::iter::once(source_id.clone())
4446            .chain(index.columns.iter().map(|column| quote_ident(column)))
4447            .collect::<Vec<_>>()
4448            .join(", ");
4449        let new_values = std::iter::once(format!("CAST(new.{pk} AS TEXT)"))
4450            .chain(
4451                index
4452                    .columns
4453                    .iter()
4454                    .map(|column| format!("new.{}", quote_ident(column))),
4455            )
4456            .collect::<Vec<_>>()
4457            .join(", ");
4458        let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
4459        let delete_new = format!("DELETE FROM {fts} WHERE rowid = (SELECT id FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)); DELETE FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)");
4460        let delete_old = format!("DELETE FROM {fts} WHERE rowid = (SELECT id FROM {mapping} WHERE source_id = CAST(old.{pk} AS TEXT)); DELETE FROM {mapping} WHERE source_id = CAST(old.{pk} AS TEXT)");
4461        let insert_new = format!("INSERT INTO {mapping}(source_id) VALUES (CAST(new.{pk} AS TEXT)); INSERT INTO {fts} (rowid, {projection_columns}) VALUES ((SELECT id FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)), {new_values})");
4462        let bi = quote_ident(&format!("{}_bi", index.name));
4463        let ai = quote_ident(&format!("{}_ai", index.name));
4464        let ad = quote_ident(&format!("{}_ad", index.name));
4465        let au = quote_ident(&format!("{}_au", index.name));
4466        let replacement_exists = format!("EXISTS (SELECT 1 FROM {source} WHERE {pk} = new.{pk})");
4467        let sql = format!(
4468            "DROP TRIGGER IF EXISTS {bi};
4469             DROP TRIGGER IF EXISTS {ai};
4470             DROP TRIGGER IF EXISTS {ad};
4471             DROP TRIGGER IF EXISTS {au};
4472             CREATE TRIGGER {bi} BEFORE INSERT ON {source} WHEN {replacement_exists} BEGIN {delete_new}; END;
4473             CREATE TRIGGER {ai} AFTER INSERT ON {source} BEGIN {insert_new}; END;
4474             CREATE TRIGGER {ad} AFTER DELETE ON {source} BEGIN {delete_old}; END;
4475             CREATE TRIGGER {au} AFTER UPDATE ON {source} BEGIN {delete_old}; {delete_new}; {insert_new}; END;",
4476        );
4477        self.conn
4478            .execute_batch(&sql)
4479            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
4480    }
4481
4482    fn rebuild_fts_projection(
4483        &self,
4484        table: &TableSchema,
4485        index: &FtsIndexSchema,
4486    ) -> Result<(), String> {
4487        let fts = quote_ident(&index.name);
4488        let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
4489        let pk = quote_ident(&table.primary_key);
4490        let indexed_columns = index
4491            .columns
4492            .iter()
4493            .map(|column| quote_ident(column))
4494            .collect::<Vec<_>>();
4495        let projection_columns = std::iter::once(source_id.clone())
4496            .chain(indexed_columns.iter().cloned())
4497            .collect::<Vec<_>>()
4498            .join(", ");
4499        let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
4500        let sql = format!(
4501            "DELETE FROM {fts}; DELETE FROM {mapping}; INSERT INTO {fts} ({projection_columns}) SELECT CAST({pk} AS TEXT), {columns} FROM {source}; INSERT INTO {mapping}(id, source_id) SELECT rowid, {source_id} FROM {fts};",
4502            columns = indexed_columns.join(", "),
4503            source = visible_table(&table.name),
4504        );
4505        self.conn
4506            .execute_batch(&sql)
4507            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
4508    }
4509
4510    fn create_fts_projection(
4511        &self,
4512        table: &TableSchema,
4513        index: &FtsIndexSchema,
4514    ) -> Result<(), String> {
4515        self.begin_observation("syncular_fts_schema")?;
4516        let result = (|| {
4517            let existed = self.fts_projection_exists(index)?;
4518            let tokenizer = index.tokenize.replace('\'', "''");
4519            let columns = index
4520                .columns
4521                .iter()
4522                .map(|column| quote_ident(column))
4523                .collect::<Vec<_>>()
4524                .join(", ");
4525            let sql = format!(
4526            "CREATE VIRTUAL TABLE IF NOT EXISTS {fts} USING fts5({source_id} UNINDEXED, {columns}, tokenize='{tokenizer}')",
4527            fts = quote_ident(&index.name),
4528            source_id = quote_ident(FTS_SOURCE_ID_COLUMN),
4529        );
4530            self.conn.execute(&sql, []).map_err(|error| {
4531                format!(
4532                    "cannot create local FTS5 projection {:?}: {error}",
4533                    index.name
4534                )
4535            })?;
4536            let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
4537            let mapped: bool = self
4538                .conn
4539                .query_row(
4540                    "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?1)",
4541                    [format!("_syncular_fts_{}", index.name)],
4542                    |row| row.get(0),
4543                )
4544                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4545            self.conn.execute_batch(&format!("CREATE TABLE IF NOT EXISTS {mapping}(id INTEGER PRIMARY KEY, source_id TEXT NOT NULL UNIQUE)")).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4546            if !mapped && existed {
4547                self.conn
4548                    .execute_batch(&format!(
4549                        "INSERT INTO {mapping}(id, source_id) SELECT rowid, {} FROM {}",
4550                        quote_ident(FTS_SOURCE_ID_COLUMN),
4551                        quote_ident(&index.name)
4552                    ))
4553                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4554            }
4555            self.create_fts_triggers(table, index)?;
4556            if !existed {
4557                self.rebuild_fts_projection(table, index)?;
4558            }
4559            Ok(())
4560        })();
4561        if let Err(error) = result {
4562            self.rollback_observation("syncular_fts_schema");
4563            return Err(error);
4564        }
4565        if let Err(error) = self.conn.execute_batch("RELEASE syncular_fts_schema") {
4566            self.rollback_observation("syncular_fts_schema");
4567            return Err(error.to_string());
4568        }
4569        Ok(())
4570    }
4571
4572    // -- persistence write-through --------------------------------------------
4573
4574    fn persist_outbox_insert(&self, commit: &OutboxCommit) -> Result<(), String> {
4575        let ops: Vec<Value> = commit
4576            .ops
4577            .iter()
4578            .map(|op| {
4579                let mut entry = serde_json::json!({
4580                    "op": if op.upsert { "upsert" } else { "delete" },
4581                    "table": op.table,
4582                    "rowId": op.row_id,
4583                    "baseVersion": op.base_version,
4584                    "values": op.values.clone().map(Value::Object),
4585                });
4586                if op.insert {
4587                    entry["insert"] = Value::Bool(true);
4588                }
4589                entry
4590            })
4591            .collect();
4592        self.conn
4593            .execute(
4594                "INSERT OR REPLACE INTO _syncular_outbox (commit_id, ops_json) VALUES (?1, ?2)",
4595                rusqlite::params![commit.client_commit_id, Value::Array(ops).to_string()],
4596            )
4597            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4598        for operation in &commit.ops {
4599            let Some(values) = &operation.values else {
4600                continue;
4601            };
4602            let Some(table) = self.schema.table(&operation.table) else {
4603                continue;
4604            };
4605            for column in table
4606                .columns
4607                .iter()
4608                .filter(|column| column.ty == ColumnType::BlobRef)
4609            {
4610                let Some(Value::String(raw)) = values.get(&column.name) else {
4611                    continue;
4612                };
4613                let Ok(blob_ref) = serde_json::from_str::<Value>(raw) else {
4614                    continue;
4615                };
4616                let Some(blob_id) = blob_ref.get("blobId").and_then(Value::as_str) else {
4617                    continue;
4618                };
4619                self.conn
4620                    .execute(
4621                        "INSERT OR IGNORE INTO _syncular_blob_commit_refs(commit_id, blob_id)
4622                         SELECT ?1, ?2 WHERE EXISTS (SELECT 1 FROM _syncular_blobs WHERE blob_id = ?2)",
4623                        rusqlite::params![commit.client_commit_id, blob_id],
4624                    )
4625                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4626            }
4627        }
4628        Ok(())
4629    }
4630
4631    fn delete_outbox_persisted(&self, client_commit_id: &str) -> Result<(), String> {
4632        self.conn
4633            .execute(
4634                "DELETE FROM _syncular_blob_commit_refs WHERE commit_id = ?1",
4635                rusqlite::params![client_commit_id],
4636            )
4637            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4638        self.conn
4639            .execute(
4640                "DELETE FROM _syncular_outbox WHERE commit_id = ?1",
4641                rusqlite::params![client_commit_id],
4642            )
4643            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4644        self.conn.execute("DELETE FROM _syncular_row_deliveries WHERE NOT EXISTS(SELECT 1 FROM _syncular_outbox,json_each(ops_json) WHERE json_extract(json_each.value,'$.table')=tbl AND json_extract(json_each.value,'$.rowId')=id)", [])
4645            .map(|_| ())
4646            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
4647    }
4648
4649    fn outcome_status_name(status: CommitOutcomeStatus) -> &'static str {
4650        match status {
4651            CommitOutcomeStatus::Applied => "applied",
4652            CommitOutcomeStatus::Cached => "cached",
4653            CommitOutcomeStatus::Conflict => "conflict",
4654            CommitOutcomeStatus::Rejected => "rejected",
4655        }
4656    }
4657
4658    fn outcome_resolution_name(resolution: CommitOutcomeResolution) -> &'static str {
4659        match resolution {
4660            CommitOutcomeResolution::Active => "active",
4661            CommitOutcomeResolution::ResolvedKeepServer => "resolved_keep_server",
4662            CommitOutcomeResolution::Superseded => "superseded",
4663            CommitOutcomeResolution::Dismissed => "dismissed",
4664        }
4665    }
4666
4667    fn parse_outcome_status(value: &str) -> Result<CommitOutcomeStatus, String> {
4668        match value {
4669            "applied" => Ok(CommitOutcomeStatus::Applied),
4670            "cached" => Ok(CommitOutcomeStatus::Cached),
4671            "conflict" => Ok(CommitOutcomeStatus::Conflict),
4672            "rejected" => Ok(CommitOutcomeStatus::Rejected),
4673            _ => Err(format!("invalid persisted commit outcome status {value:?}")),
4674        }
4675    }
4676
4677    fn parse_outcome_resolution(value: &str) -> Result<CommitOutcomeResolution, String> {
4678        match value {
4679            "active" => Ok(CommitOutcomeResolution::Active),
4680            "resolved_keep_server" => Ok(CommitOutcomeResolution::ResolvedKeepServer),
4681            "superseded" => Ok(CommitOutcomeResolution::Superseded),
4682            "dismissed" => Ok(CommitOutcomeResolution::Dismissed),
4683            _ => Err(format!(
4684                "invalid persisted commit outcome resolution {value:?}"
4685            )),
4686        }
4687    }
4688
4689    fn persist_commit_outcome(
4690        &self,
4691        client_commit_id: &str,
4692        status: CommitOutcomeStatus,
4693        results: &[CommitOperationOutcome],
4694        operations: Option<&[OutboxOp]>,
4695    ) -> Result<(), String> {
4696        let results_json = serde_json::to_string(results).map_err(|error| error.to_string())?;
4697        let operations_json = operations
4698            .map(|items| {
4699                serde_json::to_string(&items.iter().map(CommitOperation::from).collect::<Vec<_>>())
4700            })
4701            .transpose()
4702            .map_err(|error| error.to_string())?;
4703        self.conn
4704            .execute(
4705                "INSERT INTO _syncular_commit_outcomes (
4706                   client_commit_id, status, recorded_at_ms, results_json,
4707                   operations_json, resolution
4708                 ) VALUES (?1, ?2, ?3, ?4, ?5, 'active')",
4709                rusqlite::params![
4710                    client_commit_id,
4711                    Self::outcome_status_name(status),
4712                    self.clock_now_ms(),
4713                    results_json,
4714                    operations_json
4715                ],
4716            )
4717            .map(|_| ())
4718            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
4719    }
4720
4721    /// §7.2: the journal holds a stored cell only in its canonical shape. A cell
4722    /// the byte-envelope rule refuses cannot be journaled verbatim.
4723    fn stored_cell_is_representable(cell: &Value) -> bool {
4724        cell.is_null()
4725            || cell.is_string()
4726            || cell.is_boolean()
4727            || cell.is_number()
4728            || is_canonical_bytes_envelope(cell)
4729    }
4730
4731    fn stored_row_is_representable(value: &Value) -> bool {
4732        value
4733            .as_object()
4734            .is_some_and(|row| row.values().all(Self::stored_cell_is_representable))
4735    }
4736
4737    fn stored_operation_is_representable(value: &Value) -> bool {
4738        value.get("table").is_some_and(Value::is_string)
4739            && value.get("rowId").is_some_and(Value::is_string)
4740            && value.get("baseVersion").is_none_or(|v| {
4741                v.as_i64()
4742                    .is_some_and(|n| n.unsigned_abs() <= MAX_JS_SAFE_INTEGER)
4743            })
4744            && match value.get("op").and_then(Value::as_str) {
4745                Some("upsert") => {
4746                    value
4747                        .get("values")
4748                        .is_some_and(Self::stored_row_is_representable)
4749                        && value
4750                            .get("insert")
4751                            .is_none_or(|insert| *insert == Value::Bool(true))
4752                }
4753                Some("delete") => value.get("values").is_none() && value.get("insert").is_none(),
4754                _ => false,
4755            }
4756    }
4757
4758    /// §7.2/§7.4.4: a dropped commit keeps its operation envelope in the journal
4759    /// only when the journal reader accepts the serialized operation, which is the
4760    /// rule `outcome_from_row` applies to this same row later.
4761    fn outbox_op_is_journalable(operation: &OutboxOp) -> bool {
4762        serde_json::to_value(CommitOperation::from(operation))
4763            .is_ok_and(|value| Self::stored_operation_is_representable(&value))
4764    }
4765
4766    fn outcome_from_row(row: StoredCommitOutcomeRow) -> Result<CommitOutcome, String> {
4767        let StoredCommitOutcomeRow {
4768            sequence,
4769            client_commit_id,
4770            status,
4771            recorded_at_ms,
4772            results_json,
4773            operations_json,
4774            resolution,
4775            resolved_at_ms,
4776            replacement_client_commit_id,
4777        } = row;
4778        if sequence.unsigned_abs() > MAX_JS_SAFE_INTEGER
4779            || recorded_at_ms.unsigned_abs() > MAX_JS_SAFE_INTEGER
4780            || resolved_at_ms.is_some_and(|value| value.unsigned_abs() > MAX_JS_SAFE_INTEGER)
4781        {
4782            return Err("sync.local_corrupt: persisted commit outcome is invalid".to_owned());
4783        }
4784        let invalid = || "sync.local_corrupt: persisted commit outcome is invalid".to_owned();
4785        let stored: Vec<Value> = serde_json::from_str(&results_json).map_err(|_| invalid())?;
4786        for result in &stored {
4787            let nested = match result.get("status").and_then(Value::as_str) {
4788                Some("conflict") => result.get("conflict"),
4789                Some("error") => result.get("rejection"),
4790                _ => None,
4791            };
4792            if nested
4793                .and_then(|record| record.get("operation"))
4794                .is_some_and(|op| !Self::stored_operation_is_representable(op))
4795                || (result.get("status").and_then(Value::as_str) == Some("conflict")
4796                    && nested
4797                        .and_then(|record| record.get("serverRow"))
4798                        .is_some_and(|row| !Self::stored_row_is_representable(row)))
4799            {
4800                return Err(invalid());
4801            }
4802        }
4803        let results: Vec<CommitOperationOutcome> =
4804            serde_json::from_value(Value::Array(stored)).map_err(|_| invalid())?;
4805        let has_conflict = results
4806            .iter()
4807            .any(|result| matches!(result, CommitOperationOutcome::Conflict { .. }));
4808        let has_error = results
4809            .iter()
4810            .any(|result| matches!(result, CommitOperationOutcome::Error { .. }));
4811        let status = Self::parse_outcome_status(&status).map_err(|_| invalid())?;
4812        let redacted = results.is_empty() && operations_json.is_none();
4813        if match status {
4814            CommitOutcomeStatus::Applied | CommitOutcomeStatus::Cached => has_conflict || has_error,
4815            CommitOutcomeStatus::Conflict => !(has_conflict || redacted),
4816            CommitOutcomeStatus::Rejected => has_conflict || !(has_error || redacted),
4817        } {
4818            return Err(invalid());
4819        }
4820        let operations: Option<Vec<CommitOperation>> = operations_json
4821            .map(|raw| {
4822                let stored: Vec<Value> = serde_json::from_str(&raw).map_err(|_| invalid())?;
4823                if !stored.iter().all(Self::stored_operation_is_representable) {
4824                    return Err(invalid());
4825                }
4826                serde_json::from_value(Value::Array(stored)).map_err(|_| invalid())
4827            })
4828            .transpose()?;
4829        for result in &results {
4830            match result {
4831                CommitOperationOutcome::Conflict { conflict }
4832                    if conflict.server_version.unsigned_abs() > MAX_JS_SAFE_INTEGER =>
4833                {
4834                    return Err("sync.local_corrupt: persisted commit outcome is invalid".to_owned())
4835                }
4836                CommitOperationOutcome::Error { rejection } => {
4837                    if let Some(details) = &rejection.details {
4838                        let encoded = serde_json::to_string(details).map_err(|_| {
4839                            "sync.local_corrupt: persisted commit outcome is invalid".to_owned()
4840                        })?;
4841                        RejectionDetails::parse(&encoded).map_err(|_| {
4842                            "sync.local_corrupt: persisted commit outcome is invalid".to_owned()
4843                        })?;
4844                    }
4845                }
4846                _ => {}
4847            }
4848        }
4849        Ok(CommitOutcome {
4850            sequence,
4851            client_commit_id,
4852            status,
4853            recorded_at_ms,
4854            results,
4855            operations,
4856            retained_rows: None,
4857            resolution: Self::parse_outcome_resolution(&resolution)?,
4858            resolved_at_ms,
4859            replacement_client_commit_id,
4860        })
4861    }
4862
4863    /// Resolve declared unique keys through SQLite, including affinity and NULL semantics.
4864    fn unique_conflicts(
4865        &self,
4866        table: &TableSchema,
4867        values: &Map<String, Value>,
4868        base: bool,
4869    ) -> Result<Vec<RetainedUniqueConflict>, String> {
4870        let source = if base {
4871            base_table(&table.name)
4872        } else {
4873            visible_table(&table.name)
4874        };
4875        let mut conflicts = Vec::new();
4876        for index in table.indexes.iter().filter(|index| index.unique) {
4877            let mut params = vec![json_param_to_sql(
4878                values.get(&table.primary_key).ok_or("sync.local_corrupt")?,
4879            )?];
4880            for name in &index.columns {
4881                let column = table
4882                    .columns
4883                    .iter()
4884                    .find(|column| column.name == *name)
4885                    .ok_or("sync.local_corrupt")?;
4886                let cell = json_to_column_value(column, values.get(name))?;
4887                params.push(owned_sql_value(RowParam::Cell(&cell))?);
4888            }
4889            let sql = format!(
4890                "SELECT * FROM {source} WHERE {} != ? AND {}",
4891                quote_ident(&table.primary_key),
4892                index
4893                    .columns
4894                    .iter()
4895                    .map(|column| format!("{} = ?", quote_ident(column)))
4896                    .collect::<Vec<_>>()
4897                    .join(" AND ")
4898            );
4899            let mut statement = self
4900                .conn
4901                .prepare_cached(&sql)
4902                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4903            let records = statement
4904                .query_map(rusqlite::params_from_iter(params), |record| {
4905                    let values = table
4906                        .columns
4907                        .iter()
4908                        .enumerate()
4909                        .map(|(index, column)| {
4910                            let value = sql_ref_to_json(column, record.get_ref(index)?);
4911                            json_to_column_value(column, Some(&value)).map_err(|message| {
4912                                rusqlite::Error::ToSqlConversionFailure(message.into())
4913                            })
4914                        })
4915                        .collect::<Result<Row, _>>()?;
4916                    Ok((values, record.get::<_, i64>(table.columns.len())?))
4917                })
4918                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4919            for record in records {
4920                let (row, version) =
4921                    record.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4922                let row_id = render_row_id_json(Some(&column_value_to_json(&row[table.pk_index])))?;
4923                conflicts.push(RetainedUniqueConflict {
4924                    index: index.name.clone(),
4925                    columns: index.columns.clone(),
4926                    row_id,
4927                    server_row: table
4928                        .columns
4929                        .iter()
4930                        .zip(&row)
4931                        .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
4932                        .collect(),
4933                    server_version: version,
4934                });
4935            }
4936        }
4937        Ok(conflicts)
4938    }
4939
4940    fn attach_retained_rows(&self, mut outcome: CommitOutcome) -> Result<CommitOutcome, String> {
4941        if let Some((commit, initial)) = self
4942            .failed_commits
4943            .iter()
4944            .find(|(commit, _)| commit.client_commit_id == outcome.client_commit_id)
4945        {
4946            let mut rows = Vec::new();
4947            for operation in &commit.ops {
4948                // A table the current schema removed has no rows to retain.
4949                let Some(table) = self.schema.table(&operation.table) else {
4950                    continue;
4951                };
4952                let key =
4953                    serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
4954                        .map_err(|_| "sync.local_corrupt".to_owned())?;
4955                let base = self.stored_row(table, &operation.row_id, true)?;
4956                rows.push(RetainedCommitRow {
4957                    table: operation.table.clone(),
4958                    row_id: operation.row_id.clone(),
4959                    unique_conflicts: if operation.upsert {
4960                        self.unique_conflicts(
4961                            table,
4962                            initial
4963                                .get(&key)
4964                                .and_then(Value::as_object)
4965                                .or(operation.values.as_ref())
4966                                .ok_or("sync.local_corrupt")?,
4967                            true,
4968                        )?
4969                    } else {
4970                        Vec::new()
4971                    },
4972                    local_row: if operation.upsert {
4973                        initial
4974                            .get(&key)
4975                            .and_then(Value::as_object)
4976                            .cloned()
4977                            .or_else(|| operation.values.clone())
4978                    } else {
4979                        None
4980                    },
4981                    server_row: base.as_ref().map(|(values, _)| {
4982                        table
4983                            .columns
4984                            .iter()
4985                            .zip(values)
4986                            .map(|(column, value)| {
4987                                (column.name.clone(), column_value_to_json(value))
4988                            })
4989                            .collect()
4990                    }),
4991                    server_version: base.map(|(_, version)| version),
4992                });
4993            }
4994            outcome.retained_rows = Some(rows);
4995        }
4996        Ok(outcome)
4997    }
4998
4999    pub fn commit_outcome(&self, client_commit_id: &str) -> Result<Option<CommitOutcome>, String> {
5000        let row = self
5001            .conn
5002            .query_row(
5003                "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
5004                        resolution, resolved_at_ms, replacement_client_commit_id
5005                   FROM _syncular_commit_outcomes WHERE client_commit_id = ?1",
5006                rusqlite::params![client_commit_id],
5007                |row| {
5008                    Ok(StoredCommitOutcomeRow {
5009                        sequence: row.get(0)?,
5010                        client_commit_id: row.get(1)?,
5011                        status: row.get(2)?,
5012                        recorded_at_ms: row.get(3)?,
5013                        results_json: row.get(4)?,
5014                        operations_json: row.get(5)?,
5015                        resolution: row.get(6)?,
5016                        resolved_at_ms: row.get(7)?,
5017                        replacement_client_commit_id: row.get(8)?,
5018                    })
5019                },
5020            )
5021            .optional()
5022            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5023        row.map(|row| self.attach_retained_rows(Self::outcome_from_row(row)?))
5024            .transpose()
5025    }
5026
5027    /// Internal host step. The host must run its idle drive before finishing.
5028    #[doc(hidden)]
5029    pub fn prepare_current_base(
5030        &self,
5031        client_commit_id: &str,
5032        sequence: i64,
5033    ) -> Result<CurrentBasePreparation, CurrentBaseProof> {
5034        let outcome = self
5035            .commit_outcome(client_commit_id)
5036            .map_err(|_| CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed))?;
5037        let valid = outcome.as_ref().is_some_and(|outcome| {
5038            outcome.sequence == sequence
5039                && outcome.resolution == CommitOutcomeResolution::Active
5040                && matches!(
5041                    outcome.status,
5042                    CommitOutcomeStatus::Conflict | CommitOutcomeStatus::Rejected
5043                )
5044                && outcome
5045                    .operations
5046                    .as_ref()
5047                    .zip(outcome.retained_rows.as_ref())
5048                    .is_some_and(|(operations, rows)| {
5049                        !rows.is_empty() && operations.len() == rows.len()
5050                    })
5051        });
5052        if !valid {
5053            return Err(CurrentBaseProof::failure(
5054                CurrentBaseProofReason::OutcomeUnavailable,
5055            ));
5056        }
5057        if !self.transport_enabled {
5058            return Err(CurrentBaseProof::failure(CurrentBaseProofReason::Offline));
5059        }
5060        Ok(CurrentBasePreparation {
5061            client_commit_id: client_commit_id.to_owned(),
5062            sequence,
5063            units: load_units(&self.conn)
5064                .map_err(|_| CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed))?,
5065            transport_epoch: self.transport_epoch,
5066            registration_epoch: self.registration_epoch,
5067            security_epoch: self.security_epoch,
5068        })
5069    }
5070
5071    /// Internal host step after the completed post-rejection idle drive.
5072    #[doc(hidden)]
5073    pub fn finish_current_base(
5074        &mut self,
5075        prepared: CurrentBasePreparation,
5076        drive: SyncOutcome,
5077    ) -> CurrentBaseProof {
5078        if !self.transport_enabled || self.transport_epoch != prepared.transport_epoch {
5079            return CurrentBaseProof::failure(CurrentBaseProofReason::Offline);
5080        }
5081        if self.registration_epoch != prepared.registration_epoch {
5082            return CurrentBaseProof::failure(CurrentBaseProofReason::MissingCoverage);
5083        }
5084        if self.security_preflight || self.security_epoch != prepared.security_epoch {
5085            return CurrentBaseProof::failure(CurrentBaseProofReason::SyncFailed);
5086        }
5087        let report = match drive {
5088            SyncOutcome::Ok(report) => report,
5089            SyncOutcome::BudgetExhausted(_) => {
5090                return CurrentBaseProof::failure(CurrentBaseProofReason::BudgetExhausted)
5091            }
5092            _ => return CurrentBaseProof::failure(CurrentBaseProofReason::SyncFailed),
5093        };
5094        if report.schema_floor.is_some() {
5095            return CurrentBaseProof::failure(CurrentBaseProofReason::SchemaFloor);
5096        }
5097        if self
5098            .conn
5099            .execute_batch("SAVEPOINT syncular_base_proof")
5100            .is_err()
5101        {
5102            return CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed);
5103        }
5104        let proof = (|| {
5105            let snapshot = snapshot_read_connection(
5106                &self.conn,
5107                &SnapshotReadRequest {
5108                    units: prepared.units.iter().map(|unit| unit.id.clone()).collect(),
5109                    commit_ids: vec![prepared.client_commit_id.clone()],
5110                    ..Default::default()
5111                },
5112            )
5113            .map_err(|_| CurrentBaseProofReason::SnapshotFailed)?;
5114            let current = self
5115                .commit_outcome(&prepared.client_commit_id)
5116                .map_err(|_| CurrentBaseProofReason::SnapshotFailed)?;
5117            let unavailable = if report.revoked.is_empty() {
5118                CurrentBaseProofReason::OutcomeUnavailable
5119            } else {
5120                CurrentBaseProofReason::Revoked
5121            };
5122            let Some(outcome) = current else {
5123                return Err(unavailable);
5124            };
5125            if !matches!(
5126                snapshot.deliveries.first(),
5127                Some(CommitDelivery::Known { .. })
5128            ) || outcome.sequence != prepared.sequence
5129                || outcome.resolution != CommitOutcomeResolution::Active
5130                || !matches!(
5131                    outcome.status,
5132                    CommitOutcomeStatus::Conflict | CommitOutcomeStatus::Rejected
5133                )
5134            {
5135                return Err(unavailable);
5136            }
5137            let (Some(operations), Some(rows)) = (outcome.operations, outcome.retained_rows) else {
5138                return Err(unavailable);
5139            };
5140            if rows.is_empty() || operations.len() != rows.len() {
5141                return Err(unavailable);
5142            }
5143            let after =
5144                load_units(&self.conn).map_err(|_| CurrentBaseProofReason::SnapshotFailed)?;
5145            let complete: HashSet<&str> = snapshot
5146                .units
5147                .iter()
5148                .filter_map(|unit| match unit {
5149                    UnitCatchup::Known {
5150                        id,
5151                        bootstrap_complete: true,
5152                        ..
5153                    } => Some(id.as_str()),
5154                    _ => None,
5155                })
5156                .collect();
5157            // A unit id names one scope map (§4.1), so the same id before and
5158            // after the drive is the same registration.
5159            let candidates: Vec<&UnitRecord> = after
5160                .iter()
5161                .filter(|unit| {
5162                    complete.contains(unit.id.as_str())
5163                        && prepared.units.iter().any(|before| before.id == unit.id)
5164                })
5165                .collect();
5166            let covered = |table_name: &str, row: &Map<String, Value>| {
5167                let Some(table) = self.schema.table(table_name) else {
5168                    return false;
5169                };
5170                candidates.iter().any(|unit| {
5171                    covers(table, &unit.scopes)
5172                        && pair_scopes(&unit.scopes, table_name, unit.echo.as_deref()).is_some_and(
5173                            |scopes| {
5174                                scopes.iter().all(|(variable, allowed)| {
5175                                    table
5176                                        .scope_column(variable)
5177                                        .and_then(|column| match row.get(column) {
5178                                            Some(Value::String(value)) => Some(value.clone()),
5179                                            Some(Value::Bool(value)) => Some(value.to_string()),
5180                                            Some(Value::Number(value)) => {
5181                                                value.as_f64().map(|number| {
5182                                                    ryu_js::Buffer::new()
5183                                                        .format_finite(number)
5184                                                        .to_owned()
5185                                                })
5186                                            }
5187                                            _ => None,
5188                                        })
5189                                        .is_some_and(|value| {
5190                                            !value.is_empty() && allowed.contains(&value)
5191                                        })
5192                                })
5193                            },
5194                        )
5195                })
5196            };
5197            for row in &rows {
5198                let Some(base) = row.server_row.as_ref().or(row.local_row.as_ref()) else {
5199                    return Err(if report.revoked.is_empty() {
5200                        CurrentBaseProofReason::MissingCoverage
5201                    } else {
5202                        CurrentBaseProofReason::Revoked
5203                    });
5204                };
5205                if !covered(&row.table, base)
5206                    || row
5207                        .unique_conflicts
5208                        .iter()
5209                        .any(|conflict| !covered(&row.table, &conflict.server_row))
5210                {
5211                    return Err(if report.revoked.is_empty() {
5212                        CurrentBaseProofReason::MissingCoverage
5213                    } else {
5214                        CurrentBaseProofReason::Revoked
5215                    });
5216                }
5217            }
5218            Ok(rows)
5219        })();
5220        if self
5221            .conn
5222            .execute_batch("RELEASE syncular_base_proof")
5223            .is_err()
5224        {
5225            return CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed);
5226        }
5227        match proof {
5228            Ok(rows) => CurrentBaseProof {
5229                proven: true,
5230                rows: Some(rows),
5231                reason: None,
5232            },
5233            Err(reason) => CurrentBaseProof::failure(reason),
5234        }
5235    }
5236
5237    pub fn prove_current_base(
5238        &mut self,
5239        transport: &mut dyn Transport,
5240        client_commit_id: &str,
5241        sequence: i64,
5242        max_rounds: Option<u32>,
5243    ) -> CurrentBaseProof {
5244        let prepared = match self.prepare_current_base(client_commit_id, sequence) {
5245            Ok(prepared) => prepared,
5246            Err(proof) => return proof,
5247        };
5248        let drive = self.sync_until_idle(transport, max_rounds);
5249        self.finish_current_base(prepared, drive)
5250    }
5251
5252    pub fn commit_outcomes(&self, query: CommitOutcomeQuery) -> Result<Vec<CommitOutcome>, String> {
5253        if query.limit == Some(0) {
5254            return Err("sync.invalid_request: commit outcome limit must be positive".to_owned());
5255        }
5256        let mut sql = String::from(
5257            "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
5258                    resolution, resolved_at_ms, replacement_client_commit_id
5259               FROM _syncular_commit_outcomes",
5260        );
5261        if query.active_only {
5262            sql.push_str(" WHERE resolution = 'active' AND status IN ('conflict', 'rejected')");
5263        }
5264        sql.push_str(" ORDER BY seq DESC");
5265        if let Some(limit) = query.limit {
5266            sql.push_str(&format!(" LIMIT {limit}"));
5267        }
5268        let mut stmt = self
5269            .conn
5270            .prepare(&sql)
5271            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5272        let rows = stmt
5273            .query_map([], |row| {
5274                Ok(StoredCommitOutcomeRow {
5275                    sequence: row.get(0)?,
5276                    client_commit_id: row.get(1)?,
5277                    status: row.get(2)?,
5278                    recorded_at_ms: row.get(3)?,
5279                    results_json: row.get(4)?,
5280                    operations_json: row.get(5)?,
5281                    resolution: row.get(6)?,
5282                    resolved_at_ms: row.get(7)?,
5283                    replacement_client_commit_id: row.get(8)?,
5284                })
5285            })
5286            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5287        let mut outcomes = Vec::new();
5288        for row in rows {
5289            outcomes.push(self.attach_retained_rows(Self::outcome_from_row(
5290                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
5291            )?)?);
5292        }
5293        Ok(outcomes)
5294    }
5295
5296    fn prune_commit_outcomes(&self) -> Result<(), String> {
5297        #[cfg(test)]
5298        self.outcome_prune_count
5299            .set(self.outcome_prune_count.get() + 1);
5300        let max_entries = self.limits.outcome_retention_max_entries.unwrap_or(1_000);
5301        let count = self
5302            .conn
5303            .query_row(
5304                "SELECT COUNT(*) FROM _syncular_commit_outcomes",
5305                [],
5306                |row| row.get::<_, i64>(0),
5307            )
5308            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
5309            as usize;
5310        let excess = count.saturating_sub(max_entries);
5311        if excess == 0 {
5312            return Ok(());
5313        }
5314        let mut stmt = self
5315            .conn
5316            .prepare(
5317                "SELECT seq FROM _syncular_commit_outcomes
5318                  WHERE status IN ('applied', 'cached') OR resolution != 'active'
5319                  ORDER BY seq ASC LIMIT ?1",
5320            )
5321            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5322        let rows = stmt
5323            .query_map(rusqlite::params![excess as i64], |row| row.get::<_, i64>(0))
5324            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5325        let sequences = rows
5326            .collect::<Result<Vec<_>, _>>()
5327            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5328        drop(stmt);
5329        for sequence in sequences {
5330            self.conn
5331                .execute(
5332                    "DELETE FROM _syncular_commit_outcomes WHERE seq = ?1",
5333                    rusqlite::params![sequence],
5334                )
5335                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5336        }
5337        Ok(())
5338    }
5339
5340    // -- units (§4.1) ------------------------------------------------------------
5341
5342    /// Every registered unit in registration order (§4.1).
5343    pub fn units(&self) -> Result<Vec<UnitRecord>, String> {
5344        self.stored(load_units(&self.conn))
5345    }
5346
5347    pub fn unit(&self, id: &str) -> Result<Option<UnitRecord>, String> {
5348        self.stored(get_unit(&self.conn, id))
5349    }
5350
5351    /// Register one unit (§4.1) and return its id.
5352    pub fn subscribe(&mut self, scopes: UnitScopes) -> Result<String, String> {
5353        Ok(self.subscribe_many(vec![scopes])?.remove(0))
5354    }
5355
5356    /// Register several units in one local transaction and return their ids:
5357    /// one commit for all of them (on OPFS a commit costs milliseconds, so a
5358    /// cold start with hundreds of units pays it once). An error rolls back
5359    /// every registration of the call. Registering an active unit again
5360    /// changes nothing; a revoked unit re-arms (§3.3). A call that registers or
5361    /// re-arms a unit raises sync-needed with an interactive intent.
5362    pub fn subscribe_many(&mut self, units: Vec<UnitScopes>) -> Result<Vec<String>, String> {
5363        self.begin_observation("syncular_subscribe")?;
5364        let mut changed = false;
5365        let registered = (|| {
5366            let mut ids = Vec::with_capacity(units.len());
5367            for scopes in units {
5368                let (id, held) = self.hold_unit(scopes, true)?;
5369                changed |= held;
5370                ids.push(id);
5371            }
5372            self.conn
5373                .execute_batch("RELEASE syncular_subscribe")
5374                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5375            Ok(ids)
5376        })();
5377        let ids = match registered {
5378            Ok(ids) => ids,
5379            Err(error) => {
5380                self.rollback_observation("syncular_subscribe");
5381                return Err(error);
5382            }
5383        };
5384        if changed {
5385            self.registration_epoch = self.registration_epoch.wrapping_add(1);
5386            self.set_sync_needed(true, true);
5387        }
5388        Ok(ids)
5389    }
5390
5391    /// Release the explicit reference to a unit. The unit leaves the
5392    /// registration list when no window base holds it (§4.1); its rows stay.
5393    pub fn unsubscribe(&mut self, id: &str) -> Result<(), String> {
5394        let Some(unit) = self.stored(get_unit(&self.conn, id))? else {
5395            return Ok(());
5396        };
5397        if !unit.explicit {
5398            return Ok(());
5399        }
5400        if !self.stored(window_refs(&self.conn, id))?.is_empty() {
5401            return self.stored(save_unit(
5402                &self.conn,
5403                &UnitRecord {
5404                    explicit: false,
5405                    ..unit
5406                },
5407            ));
5408        }
5409        self.conn
5410            .execute("DELETE FROM _syncular_units WHERE id = ?1", [id])
5411            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5412        self.registration_epoch = self.registration_epoch.wrapping_add(1);
5413        if unit.status == UnitStatus::Active {
5414            self.set_sync_needed(true, true);
5415        }
5416        Ok(())
5417    }
5418
5419    /// Take one reference on a unit (§4.1): validate the scope map, register a
5420    /// new unit catching up with cursor `-1`, re-arm a revoked one (§3.3).
5421    /// Returns the id and whether the registration list changed.
5422    fn hold_unit(&mut self, scopes: UnitScopes, explicit: bool) -> Result<(String, bool), String> {
5423        if scopes.is_empty()
5424            || scopes.values().any(|value| value == "*")
5425            || !self
5426                .schema
5427                .tables
5428                .iter()
5429                .any(|table| covers(table, &scopes))
5430        {
5431            return Err("sync.invalid_subscription: a unit names one non-wildcard value per variable and covers a table".to_owned());
5432        }
5433        let id = ssp2::registration::unit_id(&scope_pairs(&scopes));
5434        let existing = self.stored(get_unit(&self.conn, &id))?;
5435        if let Some(existing) = existing
5436            .as_ref()
5437            .filter(|unit| unit.status != UnitStatus::Revoked)
5438        {
5439            if explicit && !existing.explicit {
5440                self.stored(save_unit(
5441                    &self.conn,
5442                    &UnitRecord {
5443                        explicit,
5444                        ..existing.clone()
5445                    },
5446                ))?;
5447            }
5448            return Ok((id, false));
5449        }
5450        self.delete_pending_evict(&id)?;
5451        self.stored(save_unit(
5452            &self.conn,
5453            &UnitRecord {
5454                id: id.clone(),
5455                scopes,
5456                explicit: explicit || existing.is_some_and(|unit| unit.explicit),
5457                status: UnitStatus::Active,
5458                reason_code: None,
5459                cursor: Some(-1),
5460                bootstrap_state: None,
5461                seeded: Vec::new(),
5462                echo: None,
5463            },
5464        ))?;
5465        Ok((id, true))
5466    }
5467
5468    // -- windows (§4.8) -----------------------------------------------------------
5469
5470    /// §4.8: set the windowed-in values of a base, one unit per value. Added
5471    /// values take a reference on their unit (a new unit bootstraps on the
5472    /// next sync); removed values release theirs, and a unit no reference
5473    /// holds leaves the registration list fused with its first eviction chunk
5474    /// (E1–E4). Idempotent; re-entry cancels a deferred eviction.
5475    pub fn set_window(
5476        &mut self,
5477        base: &WindowBase,
5478        units: &[String],
5479    ) -> Result<CommandEffects, String> {
5480        let base_key = window_base_key(base);
5481        let live = self.load_window_units(&base_key)?;
5482        let mut changed = false;
5483        for value in units {
5484            if live.iter().any(|(held, _)| held == value) {
5485                continue;
5486            }
5487            let rollback = self.capture();
5488            self.begin_observation("syncular_window")?;
5489            let widened = (|| {
5490                let (id, registered) = self.hold_unit(window_unit_scopes(base, value), false)?;
5491                self.sync_needed |= registered;
5492                self.conn
5493                    .execute(
5494                        "INSERT OR REPLACE INTO _syncular_windows(base, unit, sub_id) VALUES (?1, ?2, ?3)",
5495                        rusqlite::params![base_key, value, id],
5496                    )
5497                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5498                let mut batch = ChangeAccumulator::default();
5499                batch.window(&base_key, value);
5500                batch.status = true;
5501                self.finish_observation("syncular_window", batch)
5502            })();
5503            if let Err(error) = widened {
5504                self.rollback_observation("syncular_window");
5505                self.restore(rollback);
5506                return Err(error);
5507            }
5508            changed = true;
5509            self.registration_epoch = self.registration_epoch.wrapping_add(1);
5510        }
5511        for (value, id) in live {
5512            if units.contains(&value) {
5513                continue;
5514            }
5515            self.rebuild_overlay_if_dirty()?;
5516            let rollback = self.capture();
5517            self.begin_observation("syncular_window")?;
5518            let remaining = self.release_window_unit(&base_key, &value, &id);
5519            let remaining = match remaining {
5520                Ok(remaining) => remaining,
5521                Err(error) => {
5522                    self.rollback_observation("syncular_window");
5523                    self.restore(rollback);
5524                    return Err(error);
5525                }
5526            };
5527            changed = true;
5528            self.registration_epoch = self.registration_epoch.wrapping_add(1);
5529            if remaining {
5530                self.drain_pending_evictions()?;
5531            }
5532        }
5533        Ok(if changed && self.transport_enabled {
5534            CommandEffects::interactive()
5535        } else {
5536            CommandEffects::none()
5537        })
5538    }
5539
5540    /// §4.8 completeness oracle: the windowed-in values of a base and those
5541    /// whose unit is not complete. A value is complete once its unit is active
5542    /// and no longer bootstrapping; registration alone is not completeness.
5543    pub fn window_state(&self, base: &WindowBase) -> Result<WindowState, String> {
5544        persisted_window_state(&self.conn, base).map_err(|failure| failure.message)
5545    }
5546
5547    /// Registry rows for a base, ordered by value: (value, unit id).
5548    fn load_window_units(&self, base_key: &str) -> Result<Vec<(String, String)>, String> {
5549        let mut statement = self
5550            .conn
5551            .prepare_cached(
5552                "SELECT unit, sub_id FROM _syncular_windows WHERE base = ?1 ORDER BY unit ASC",
5553            )
5554            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5555        let rows = statement
5556            .query_map([base_key], |row| Ok((row.get(0)?, row.get(1)?)))
5557            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5558        rows.collect::<Result<_, _>>()
5559            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5560    }
5561
5562    /// Every window registry row: (base key, value).
5563    fn load_window_registry(&self) -> Result<Vec<(String, String)>, String> {
5564        let mut statement = self
5565            .conn
5566            .prepare_cached("SELECT base, unit FROM _syncular_windows ORDER BY base, unit")
5567            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5568        let rows = statement
5569            .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))
5570            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5571        rows.collect::<Result<_, _>>()
5572            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5573    }
5574
5575    /// §4.8: release a window value's reference inside the caller's
5576    /// observation. When no reference holds the unit, unregister it fused with
5577    /// its first eviction chunk (E3): delete the rows of every covered table
5578    /// matching the persisted echo, except rows pinned by the outbox (E1) or
5579    /// owned by another unit (§3.3), and park the per-table scopes of what
5580    /// remains in the pending-eviction registry. Fails closed without a local
5581    /// scope-column mapping. Returns whether unpinned rows remain.
5582    fn release_window_unit(
5583        &mut self,
5584        base_key: &str,
5585        value: &str,
5586        id: &str,
5587    ) -> Result<bool, String> {
5588        self.conn
5589            .execute(
5590                "DELETE FROM _syncular_windows WHERE base = ?1 AND unit = ?2",
5591                rusqlite::params![base_key, value],
5592            )
5593            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5594        let mut batch = ChangeAccumulator::default();
5595        batch.window(base_key, value);
5596        batch.status = true;
5597        let unit = self.stored(get_unit(&self.conn, id))?;
5598        let mut remaining = false;
5599        if let Some(unit) = unit.filter(|unit| !unit.explicit) {
5600            if self.stored(window_refs(&self.conn, id))?.is_empty() {
5601                let units = self.stored(load_units(&self.conn))?;
5602                self.conn
5603                    .execute("DELETE FROM _syncular_units WHERE id = ?1", [id])
5604                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5605                self.sync_needed = true;
5606                for table in self.schema.tables.clone() {
5607                    let Some(effective) = unit
5608                        .echo
5609                        .as_deref()
5610                        .filter(|_| covers(&table, &unit.scopes))
5611                        .and_then(|echo| pair_scopes(&unit.scopes, &table.name, Some(echo)))
5612                    else {
5613                        continue;
5614                    };
5615                    let pinned = self.pinned_row_ids(&table.name);
5616                    let owners = unit_owners(&units, &table, id);
5617                    let (more, deferred) =
5618                        self.evict_scope_rows(&table.name, &effective, &owners, &pinned)?;
5619                    remaining |= more;
5620                    if deferred {
5621                        self.save_pending_evict(id, &table.name, &effective)?;
5622                    }
5623                    self.record_scope_map(&mut batch, &table.name, &effective);
5624                }
5625            }
5626        }
5627        self.finish_observation("syncular_window", batch)?;
5628        Ok(remaining)
5629    }
5630
5631    /// §4.8 E1: delete base rows owned by the effective scopes
5632    /// ([`local_scope_predicate`]) EXCEPT pinned primary keys. Returns (more
5633    /// unpinned work, any scoped rows remain). A missing scope-column mapping
5634    /// fails without deleting rows.
5635    fn evict_scope_rows(
5636        &self,
5637        table_name: &str,
5638        effective: &[(String, Vec<String>)],
5639        owners: &[Vec<(String, Vec<String>)>],
5640        pinned: &std::collections::HashSet<String>,
5641    ) -> Result<(bool, bool), String> {
5642        let table = self.schema.table(table_name).ok_or("sync.unknown_table")?;
5643        let Some((scope, mut params)) = local_scope_predicate(table, effective, owners)
5644            .map_err(|()| "sync.scope_revoked".to_owned())?
5645        else {
5646            return Ok((false, false));
5647        };
5648        let scope_params = params.len();
5649        let pk = quote_ident(&table.primary_key);
5650        // Pinned ids are rowId text; the text form matches every key type.
5651        let predicate = if pinned.is_empty() {
5652            scope.clone()
5653        } else {
5654            params.extend(pinned.iter().cloned().map(SqlValue::Text));
5655            let holes = vec!["?"; pinned.len()].join(", ");
5656            format!("{scope} AND CAST({pk} AS TEXT) NOT IN ({holes})")
5657        };
5658        let base = base_table(table_name);
5659        let visible = visible_table(table_name);
5660        let mut ids: Vec<SqlValue> = Vec::new();
5661        for source in [&base, &visible] {
5662            let mut stmt = self
5663                .conn
5664                .prepare_cached(&format!(
5665                    "SELECT {pk} FROM {source} WHERE {predicate} LIMIT 1024"
5666                ))
5667                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5668            ids = stmt
5669                .query_map(rusqlite::params_from_iter(&params), |row| row.get(0))
5670                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
5671                .collect::<Result<_, _>>()
5672                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5673            if !ids.is_empty() {
5674                break;
5675            }
5676        }
5677        if !ids.is_empty() {
5678            for id in &ids {
5679                self.conn
5680                    .execute(
5681                        "DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2",
5682                        rusqlite::params![table_name, id],
5683                    )
5684                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5685            }
5686            let holes = vec!["?"; ids.len()].join(", ");
5687            for source in [&base, &visible] {
5688                self.conn
5689                    .execute(
5690                        &format!("DELETE FROM {source} WHERE {pk} IN ({holes})"),
5691                        rusqlite::params_from_iter(&ids),
5692                    )
5693                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5694            }
5695        }
5696        let mut remaining = false;
5697        let mut deferred = false;
5698        for source in [&base, &visible] {
5699            remaining |= self
5700                .conn
5701                .query_row(
5702                    &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {predicate})"),
5703                    rusqlite::params_from_iter(&params),
5704                    |row| row.get::<_, bool>(0),
5705                )
5706                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5707            deferred |= self
5708                .conn
5709                .query_row(
5710                    &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {scope})"),
5711                    rusqlite::params_from_iter(&params[..scope_params]),
5712                    |row| row.get::<_, bool>(0),
5713                )
5714                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5715        }
5716        Ok((remaining, deferred))
5717    }
5718
5719    /// §4.8 E1: continue deferred evictions after the outbox drains or between
5720    /// chunks, retaining outbox pins and their durable marker. A unit that
5721    /// re-entered the window has no pending entry left.
5722    fn drain_pending_evictions(&mut self) -> Result<(), String> {
5723        let pending = self.load_pending_evictions()?;
5724        if pending.is_empty() {
5725            return Ok(());
5726        }
5727        self.rebuild_overlay_if_dirty()?;
5728        for (unit_id, table_name, effective) in pending {
5729            let Some(table) = self.schema.table(&table_name).cloned() else {
5730                self.delete_pending_evict_table(&unit_id, &table_name)?;
5731                continue;
5732            };
5733            loop {
5734                std::thread::yield_now();
5735                self.begin_observation("syncular_evict")?;
5736                let chunk = (|| {
5737                    let pinned = self.pinned_row_ids(&table_name);
5738                    let owners =
5739                        unit_owners(&self.stored(load_units(&self.conn))?, &table, &unit_id);
5740                    let (remaining, deferred) =
5741                        self.evict_scope_rows(&table_name, &effective, &owners, &pinned)?;
5742                    if !deferred {
5743                        self.delete_pending_evict_table(&unit_id, &table_name)?;
5744                    }
5745                    let mut batch = ChangeAccumulator::default();
5746                    self.record_scope_map(&mut batch, &table_name, &effective);
5747                    self.finish_observation("syncular_evict", batch)?;
5748                    Ok(remaining)
5749                })();
5750                match chunk {
5751                    Ok(true) => {}
5752                    Ok(false) => break,
5753                    Err(error) => {
5754                        self.rollback_observation("syncular_evict");
5755                        return Err(error);
5756                    }
5757                }
5758            }
5759        }
5760        Ok(())
5761    }
5762
5763    /// §4.8 E1: primary keys of `table` referenced by a pending outbox
5764    /// commit — rows that MUST NOT be evicted until the commit drains.
5765    fn pinned_row_ids(&self, table: &str) -> std::collections::HashSet<String> {
5766        let mut pinned = std::collections::HashSet::new();
5767        for commit in self
5768            .outbox
5769            .iter()
5770            .map(|commit| &**commit)
5771            .chain(self.failed_commits.iter().map(|(commit, _)| commit))
5772        {
5773            for op in &commit.ops {
5774                if op.table == table {
5775                    pinned.insert(op.row_id.clone());
5776                }
5777            }
5778        }
5779        pinned
5780    }
5781
5782    fn save_pending_evict(
5783        &self,
5784        unit_id: &str,
5785        table: &str,
5786        effective: &[(String, Vec<String>)],
5787    ) -> Result<(), String> {
5788        self.conn.execute(
5789            "INSERT OR REPLACE INTO _syncular_pending_evictions(unit_id, tbl, effective_scopes) VALUES (?1, ?2, ?3)",
5790            rusqlite::params![unit_id, table, scope_map_to_json(effective).to_string()],
5791        ).map(|_| ()).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5792    }
5793
5794    /// Re-entry cancels every pending eviction of the unit.
5795    fn delete_pending_evict(&self, unit_id: &str) -> Result<(), String> {
5796        self.conn
5797            .execute(
5798                "DELETE FROM _syncular_pending_evictions WHERE unit_id = ?1",
5799                [unit_id],
5800            )
5801            .map(|_| ())
5802            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5803    }
5804
5805    fn delete_pending_evict_table(&self, unit_id: &str, table: &str) -> Result<(), String> {
5806        self.conn
5807            .execute(
5808                "DELETE FROM _syncular_pending_evictions WHERE unit_id = ?1 AND tbl = ?2",
5809                [unit_id, table],
5810            )
5811            .map(|_| ())
5812            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5813    }
5814
5815    fn load_pending_evictions(&self) -> Result<Vec<PendingEvict>, String> {
5816        let mut stmt = self
5817            .conn
5818            .prepare("SELECT unit_id, tbl, effective_scopes FROM _syncular_pending_evictions")
5819            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5820        let rows = stmt
5821            .query_map([], |row| {
5822                Ok((
5823                    row.get::<_, String>(0)?,
5824                    row.get::<_, String>(1)?,
5825                    row.get::<_, String>(2)?,
5826                ))
5827            })
5828            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5829        rows.map(|row| {
5830            let (unit_id, table, json) =
5831                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5832            let value: Value = serde_json::from_str(&json)
5833                .map_err(|_| "sync.local_corrupt: invalid pending eviction scopes".to_owned())?;
5834            let effective = json_to_scope_map(&value)
5835                .map_err(|_| "sync.local_corrupt: invalid pending eviction scopes".to_owned())?;
5836            Ok((unit_id, table, effective))
5837        })
5838        .collect()
5839    }
5840
5841    /// Select failed-intent retention; reload durable retained aggregates.
5842    pub fn set_retain_failed_commits(&mut self, enabled: bool) -> Result<(), String> {
5843        self.retain_failed_commits = enabled;
5844        for (commit, _) in self.failed_commits.iter() {
5845            for operation in &commit.ops {
5846                self.overlay_dirty.table(&operation.table);
5847            }
5848        }
5849        self.failed_commits = self.load_failed_commits()?.into();
5850        for (commit, _) in self.failed_commits.iter() {
5851            for operation in &commit.ops {
5852                self.overlay_dirty.table(&operation.table);
5853            }
5854        }
5855        Ok(())
5856    }
5857
5858    fn load_failed_commits(&self) -> Result<Vec<FailedCommit>, String> {
5859        let mut statement = self.conn.prepare("SELECT client_commit_id, operations_json, initial_json FROM _syncular_failed_commits ORDER BY seq")
5860            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5861        let rows = statement
5862            .query_map([], |row| {
5863                Ok((
5864                    row.get::<_, String>(0)?,
5865                    row.get::<_, String>(1)?,
5866                    row.get::<_, String>(2)?,
5867                ))
5868            })
5869            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5870        rows.map(|row| {
5871            let (id, operations, initial) =
5872                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5873            let operations: Vec<CommitOperation> =
5874                serde_json::from_str(&operations).map_err(|_| "sync.local_corrupt".to_owned())?;
5875            let initial =
5876                serde_json::from_str(&initial).map_err(|_| "sync.local_corrupt".to_owned())?;
5877            Ok((
5878                OutboxCommit {
5879                    client_commit_id: id,
5880                    ops: operations
5881                        .into_iter()
5882                        .map(|op| OutboxOp {
5883                            upsert: op.op == "upsert",
5884                            table: op.table,
5885                            row_id: op.row_id,
5886                            base_version: op.base_version,
5887                            values: op.values,
5888                            insert: op.insert,
5889                        })
5890                        .collect(),
5891                },
5892                initial,
5893            ))
5894        })
5895        .collect()
5896    }
5897
5898    fn retain_failed_commit(&mut self, id: &str, operations: &[OutboxOp]) -> Result<(), String> {
5899        let mut initial = Map::new();
5900        for operation in operations {
5901            // A table the current schema removed has no rows to retain.
5902            let Some(table) = self.schema.table(&operation.table) else {
5903                continue;
5904            };
5905            let initial_row = match self.stored_row(table, &operation.row_id, true)? {
5906                Some(row) => Some(row),
5907                None => self.visible_row(table, &operation.row_id)?,
5908            };
5909            if let Some((values, _)) = initial_row {
5910                let mut intended: Map<String, Value> = table
5911                    .columns
5912                    .iter()
5913                    .zip(values.iter())
5914                    .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
5915                    .collect();
5916                if let Some(patch) = &operation.values {
5917                    intended.extend(patch.clone());
5918                }
5919                initial.insert(
5920                    serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
5921                        .map_err(|_| "sync.local_corrupt".to_owned())?,
5922                    Value::Object(intended),
5923                );
5924            }
5925        }
5926        self.conn.execute("INSERT INTO _syncular_failed_commits(client_commit_id, operations_json, initial_json) VALUES (?1, ?2, ?3)", rusqlite::params![id,
5927            serde_json::to_string(&operations.iter().map(CommitOperation::from).collect::<Vec<_>>()).map_err(|_| "sync.local_corrupt".to_owned())?,
5928            serde_json::to_string(&initial).map_err(|_| "sync.local_corrupt".to_owned())?])
5929            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5930        self.failed_commits.push((
5931            OutboxCommit {
5932                client_commit_id: id.to_owned(),
5933                ops: operations.to_vec(),
5934            },
5935            initial,
5936        ));
5937        Ok(())
5938    }
5939
5940    fn drop_failed_commits(
5941        &mut self,
5942        matches: impl Fn(&str, &Map<String, Value>) -> bool,
5943    ) -> Result<Vec<OutboxCommit>, String> {
5944        let mut doomed = self.drop_acknowledged_rows(&matches)?;
5945        for (commit, initial) in self.failed_commits.iter() {
5946            for operation in &commit.ops {
5947                let key =
5948                    serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
5949                        .map_err(|_| "sync.local_corrupt".to_owned())?;
5950                if initial
5951                    .get(&key)
5952                    .and_then(Value::as_object)
5953                    .is_some_and(|values| matches(&operation.table, values))
5954                {
5955                    doomed.push(commit.clone());
5956                    break;
5957                }
5958            }
5959        }
5960        for commit in &doomed {
5961            self.conn
5962                .execute(
5963                    "DELETE FROM _syncular_failed_commits WHERE client_commit_id = ?1",
5964                    rusqlite::params![commit.client_commit_id],
5965                )
5966                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5967        }
5968        for commit in &doomed {
5969            self.conn.execute("UPDATE _syncular_commit_outcomes SET operations_json = NULL, results_json = '[]' WHERE client_commit_id = ?1", rusqlite::params![commit.client_commit_id]).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5970        }
5971        self.conflicts.retain(|record| {
5972            !doomed
5973                .iter()
5974                .any(|commit| commit.client_commit_id == record.client_commit_id)
5975        });
5976        self.rejections.retain(|record| {
5977            !doomed
5978                .iter()
5979                .any(|commit| commit.client_commit_id == record.client_commit_id)
5980        });
5981        self.failed_commits.retain(|(commit, _)| {
5982            !doomed
5983                .iter()
5984                .any(|dropped| dropped.client_commit_id == commit.client_commit_id)
5985        });
5986        for commit in &doomed {
5987            for operation in &commit.ops {
5988                self.overlay_dirty.table(&operation.table);
5989            }
5990        }
5991        Ok(doomed)
5992    }
5993
5994    /// Record one commit of local mutations. The returned commit ID is the
5995    /// §2.3 idempotency key. Failures are structured so callers read
5996    /// `code`/`message`/`details` instead of parsing a string prefix.
5997    pub fn mutate(&mut self, mutations: Vec<Mutation>) -> Result<String, ClientError> {
5998        self.mutate_with_commit_id(mutations, None)
5999    }
6000
6001    /// [`Self::mutate`] under a caller-supplied `clientCommitId` (§7.1), so an
6002    /// app that loses the reply knows its commit. The ID is non-empty (§6.1);
6003    /// one this replica still holds fails with `client.commit_id_reused`.
6004    pub fn mutate_with_commit_id(
6005        &mut self,
6006        mutations: Vec<Mutation>,
6007        client_commit_id: Option<String>,
6008    ) -> Result<String, ClientError> {
6009        let prior = self.storage_failure.borrow_mut().take();
6010        if let Some(id) = &client_commit_id {
6011            let held = if id.is_empty() {
6012                Err("sync.invalid_request: clientCommitId is empty".to_owned())
6013            } else {
6014                self.conn
6015                    .query_row(
6016                        "SELECT EXISTS(SELECT 1 FROM _syncular_outbox WHERE commit_id=?1 UNION ALL SELECT 1 FROM _syncular_commit_outcomes WHERE client_commit_id=?1)",
6017                        [id],
6018                        |row| row.get(0),
6019                    )
6020                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
6021            };
6022            match held {
6023                Ok(false) => {}
6024                Ok(true) => {
6025                    return self.finish_authoring(
6026                        prior,
6027                        Err(ClientError {
6028                            code: "client.commit_id_reused".to_owned(),
6029                            message: "clientCommitId is held".to_owned(),
6030                            details: Some(json!({ "clientCommitId": id })),
6031                            retryable: false,
6032                        }),
6033                    )
6034                }
6035                Err(message) => return self.finish_authoring(prior, Err(message.into())),
6036            }
6037        }
6038        let outcome = (|| -> Result<String, ClientError> {
6039            if mutations.is_empty() {
6040                return Err(
6041                    "sync.invalid_request: a commit must contain at least one operation (§6.1)"
6042                        .to_owned()
6043                        .into(),
6044                );
6045            }
6046            let mut ops = Vec::with_capacity(mutations.len());
6047            for mutation in mutations {
6048                match mutation {
6049                    Mutation::Patch {
6050                        table,
6051                        values,
6052                        base_version,
6053                    } => {
6054                        let schema_table = self
6055                            .schema
6056                            .table(&table)
6057                            .ok_or_else(|| "sync.unknown_table".to_owned())?;
6058                        let values = normalize_values_casing(schema_table, values)?;
6059                        let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
6060                        ops.push(self.prepare_patch(&table, &row_id, values, base_version)?);
6061                    }
6062                    Mutation::Upsert {
6063                        table,
6064                        values,
6065                        base_version,
6066                    } => {
6067                        let schema_table = self.schema.table(&table).ok_or_else(|| {
6068                            format!("sync.unknown_table: unknown table {table:?}")
6069                        })?;
6070                        // §5: value keys are accepted in snake_case AND the
6071                        // generated row types' camelCase; normalize to SQL truth
6072                        // before the pk lookup / codec see them.
6073                        let values = normalize_values_casing(schema_table, values)?;
6074                        let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
6075                        // §6.7 `mutate` marks every column present: fill missing
6076                        // nullable columns with NULL before the payload is built.
6077                        let values = full_row_values(schema_table, values)?;
6078                        self.refuse_sealed_row(schema_table, &row_id)?;
6079                        // §5.11: validate the payload encodes with the current
6080                        // codec. A key-selection or unknown-key failure is a
6081                        // durable rejection at the push seam (§10.3), never an
6082                        // author-time error. Full-row `mutate` presents every
6083                        // column, so no stored fallback is needed.
6084                        self.validate_author_encode(schema_table, &row_id, &values)?;
6085                        // §7.1 insert intent: the row is absent from the
6086                        // visible replica at call time, before this batch
6087                        // applies anything.
6088                        let insert = !self.visible_row_exists(schema_table, &row_id)?;
6089                        ops.push(OutboxOp {
6090                            upsert: true,
6091                            table,
6092                            row_id,
6093                            base_version,
6094                            values: Some(values),
6095                            insert,
6096                        });
6097                    }
6098                    Mutation::Delete {
6099                        table,
6100                        row_id,
6101                        base_version,
6102                    } => {
6103                        let schema_table = self.schema.table(&table).ok_or_else(|| {
6104                            format!("sync.unknown_table: unknown table {table:?}")
6105                        })?;
6106                        self.refuse_sealed_row(schema_table, &row_id)?;
6107                        // §2.4: the server rejects a rowId holding U+0000.
6108                        if row_id.contains('\0') {
6109                            return Err("sync.invalid_request: rowId holds U+0000 (§2.4)"
6110                                .to_owned()
6111                                .into());
6112                        }
6113                        ops.push(OutboxOp {
6114                            upsert: false,
6115                            table,
6116                            row_id,
6117                            base_version,
6118                            values: None,
6119                            insert: false,
6120                        });
6121                    }
6122                }
6123            }
6124            Ok(self.record_outbox_commit(ops, client_commit_id)?)
6125        })();
6126        self.finish_authoring(prior, outcome)
6127    }
6128
6129    /// Scope the retained `storage_failure` diagnostic to this call so an
6130    /// earlier operation's failure cannot classify a new one, and surface the
6131    /// call's own classified failure with its details. A fresh classified
6132    /// failure replaces the retained diagnostic; otherwise the earlier one is
6133    /// kept for the next sync.
6134    fn finish_authoring(
6135        &self,
6136        prior: Option<QueryReadFailure>,
6137        outcome: Result<String, ClientError>,
6138    ) -> Result<String, ClientError> {
6139        let fresh = self.storage_failure.borrow_mut().take();
6140        *self.storage_failure.borrow_mut() = fresh.clone().or(prior);
6141        outcome.map_err(|error| fresh.map(ClientError::from).unwrap_or(error))
6142    }
6143
6144    fn record_outbox_commit(
6145        &mut self,
6146        ops: Vec<OutboxOp>,
6147        client_commit_id: Option<String>,
6148    ) -> Result<String, String> {
6149        let commit = OutboxCommit {
6150            client_commit_id: client_commit_id.unwrap_or_else(|| uuid::Uuid::new_v4().to_string()),
6151            ops,
6152        };
6153        self.begin_observation("syncular_mutation")?;
6154        let mut batch = ChangeAccumulator::default();
6155        for op in &commit.ops {
6156            let mut precise = self.record_row_scopes(&mut batch, &op.table, &op.row_id, false);
6157            if let Some(values) = &op.values {
6158                if let Some(table) = self.schema.table(&op.table) {
6159                    for scope in &table.scope_variables {
6160                        if let Some(Value::String(value)) = values.get(&scope.column) {
6161                            batch.scope(&op.table, format!("{}:{value}", scope.prefix));
6162                            precise = true;
6163                        }
6164                    }
6165                }
6166            }
6167            if !precise {
6168                batch.table(&op.table);
6169            }
6170        }
6171        if let Err(error) = self.persist_outbox_insert(&commit) {
6172            self.rollback_observation("syncular_mutation");
6173            return Err(error);
6174        }
6175        // A clean overlay contains the FIFO fold of every previous commit, so
6176        // the new commit applies over it. §7.1: an operation that does not
6177        // apply (a secondary unique collision) fails the whole commit; the
6178        // savepoint rollback removes the outbox entry and every visible write.
6179        // The append undoes itself: a rollback capture would copy the outbox.
6180        let was_dirty = self.overlay_dirty.snapshot();
6181        let id = commit.client_commit_id.clone();
6182        let appended = self.rebuild_overlay_if_dirty().and_then(|()| {
6183            commit
6184                .ops
6185                .iter()
6186                .try_for_each(|op| self.apply_outbox_op(op))?;
6187            self.outbox.push(Arc::new(commit));
6188            batch.status = true;
6189            self.finish_observation("syncular_mutation", batch)
6190                .inspect_err(|_| drop(self.outbox.pop()))
6191        });
6192        if let Err(error) = appended {
6193            self.rollback_observation("syncular_mutation");
6194            self.overlay_dirty.restore(was_dirty);
6195            return Err(error);
6196        }
6197        Ok(id)
6198    }
6199
6200    /// Record one §6.1 sparse upsert: the primary key plus the supplied
6201    /// non-scope columns are present; every other column is absent and stays
6202    /// untouched on the server and in the local overlay. An absent local base
6203    /// refuses the whole batch with sync.row_missing before enqueueing (§7.1).
6204    /// Failures are structured so callers read `code`/`message`/`details`.
6205    pub fn patch(
6206        &mut self,
6207        table: &str,
6208        row_id: &str,
6209        partial: Map<String, Value>,
6210        base_version: Option<i64>,
6211    ) -> Result<String, ClientError> {
6212        let prior = self.storage_failure.borrow_mut().take();
6213        let outcome = self.perform_patch(table, row_id, partial, base_version);
6214        self.finish_authoring(prior, outcome)
6215    }
6216
6217    /// The unscoped patch body. Public authoring entry points scope
6218    /// `storage_failure` around the whole operation, including any pre-read.
6219    fn perform_patch(
6220        &mut self,
6221        table: &str,
6222        row_id: &str,
6223        partial: Map<String, Value>,
6224        base_version: Option<i64>,
6225    ) -> Result<String, ClientError> {
6226        let operation = self.prepare_patch(table, row_id, partial, base_version)?;
6227        Ok(self.record_outbox_commit(vec![operation], None)?)
6228    }
6229
6230    fn prepare_patch(
6231        &self,
6232        table: &str,
6233        row_id: &str,
6234        partial: Map<String, Value>,
6235        base_version: Option<i64>,
6236    ) -> Result<OutboxOp, ClientError> {
6237        let schema_table = self
6238            .schema
6239            .table(table)
6240            .ok_or_else(|| format!("sync.unknown_table: unknown table {table:?}"))?;
6241        let partial = normalize_values_casing(schema_table, partial)?;
6242        if let Some(pk) = partial.get(&schema_table.primary_key) {
6243            if render_row_id_json(Some(pk)).ok().as_deref() != Some(row_id) {
6244                return Err(invalid_authoring(
6245                    "patch cannot change the primary key",
6246                    json!({ "table": table }),
6247                ));
6248            }
6249        }
6250        if !self.visible_row_exists(schema_table, row_id)? {
6251            return Err("sync.row_missing: a sparse patch requires a local row"
6252                .to_owned()
6253                .into());
6254        }
6255        self.refuse_sealed_row(schema_table, row_id)?;
6256        // §3.4 rule 5 / §6.2: scope columns are immutable on update. The
6257        // server accepts a present scope column whose value equals the
6258        // stored row and applies it as a no-op, so a decoded envelope
6259        // round-trips. The local row is the only value the client can prove
6260        // equality against; when the row is absent locally the client cannot
6261        // prove equality against the server's stored row and fails closed.
6262        // A primary key that is also a scope column needs no stored-row
6263        // read: the primary key in a sparse payload is by construction the
6264        // row id being patched (§6.1), so its value is proven equal already.
6265        for scope in &schema_table.scope_variables {
6266            if !partial.contains_key(&scope.column) || scope.column == schema_table.primary_key {
6267                continue;
6268            }
6269            let Some(column) = schema_table
6270                .columns
6271                .iter()
6272                .find(|column| column.name == scope.column)
6273            else {
6274                continue;
6275            };
6276            let stored = self
6277                .conn
6278                .query_row(
6279                    &format!(
6280                        "SELECT {} FROM {} WHERE {}",
6281                        quote_ident(&scope.column),
6282                        visible_table(&schema_table.name),
6283                        row_id_predicate(schema_table)
6284                    ),
6285                    rusqlite::params![row_id],
6286                    |row| Ok(sql_ref_to_json(column, row.get_ref(0)?)),
6287                )
6288                .optional()
6289                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6290            // Decode the supplied value through the same column-type path the
6291            // rest of the patch encoding uses, so a value that only equals
6292            // the stored one in another number representation still compares
6293            // equal.
6294            let incoming = json_to_column_value(column, partial.get(&scope.column))
6295                .ok()
6296                .map(|value| column_value_to_json(&value));
6297            // A stored blob never equals a patch value, matching the server's
6298            // scope-column comparison (§3.4): bytes are unstorable as scope
6299            // values.
6300            let equal = stored.as_ref().is_some_and(|stored| {
6301                stored.get("$bytes").is_none() && incoming.as_ref() == Some(stored)
6302            });
6303            if !equal {
6304                return Err(invalid_authoring(
6305                    "patch cannot write a scope column (§3.4)",
6306                    json!({ "table": table, "column": scope.column }),
6307                ));
6308            }
6309        }
6310        let mut values = partial;
6311        // Presence-set semantics: a proven-equal scope column is dropped, so
6312        // the patch leaves the stored value untouched (§6.2).
6313        for scope in &schema_table.scope_variables {
6314            values.remove(&scope.column);
6315        }
6316        let pk_value = match schema_table.columns[schema_table.pk_index].ty {
6317            // §2.4: the rowId of an integer key is its canonical decimal form.
6318            ColumnType::Integer => match row_id.parse::<i64>() {
6319                Ok(v)
6320                    if v.to_string() == row_id
6321                        && (-ssp2::primitives::I64_SAFE_MAX..=ssp2::primitives::I64_SAFE_MAX)
6322                            .contains(&v) =>
6323                {
6324                    Value::from(v)
6325                }
6326                _ => {
6327                    return Err(format!(
6328                    "sync.invalid_request: table {table:?}: invalid integer primary key {row_id:?}"
6329                )
6330                    .into())
6331                }
6332            },
6333            ColumnType::Boolean => match row_id {
6334                "true" => Value::Bool(true),
6335                "false" => Value::Bool(false),
6336                _ => {
6337                    return Err(format!(
6338                    "sync.invalid_request: table {table:?}: invalid boolean primary key {row_id:?}"
6339                )
6340                    .into())
6341                }
6342            },
6343            _ => Value::from(row_id.to_owned()),
6344        };
6345        values.insert(schema_table.primary_key.clone(), pk_value);
6346        // §6.1: a patch cannot null a non-nullable column.
6347        if let Some(column) = schema_table
6348            .columns
6349            .iter()
6350            .find(|column| !column.nullable && values.get(&column.name) == Some(&Value::Null))
6351        {
6352            return Err(invalid_authoring(
6353                "a non-nullable column is null (§6.1)",
6354                json!({ "table": table, "column": column.name }),
6355            ));
6356        }
6357        // §5.11: structural validation only; an absent key-id selector is
6358        // resolved at the push seam from the stored local row, and a
6359        // key-selection or unknown-key failure becomes a durable
6360        // `client.encrypt_failed` rejection there (§10.3), never an
6361        // author-time error.
6362        self.validate_author_encode(schema_table, row_id, &values)?;
6363        Ok(OutboxOp {
6364            upsert: true,
6365            table: table.to_owned(),
6366            row_id: row_id.to_owned(),
6367            base_version,
6368            values: Some(values),
6369            insert: false,
6370        })
6371    }
6372
6373    /// §5.11 key-id fallback: the stored local row's selector value as a
6374    /// positional row (only the selector slot filled). Returns None when no
6375    /// selector is configured, the row is locally absent, or the stored
6376    /// value is not a usable key id. A failed storage read is an error.
6377    fn stored_key_fallback(
6378        &self,
6379        table: &TableSchema,
6380        row_id: &str,
6381    ) -> Result<Option<Row>, String> {
6382        let Some(selector) = self.encryption.key_id_columns.get(&table.name) else {
6383            return Ok(None);
6384        };
6385        let Some(index) = table
6386            .columns
6387            .iter()
6388            .position(|column| &column.name == selector)
6389        else {
6390            return Ok(None);
6391        };
6392        let sql = format!(
6393            "SELECT {} FROM {} WHERE {}",
6394            quote_ident(selector),
6395            visible_table(&table.name),
6396            row_id_predicate(table)
6397        );
6398        let key = self
6399            .conn
6400            .query_row(&sql, rusqlite::params![row_id], |row| {
6401                row.get::<_, Option<String>>(0)
6402            })
6403            .optional()
6404            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
6405            .flatten()
6406            .filter(|key| !key.is_empty());
6407        let Some(key) = key else {
6408            return Ok(None);
6409        };
6410        let mut fallback: Row = vec![None; table.columns.len()];
6411        fallback[index] = Some(ColumnValue::String(key));
6412        Ok(Some(fallback))
6413    }
6414
6415    /// §5.11/§10.3 author-time encode check: a structural failure (a
6416    /// non-nullable NULL, a value the codec rejects) is an author-time
6417    /// error; a key-selection or unknown-key failure is deferred to the push
6418    /// seam, where it becomes a durable `client.encrypt_failed` rejection.
6419    fn validate_author_encode(
6420        &self,
6421        table: &TableSchema,
6422        row_id: &str,
6423        values: &Map<String, Value>,
6424    ) -> Result<(), String> {
6425        match encode_sparse_row_json(table, row_id, values, &self.encryption, None) {
6426            Err(error) if error.starts_with(ENCRYPT_FAILED_CODE) => Ok(()),
6427            result => result.map(|_| ()),
6428        }
6429    }
6430
6431    /// §5.11/§10.3: drop every pending commit whose push encode fails with
6432    /// `client.encrypt_failed` (an unresolvable key id or an unknown key) and
6433    /// raise one durable rejection per commit. `sync()` never aborts for an
6434    /// encode failure. Mirrors §7.4.4's `drop_incompatible_outbox`, including
6435    /// undoing the dropped commit's purely-optimistic rows.
6436    /// §7.1/§7.4.4: drop pending commits whose values the current codec refuses
6437    /// (an envelope or a type an earlier version accepted). `with_encryption`
6438    /// additionally drops commits whose §5.11 encryption cannot resolve a key;
6439    /// the startup boundary leaves those to the send seam, where hosts configure
6440    /// keys after open.
6441    /// Recovery runs as one observation. An enclosing observation (the guarded
6442    /// schema reset) passes its `batch`, so the rejection, outcome, and table
6443    /// domains join that change event instead of publishing a second one.
6444    fn drop_unencodable_outbox(
6445        &mut self,
6446        with_encryption: bool,
6447        batch: Option<&mut ChangeAccumulator>,
6448    ) -> Result<bool, String> {
6449        let mut failures = Vec::new();
6450        for commit in self.outbox.iter() {
6451            for op in &commit.ops {
6452                let Some(table) = self.schema.table(&op.table).filter(|_| op.upsert) else {
6453                    continue;
6454                };
6455                // An insert lacking a non-nullable column is left to
6456                // `drop_incompatible_outbox` (§7.4.4).
6457                let Some(values) = op.presented_values(table) else {
6458                    continue;
6459                };
6460                let fallback = self.stored_key_fallback(table, &op.row_id)?;
6461                let failure = match encode_sparse_row_json(
6462                    table,
6463                    &op.row_id,
6464                    &values,
6465                    &self.encryption,
6466                    fallback.as_ref(),
6467                ) {
6468                    Err(error) if error.starts_with(ENCRYPT_FAILED_CODE) => {
6469                        with_encryption.then_some((error, ENCRYPT_FAILED_CODE))
6470                    }
6471                    // §7.1/§7.4.4: a value the current codec refuses was
6472                    // persisted by a version that accepted it; it leaves
6473                    // through the incompatible-outbox path, so the FIFO
6474                    // queue never wedges.
6475                    Err(error) if error.starts_with(INVALID_REQUEST_CODE) => Some((
6476                        LEGACY_VALUE_REFUSAL_MESSAGE.to_owned(),
6477                        OUTBOX_INCOMPATIBLE_CODE,
6478                    )),
6479                    _ => None,
6480                };
6481                if let Some((error, code)) = failure {
6482                    failures.push((commit.clone(), error, code));
6483                    break;
6484                }
6485            }
6486        }
6487        if failures.is_empty() {
6488            return Ok(false);
6489        }
6490        // §7.4.4: one observation covers the durable outcome writes, the outbox
6491        // removal, the in-memory drop, and the overlay rebuild, so a storage fault
6492        // leaves the replica and the client unchanged, a successful recovery
6493        // publishes one batch with the rejection, outcome, and table domains, and
6494        // an ordinary open with nothing to recover publishes nothing.
6495        let rollback = self.capture();
6496        let owns_observation = batch.is_none();
6497        let owns_transaction = self.conn.is_autocommit();
6498        let mut domains = ChangeAccumulator::default();
6499        if owns_observation {
6500            self.begin_observation("syncular_value_recovery")?;
6501        }
6502        let dropped = (|| -> Result<(), String> {
6503            for (commit, message, code) in failures {
6504                if self.retain_failed_commits && code == ENCRYPT_FAILED_CODE {
6505                    self.retain_failed_commit(&commit.client_commit_id, &commit.ops)?;
6506                }
6507                let representable = commit.ops.iter().all(Self::outbox_op_is_journalable);
6508                let rejection = RejectionRecord {
6509                    client_commit_id: commit.client_commit_id.clone(),
6510                    op_index: 0,
6511                    code: code.to_owned(),
6512                    message,
6513                    retryable: false,
6514                    details: (code == OUTBOX_INCOMPATIBLE_CODE).then(|| RejectionDetails {
6515                        field_paths: None,
6516                        reason: Some(INVALID_STORED_VALUES_REASON.to_owned()),
6517                        required_action: None,
6518                        references: None,
6519                    }),
6520                    operation: representable
6521                        .then(|| commit.ops.first())
6522                        .flatten()
6523                        .map(CommitOperation::from),
6524                };
6525                self.persist_commit_outcome(
6526                    &commit.client_commit_id,
6527                    CommitOutcomeStatus::Rejected,
6528                    &[CommitOperationOutcome::Error {
6529                        rejection: rejection.clone(),
6530                    }],
6531                    representable.then_some(commit.ops.as_slice()),
6532                )?;
6533                self.delete_outbox_persisted(&commit.client_commit_id)?;
6534                self.outbox
6535                    .retain(|candidate| candidate.client_commit_id != commit.client_commit_id);
6536                self.rejections.push(rejection);
6537                for operation in &commit.ops {
6538                    domains.table(operation.table.as_str());
6539                }
6540            }
6541            domains.status = true;
6542            domains.rejections = true;
6543            domains.outcomes = true;
6544            self.prune_commit_outcomes()?;
6545            // The visible projection is re-derived from the surviving base + outbox.
6546            self.overlay_dirty.set(true);
6547            self.rebuild_overlay()?;
6548            Ok(())
6549        })();
6550        let result = match batch {
6551            Some(batch) => {
6552                if dropped.is_ok() {
6553                    for table in domains.tables.keys() {
6554                        batch.table(table.as_str());
6555                    }
6556                    batch.status |= domains.status;
6557                    batch.rejections |= domains.rejections;
6558                    batch.outcomes |= domains.outcomes;
6559                }
6560                dropped
6561            }
6562            None => {
6563                dropped.and_then(|()| self.finish_observation("syncular_value_recovery", domains))
6564            }
6565        };
6566        if let Err(error) = result {
6567            if owns_observation {
6568                if owns_transaction {
6569                    if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
6570                        let mut failure = self.storage_failure.borrow_mut();
6571                        let failure =
6572                            failure.get_or_insert_with(|| QueryReadFailure::from(error.clone()));
6573                        failure.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
6574                    }
6575                } else {
6576                    self.rollback_observation("syncular_value_recovery");
6577                }
6578            }
6579            self.restore(rollback);
6580            return Err(error);
6581        }
6582        Ok(true)
6583    }
6584
6585    pub fn pending_commit_ids(&self) -> Vec<String> {
6586        self.outbox
6587            .iter()
6588            .map(|c| c.client_commit_id.clone())
6589            .collect()
6590    }
6591
6592    /// §6.1: the sparse push payload bytes of every pending upsert operation,
6593    /// FIFO — exactly what the next push round would put on the wire. Used by
6594    /// conformance B.21(h) to pin cross-core byte identity.
6595    pub fn pending_payloads(&self) -> Result<Vec<Vec<u8>>, String> {
6596        let mut payloads = Vec::new();
6597        for commit in self.outbox.iter() {
6598            for op in &commit.ops {
6599                let Some(table) = self.schema.table(&op.table).filter(|_| op.upsert) else {
6600                    continue;
6601                };
6602                let Some(values) = op.presented_values(table) else {
6603                    continue;
6604                };
6605                let fallback = self.stored_key_fallback(table, &op.row_id)?;
6606                if let Ok(payload) = encode_sparse_row_json(
6607                    table,
6608                    &op.row_id,
6609                    &values,
6610                    &self.encryption,
6611                    fallback.as_ref(),
6612                ) {
6613                    payloads.push(payload);
6614                }
6615            }
6616        }
6617        Ok(payloads)
6618    }
6619
6620    pub fn conflicts(&self) -> &[ConflictRecord] {
6621        &self.conflicts
6622    }
6623
6624    pub fn rejections(&self) -> &[RejectionRecord] {
6625        &self.rejections
6626    }
6627
6628    pub fn resolve_commit_outcome(
6629        &mut self,
6630        input: ResolveCommitOutcomeInput,
6631    ) -> Result<CommitOutcome, String> {
6632        let current = self
6633            .commit_outcome(&input.client_commit_id)?
6634            .ok_or_else(|| {
6635                format!(
6636                    "sync.outcome_not_found: no durable outcome exists for {:?}",
6637                    input.client_commit_id
6638                )
6639            })?;
6640        if current.resolution != CommitOutcomeResolution::Active {
6641            return Ok(current);
6642        }
6643        if input.resolution == CommitOutcomeResolution::Active {
6644            return Err("sync.invalid_request: resolution must leave active state".to_owned());
6645        }
6646        match input.resolution {
6647            CommitOutcomeResolution::Superseded => {
6648                let replacement = input
6649                    .replacement_client_commit_id
6650                    .as_deref()
6651                    .filter(|value| !value.is_empty() && *value != input.client_commit_id)
6652                    .ok_or_else(|| {
6653                        "sync.invalid_request: superseded outcomes require a distinct replacementClientCommitId"
6654                            .to_owned()
6655                    })?;
6656                let _ = replacement;
6657            }
6658            _ if input.replacement_client_commit_id.is_some() => {
6659                return Err(
6660                    "sync.invalid_request: replacementClientCommitId is valid only for superseded outcomes"
6661                        .to_owned(),
6662                );
6663            }
6664            _ => {}
6665        }
6666        let allowed = match current.status {
6667            CommitOutcomeStatus::Conflict => matches!(
6668                input.resolution,
6669                CommitOutcomeResolution::ResolvedKeepServer | CommitOutcomeResolution::Superseded
6670            ),
6671            CommitOutcomeStatus::Rejected => {
6672                matches!(
6673                    input.resolution,
6674                    CommitOutcomeResolution::Superseded
6675                        | CommitOutcomeResolution::ResolvedKeepServer
6676                )
6677            }
6678            CommitOutcomeStatus::Applied | CommitOutcomeStatus::Cached => {
6679                input.resolution == CommitOutcomeResolution::Dismissed
6680            }
6681        };
6682        if !allowed {
6683            return Err(format!(
6684                "sync.invalid_request: resolution {:?} is invalid for {:?} outcome",
6685                input.resolution, current.status
6686            ));
6687        }
6688
6689        let rollback = self.capture();
6690        self.begin_observation("syncular_outcome_resolution")?;
6691        let result = (|| {
6692            self.conn
6693                .execute(
6694                    "UPDATE _syncular_commit_outcomes
6695                        SET resolution = ?1, resolved_at_ms = ?2,
6696                            replacement_client_commit_id = ?3
6697                      WHERE client_commit_id = ?4 AND resolution = 'active'",
6698                    rusqlite::params![
6699                        Self::outcome_resolution_name(input.resolution),
6700                        self.clock_now_ms(),
6701                        input.replacement_client_commit_id,
6702                        input.client_commit_id
6703                    ],
6704                )
6705                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6706            let resolved = self
6707                .commit_outcome(&current.client_commit_id)?
6708                .ok_or_else(|| "sync.outcome_not_found: outcome disappeared".to_owned())?;
6709            self.conn
6710                .execute(
6711                    "DELETE FROM _syncular_failed_commits WHERE client_commit_id = ?1",
6712                    rusqlite::params![input.client_commit_id],
6713                )
6714                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6715            for (commit, _) in self.failed_commits.iter() {
6716                if commit.client_commit_id == input.client_commit_id {
6717                    for operation in &commit.ops {
6718                        self.overlay_dirty.table(&operation.table);
6719                    }
6720                }
6721            }
6722            self.failed_commits
6723                .retain(|(commit, _)| commit.client_commit_id != input.client_commit_id);
6724            self.prune_commit_outcomes()?;
6725            Ok(resolved)
6726        })();
6727        let resolved = match result {
6728            Ok(outcome) => outcome,
6729            Err(error) => {
6730                self.rollback_observation("syncular_outcome_resolution");
6731                self.restore(rollback);
6732                return Err(error);
6733            }
6734        };
6735        self.conflicts
6736            .retain(|record| record.client_commit_id != current.client_commit_id);
6737        self.rejections
6738            .retain(|record| record.client_commit_id != current.client_commit_id);
6739        let mut batch = ChangeAccumulator {
6740            conflicts: current.status == CommitOutcomeStatus::Conflict,
6741            rejections: current.status == CommitOutcomeStatus::Rejected,
6742            outcomes: true,
6743            ..ChangeAccumulator::default()
6744        };
6745        if let Some(operations) = &current.operations {
6746            for operation in operations {
6747                batch.table(&operation.table);
6748            }
6749        }
6750        if let Err(error) = self
6751            .rebuild_overlay_if_dirty()
6752            .and_then(|()| self.finish_observation("syncular_outcome_resolution", batch))
6753        {
6754            self.rollback_observation("syncular_outcome_resolution");
6755            self.restore(rollback);
6756            return Err(error);
6757        }
6758        Ok(resolved)
6759    }
6760
6761    pub fn schema_floor(&self) -> Option<&SchemaFloor> {
6762        self.schema_floor.as_ref()
6763    }
6764
6765    /// §7.3.5: the client's opaque auth-lease state, if any.
6766    pub fn lease_state(&self) -> Option<&LeaseState> {
6767        self.lease_state.as_ref()
6768    }
6769
6770    /// §7.3.5: record a request-level lease error (stop-and-surface). Only
6771    /// the two lease codes set it; other errors leave leaseState untouched.
6772    fn record_lease_error(&mut self, code: &str) -> Result<(), String> {
6773        if code != "sync.auth_lease_required" && code != "sync.auth_lease_revoked" {
6774            return Ok(());
6775        }
6776        let mut next = self.lease_state.clone().unwrap_or_default();
6777        next.error_code = Some(code.to_owned());
6778        self.set_lease_state(Some(next))
6779    }
6780
6781    fn set_lease_state(&mut self, next: Option<LeaseState>) -> Result<(), String> {
6782        if self.lease_state == next {
6783            return Ok(());
6784        }
6785        self.begin_observation("syncular_lease")?;
6786        let rollback = self.capture();
6787        self.lease_state = next;
6788        let persisted = match &self.lease_state {
6789            Some(lease) => serde_json::to_string(lease)
6790                .map_err(|error| error.to_string())
6791                .and_then(|json| self.set_meta(LEASE_STATE_KEY, &json)),
6792            None => self.delete_meta(LEASE_STATE_KEY),
6793        };
6794        let batch = ChangeAccumulator {
6795            status: true,
6796            ..ChangeAccumulator::default()
6797        };
6798        if let Err(error) =
6799            persisted.and_then(|()| self.finish_observation("syncular_lease", batch))
6800        {
6801            self.rollback_observation("syncular_lease");
6802            self.restore(rollback);
6803            return Err(error);
6804        }
6805        Ok(())
6806    }
6807
6808    fn set_schema_floor(&mut self, next: Option<SchemaFloor>) -> Result<(), String> {
6809        if self.schema_floor == next {
6810            return Ok(());
6811        }
6812        self.begin_observation("syncular_schema_floor")?;
6813        let rollback = self.capture();
6814        self.stopped = next.is_some();
6815        self.schema_floor = next;
6816        let persisted = match &self.schema_floor {
6817            Some(floor) => serde_json::to_string(floor)
6818                .map_err(|error| error.to_string())
6819                .and_then(|json| self.set_meta(SCHEMA_FLOOR_KEY, &json)),
6820            None => self.delete_meta(SCHEMA_FLOOR_KEY),
6821        };
6822        let batch = ChangeAccumulator {
6823            status: true,
6824            ..ChangeAccumulator::default()
6825        };
6826        if let Err(error) =
6827            persisted.and_then(|()| self.finish_observation("syncular_schema_floor", batch))
6828        {
6829            self.rollback_observation("syncular_schema_floor");
6830            self.restore(rollback);
6831            return Err(error);
6832        }
6833        Ok(())
6834    }
6835
6836    fn set_upgrading(&mut self, value: bool) {
6837        if self.upgrading == value {
6838            return;
6839        }
6840        if self.begin_observation("syncular_upgrading").is_err() {
6841            return;
6842        }
6843        self.upgrading = value;
6844        let batch = ChangeAccumulator {
6845            status: true,
6846            ..ChangeAccumulator::default()
6847        };
6848        if self
6849            .finish_observation("syncular_upgrading", batch)
6850            .is_err()
6851        {
6852            self.rollback_observation("syncular_upgrading");
6853        }
6854    }
6855
6856    pub fn sync_needed(&self) -> bool {
6857        self.sync_needed
6858    }
6859
6860    pub fn read_rows(&self, table: &str) -> Result<Vec<RowState>, String> {
6861        let schema_table = self
6862            .schema
6863            .table(table)
6864            .ok_or_else(|| format!("unknown table {table:?}"))?;
6865        let sql = format!(
6866            "SELECT * FROM {} ORDER BY {} ASC",
6867            visible_table(table),
6868            quote_ident(&schema_table.primary_key)
6869        );
6870        let mut stmt = self
6871            .conn
6872            .prepare(&sql)
6873            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6874        let mut rows = stmt
6875            .query([])
6876            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6877        let mut out = Vec::new();
6878        while let Some(row) = rows
6879            .next()
6880            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
6881        {
6882            let mut values = Map::new();
6883            for (i, column) in schema_table.columns.iter().enumerate() {
6884                let value = row
6885                    .get_ref(i)
6886                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6887                values.insert(column.name.clone(), sql_ref_to_json(column, value));
6888            }
6889            let version: i64 = row
6890                .get(schema_table.columns.len())
6891                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6892            let row_id = match values.get(&schema_table.primary_key) {
6893                Some(Value::String(s)) => s.clone(),
6894                Some(Value::Number(n)) => n.to_string(),
6895                Some(Value::Bool(b)) => b.to_string(),
6896                other => format!("{}", other.cloned().unwrap_or(Value::Null)),
6897            };
6898            let sealed_key = if schema_table.has_encrypted_columns() {
6899                row.get(schema_table.columns.len() + 1)
6900                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
6901            } else {
6902                None
6903            };
6904            out.push(RowState {
6905                row_id,
6906                version,
6907                values,
6908                sealed_key,
6909            });
6910        }
6911        Ok(out)
6912    }
6913
6914    // -- §5.10.5 native CRDT (the `crdt-yjs` feature) --------------------------
6915    //
6916    // The Rust face of the §5.10.4 client model: a local crdt edit loads the
6917    // current stored (server-merged ⊕ pending-overlay) column bytes, applies
6918    // the op with `yrs`, re-encodes the whole doc state, and records a
6919    // baseVersion-less sparse upsert carrying only the primary key and the
6920    // crdt column (§5.10.3, §6.6). No local merge — merging is server-side;
6921    // the overlay re-materializes the edit immediately (optimistic apply,
6922    // §7.1) and the server-merged bytes arrive on the next pull, idempotently.
6923    // Byte-compatible with `@syncular/crdt-yjs`.
6924
6925    /// The current stored value of a `crdt` column for one row — the visible
6926    /// (optimistic) bytes, or `None` when the row is absent or the column is
6927    /// NULL (the empty document, §5.10.1). Errors if the column is not a
6928    /// `crdt` column (guards the app against a typo'd column name).
6929    #[cfg(feature = "crdt-yjs")]
6930    fn crdt_column_bytes(
6931        &self,
6932        table: &str,
6933        row_id: &str,
6934        column: &str,
6935    ) -> Result<Option<Vec<u8>>, String> {
6936        let schema_table = self
6937            .schema
6938            .table(table)
6939            .ok_or_else(|| format!("unknown table {table:?}"))?;
6940        let col = schema_table
6941            .columns
6942            .iter()
6943            .find(|c| c.name == column)
6944            .ok_or_else(|| format!("table {table:?} has no column {column:?}"))?;
6945        if col.ty != ColumnType::Crdt {
6946            return Err(format!("column {column:?} is not a crdt column (§5.10.1)"));
6947        }
6948        let sql = format!(
6949            "SELECT {} FROM {} WHERE {}",
6950            quote_ident(column),
6951            visible_table(table),
6952            row_id_predicate(schema_table)
6953        );
6954        let bytes: Option<Vec<u8>> = self
6955            .conn
6956            .prepare_cached(&sql)
6957            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
6958            .query_row(rusqlite::params![row_id], |row| {
6959                row.get::<_, Option<Vec<u8>>>(0)
6960            })
6961            .map_err(|e| match e {
6962                rusqlite::Error::QueryReturnedNoRows => "no such row".to_owned(),
6963                other => Self::sqlite_failure(&self.storage_failure, other),
6964            })?;
6965        Ok(bytes)
6966    }
6967
6968    /// §5.10.4 materialize: the collaborative text of a `crdt` column, decoded
6969    /// from the stored bytes with `yrs` — `YjsColumn.text(name).toString()`.
6970    /// An absent row / NULL column is the empty document (empty string).
6971    #[cfg(feature = "crdt-yjs")]
6972    pub fn crdt_text(
6973        &self,
6974        table: &str,
6975        row_id: &str,
6976        column: &str,
6977        name: &str,
6978    ) -> Result<String, String> {
6979        let bytes = self
6980            .crdt_column_bytes(table, row_id, column)?
6981            .unwrap_or_default();
6982        crate::crdt::text(&bytes, name)
6983    }
6984
6985    /// §5.10.4 push-an-update: apply a text insert to a `crdt` column and
6986    /// record the resulting crdt-only sparse operation. Returns the enqueued
6987    /// `clientCommitId`.
6988    #[cfg(feature = "crdt-yjs")]
6989    pub fn crdt_insert_text(
6990        &mut self,
6991        table: &str,
6992        row_id: &str,
6993        column: &str,
6994        name: &str,
6995        index: u32,
6996        value: &str,
6997    ) -> Result<String, ClientError> {
6998        let prior = self.storage_failure.borrow_mut().take();
6999        let outcome = (|| -> Result<String, ClientError> {
7000            let current = self
7001                .crdt_column_bytes(table, row_id, column)?
7002                .unwrap_or_default();
7003            let update = crate::crdt::insert_text(&current, name, index, value)?;
7004            self.perform_crdt_push_update(table, row_id, column, &update)
7005        })();
7006        self.finish_authoring(prior, outcome)
7007    }
7008
7009    /// §5.10.4 push-an-update: apply a text delete to a `crdt` column and
7010    /// record the resulting crdt-only sparse operation. Returns the enqueued
7011    /// `clientCommitId`.
7012    #[cfg(feature = "crdt-yjs")]
7013    pub fn crdt_delete_text(
7014        &mut self,
7015        table: &str,
7016        row_id: &str,
7017        column: &str,
7018        name: &str,
7019        index: u32,
7020        len: u32,
7021    ) -> Result<String, ClientError> {
7022        let prior = self.storage_failure.borrow_mut().take();
7023        let outcome = (|| -> Result<String, ClientError> {
7024            let current = self
7025                .crdt_column_bytes(table, row_id, column)?
7026                .unwrap_or_default();
7027            let update = crate::crdt::delete_text(&current, name, index, len)?;
7028            self.perform_crdt_push_update(table, row_id, column, &update)
7029        })();
7030        self.finish_authoring(prior, outcome)
7031    }
7032
7033    /// §5.10.4 generic escape hatch: apply an arbitrary Yjs update onto a
7034    /// `crdt` column's current state and record the resulting crdt-only sparse
7035    /// operation. The app authored the update with its own `yrs` model.
7036    /// Returns the enqueued `clientCommitId`.
7037    #[cfg(feature = "crdt-yjs")]
7038    pub fn crdt_apply_update(
7039        &mut self,
7040        table: &str,
7041        row_id: &str,
7042        column: &str,
7043        update: &[u8],
7044    ) -> Result<String, ClientError> {
7045        let prior = self.storage_failure.borrow_mut().take();
7046        let outcome = (|| -> Result<String, ClientError> {
7047            let current = self
7048                .crdt_column_bytes(table, row_id, column)?
7049                .unwrap_or_default();
7050            let next = crate::crdt::apply_update(&current, update)?;
7051            self.perform_crdt_push_update(table, row_id, column, &next)
7052        })();
7053        self.finish_authoring(prior, outcome)
7054    }
7055
7056    /// Shared tail of the crdt edit methods: record a §6.1 sparse crdt-only
7057    /// upsert — the primary key plus the crdt column, no `baseVersion` — and
7058    /// enqueue it. The server merges the crdt column and leaves every other
7059    /// column untouched (§5.10.3), so a baseVersion-less crdt edit cannot
7060    /// clobber a concurrent edit to another column. A locally absent row
7061    /// records the same partial operation; the server answers per §6.2.
7062    #[cfg(feature = "crdt-yjs")]
7063    fn perform_crdt_push_update(
7064        &mut self,
7065        table: &str,
7066        row_id: &str,
7067        column: &str,
7068        crdt_bytes: &[u8],
7069    ) -> Result<String, ClientError> {
7070        let mut bytes_obj = Map::new();
7071        bytes_obj.insert("$bytes".to_owned(), Value::from(bytes_to_hex(crdt_bytes)));
7072        let mut partial = Map::new();
7073        partial.insert(column.to_owned(), Value::Object(bytes_obj));
7074        self.perform_patch(table, row_id, partial, None)
7075    }
7076
7077    /// Run an arbitrary read-only SQL query against the local database and
7078    /// return each row as a `column-name → JSON value` map. This is the seam
7079    /// the React `useSyncQuery` live-query API needs (it takes app-authored
7080    /// SQL over the visible tables/views, not a fixed table read like
7081    /// [`read_rows`]).
7082    ///
7083    /// Bound `params` are the driver value forms: JSON strings/numbers/bools/
7084    /// null bind directly; a `{"$bytes": hex}` object binds as a BLOB — the
7085    /// same envelope the command surface uses everywhere else. Output BLOB
7086    /// columns come back as `{"$bytes": hex}` to round-trip cleanly.
7087    ///
7088    /// The result column typing is dynamic (SQLite's stored affinity), because
7089    /// arbitrary SQL can alias, join, and compute — there is no schema column
7090    /// to consult per output cell, unlike [`read_rows`].
7091    pub fn query(&self, sql: &str, params: &[QueryValue]) -> Result<Vec<QueryRow>, String> {
7092        query_connection(&self.conn, sql, params).map_err(|failure| failure.to_string())
7093    }
7094
7095    /// Repository benchmark and conformance access, present only with `bench-internals`.
7096    #[cfg(feature = "bench-internals")]
7097    #[doc(hidden)]
7098    pub fn benchmark_connection(&mut self) -> &mut Connection {
7099        &mut self.conn
7100    }
7101
7102    /// Private benchmark intervals; absent from ordinary builds and diagnostics.
7103    #[cfg(feature = "bench-internals")]
7104    #[doc(hidden)]
7105    pub fn benchmark_phases(
7106        &self,
7107        enabled: Option<bool>,
7108        reset: bool,
7109    ) -> Result<Option<Value>, String> {
7110        self.benchmark_phases.configure(enabled, reset)?;
7111        Ok(self.benchmark_phases.snapshot())
7112    }
7113
7114    /// One serialized, read-only authority observation, including durable coverage.
7115    pub fn authority_snapshot(&self) -> Result<Value, String> {
7116        validate_authority_reads(&self.schema, &self.limits.authority_reads)?;
7117        if self.limits.authority_reads.is_empty() {
7118            return Err("client.authority_read_forbidden: no authority reads were declared at client creation".into());
7119        }
7120        self.conn
7121            .execute_batch("SAVEPOINT authority_snapshot")
7122            .map_err(|e| e.to_string())?;
7123        let result = (|| {
7124            let revision = self
7125                .get_meta(LOCAL_REVISION_KEY)?
7126                .ok_or("sync.local_corrupt: authority revision is missing")?;
7127            let parsed_revision = revision
7128                .parse::<u64>()
7129                .map_err(|_| "sync.local_corrupt: authority revision is invalid")?;
7130            if parsed_revision.to_string() != revision {
7131                return Err("sync.local_corrupt: authority revision is invalid".into());
7132            }
7133            let mut tables = Vec::new();
7134            for read in &self.limits.authority_reads {
7135                let table = self
7136                    .schema
7137                    .table(&read.table)
7138                    .ok_or("client.authority_read_forbidden: authority table is not declared")?;
7139                let scalar = |column: &str, value: &Value| -> Result<String, String> {
7140                    let ty = table
7141                        .columns
7142                        .iter()
7143                        .find(|c| c.name == column)
7144                        .expect("validated authority column")
7145                        .ty;
7146                    if ty == ColumnType::Boolean {
7147                        if let Some(value) = value.as_bool() {
7148                            return Ok(value.to_string());
7149                        }
7150                        return match value.as_i64() {
7151                            Some(0) => Ok("false".into()),
7152                            Some(1) => Ok("true".into()),
7153                            _ => Err("sync.local_corrupt: authority boolean is invalid".into()),
7154                        };
7155                    }
7156                    match value {
7157                        Value::String(s) => Ok(s.clone()),
7158                        Value::Number(n) => Ok(n.to_string()),
7159                        Value::Object(o) if ty == ColumnType::Integer && o.len() == 1 => o
7160                            .get("$bigint")
7161                            .and_then(Value::as_str)
7162                            .map(str::to_owned)
7163                            .ok_or("sync.local_corrupt: authority integer is invalid".into()),
7164                        _ => {
7165                            Err("sync.local_corrupt: authority scalar is missing or invalid".into())
7166                        }
7167                    }
7168                };
7169                let matches = |row: &QueryRow| -> Result<bool, String> {
7170                    for (variable, values) in &read.scopes {
7171                        let column = table
7172                            .scope_column(variable)
7173                            .expect("validated authority scope");
7174                        let value = row
7175                            .get(column)
7176                            .ok_or("sync.local_corrupt: authority scope column is missing")?;
7177                        if value.is_null() || !values.contains(&scalar(column, value)?) {
7178                            return Ok(false);
7179                        }
7180                    }
7181                    Ok(true)
7182                };
7183                let selected = read
7184                    .columns
7185                    .iter()
7186                    .map(|c| quote_ident(c))
7187                    .collect::<Vec<_>>()
7188                    .join(",");
7189                let mut intent = BTreeSet::new();
7190                let mut relevant = BTreeSet::new();
7191                for row in query_connection(&self.conn,
7192                    "SELECT id,values_json FROM (SELECT json_extract(value,'$.table') AS tbl,json_extract(value,'$.rowId') AS id,json_extract(value,'$.values') AS values_json FROM _syncular_outbox,json_each(ops_json) UNION ALL SELECT json_extract(value,'$.table'),json_extract(value,'$.rowId'),json_extract(value,'$.values') FROM _syncular_failed_commits,json_each(operations_json) UNION ALL SELECT tbl,id,intent_json FROM _syncular_acked_rows) WHERE tbl=?",
7193                    &[Value::String(read.table.clone())]).map_err(|e| e.to_string())? {
7194                    let id = row.get("id").and_then(Value::as_str).ok_or("sync.local_corrupt: authority intent id is invalid")?.to_owned();
7195                    intent.insert(id.clone());
7196                    if let Some(raw) = row.get("values_json").and_then(Value::as_str) {
7197                        let values: QueryRow = serde_json::from_str(raw).map_err(|_| "sync.local_corrupt: authority intent values are invalid")?;
7198                        if read.scopes.keys().all(|v| values.contains_key(table.scope_column(v).expect("validated scope"))) && matches(&values)? {relevant.insert(id);}
7199                    }
7200                }
7201                let mut rows = BTreeMap::new();
7202                for full in [base_table(&read.table), visible_table(&read.table)] {
7203                    for mut values in query_connection(
7204                        &self.conn,
7205                        &format!(
7206                            "SELECT {selected},_syncular_version AS authority_version FROM {full}"
7207                        ),
7208                        &[],
7209                    )
7210                    .map_err(|e| e.to_string())?
7211                    {
7212                        if !matches(&values)? {
7213                            continue;
7214                        }
7215                        let id = scalar(
7216                            &table.primary_key,
7217                            values
7218                                .get(&table.primary_key)
7219                                .expect("selected primary key"),
7220                        )?;
7221                        relevant.insert(id.clone());
7222                        if full == base_table(&read.table) {
7223                            let version = values
7224                                .remove("authority_version")
7225                                .expect("selected version");
7226                            let version = version
7227                                .as_i64()
7228                                .ok_or("sync.local_corrupt: authority server version is invalid")?;
7229                            if version >= 0 {
7230                                rows.insert(id.clone(), json!({"values": values, "version": version, "hasLocalIntent": intent.contains(&id)}));
7231                            }
7232                        }
7233                    }
7234                }
7235                intent.retain(|id| relevant.contains(id));
7236                let rows = rows.into_values().collect::<Vec<_>>();
7237                // Every unit covering the table, as its (unit, table) pair (§4.4).
7238                let replica = self.stored(replica_cursor(&self.conn))?;
7239                let persisted = self
7240                    .stored(load_units(&self.conn))?
7241                    .into_iter()
7242                    .filter(|unit| covers(table, &unit.scopes))
7243                    .map(|unit| {
7244                        let effective = unit.echo.as_deref().and_then(|echo| {
7245                            pair_scopes(&unit.scopes, &read.table, Some(echo))
7246                        });
7247                        let requested = read
7248                            .scopes
7249                            .keys()
7250                            .map(|v| (v.clone(), json!(unit.scopes.get(v).into_iter().collect::<Vec<_>>())))
7251                            .collect::<Map<_, _>>();
7252                        let complete = effective
7253                            .as_ref()
7254                            .is_some_and(|scopes| scopes.len() == read.scopes.len())
7255                            && unit.status == UnitStatus::Active
7256                            && !unit.bootstrapping();
7257                        let exposed = effective.map(|scopes| {
7258                            read.scopes
7259                                .keys()
7260                                .map(|v| {
7261                                    let held = scopes.iter().find(|(variable, _)| variable == v);
7262                                    (v.clone(), json!(held.map_or(&Vec::new(), |(_, values)| values)))
7263                                })
7264                                .collect::<Map<_, _>>()
7265                        });
7266                        json!({"requestedScopes": requested, "status": unit.status, "cursor": unit.cursor.unwrap_or(replica), "effectiveScopes": exposed, "complete": complete})
7267                    })
7268                    .collect::<Vec<_>>();
7269                fn covered(selectors: &[(&String, &Vec<String>)], candidates: &[&Value]) -> bool {
7270                    match selectors.split_first() {
7271                        None => !candidates.is_empty(),
7272                        Some(((variable, values), rest)) => values.iter().all(|value| {
7273                            covered(
7274                                rest,
7275                                &candidates
7276                                    .iter()
7277                                    .copied()
7278                                    .filter(|p| {
7279                                        p["effectiveScopes"][variable.as_str()]
7280                                            .as_array()
7281                                            .is_some_and(|held| {
7282                                                held.contains(&Value::String(value.clone()))
7283                                                    || held.contains(&Value::String("*".into()))
7284                                            })
7285                                    })
7286                                    .collect::<Vec<_>>(),
7287                            )
7288                        }),
7289                    }
7290                }
7291                let coverage = if covered(
7292                    &read.scopes.iter().collect::<Vec<_>>(),
7293                    &persisted
7294                        .iter()
7295                        .filter(|p| p["complete"] == true)
7296                        .collect::<Vec<_>>(),
7297                ) {
7298                    "complete"
7299                } else if persisted.iter().any(|p| {
7300                    p["status"] == "active"
7301                        && read.scopes.iter().all(|(v, values)| {
7302                            values.iter().any(|value| {
7303                                p["requestedScopes"][v].as_array().is_some_and(|held| {
7304                                    held.contains(&Value::String(value.clone()))
7305                                })
7306                            })
7307                        })
7308                }) {
7309                    "pending"
7310                } else {
7311                    "missing"
7312                };
7313                tables.push(json!({"table": read.table, "rows": rows, "localIntentRowIds": intent, "scopes": read.scopes, "coverage": coverage, "persisted": persisted}));
7314            }
7315            Ok(
7316                json!({"revision": revision, "complete": tables.iter().all(|t| t["coverage"] == "complete"), "tables": tables}),
7317            )
7318        })();
7319        match result {
7320            Ok(snapshot) => {
7321                self.conn
7322                    .execute_batch("RELEASE authority_snapshot")
7323                    .map_err(|e| e.to_string())?;
7324                Ok(snapshot)
7325            }
7326            Err(error) => {
7327                self.conn
7328                    .execute_batch("ROLLBACK TO authority_snapshot; RELEASE authority_snapshot")
7329                    .map_err(|e| format!("{error}; rollback failed: {e}"))?;
7330                Err(error)
7331            }
7332        }
7333    }
7334
7335    /// Rows, coverage, and local revision from one SQLite read snapshot.
7336    /// §7.5 atomic snapshot read. An `owner` records a failure in
7337    /// `diagnostics_snapshot().query_failures` until its next successful read.
7338    pub fn query_snapshot(
7339        &mut self,
7340        sql: &str,
7341        params: &[QueryValue],
7342        coverage: &[WindowCoverage],
7343        owner: Option<QueryOwner<'_>>,
7344    ) -> Result<QuerySnapshot, String> {
7345        let result = snapshot_connection(&self.conn, sql, params, coverage);
7346        if let Some(owner) = owner {
7347            self.record_query_read(owner, result.as_ref().err());
7348        }
7349        result.map_err(|failure| failure.to_string())
7350    }
7351
7352    /// §7.6: record the outcome of one owned snapshot read, including reads a
7353    /// host ran on a separate connection. A success removes the owner's entry;
7354    /// an identical failure leaves it unchanged; a changed failure moves last.
7355    pub fn record_query_read(&mut self, owner: QueryOwner<'_>, failure: Option<&QueryReadFailure>) {
7356        let position = self
7357            .query_failures
7358            .iter()
7359            .position(|entry| entry.id == owner.id);
7360        let Some(failure) = failure else {
7361            if let Some(index) = position {
7362                self.query_failures.remove(index);
7363            }
7364            return;
7365        };
7366        let tables: Vec<String> = BTreeSet::from_iter(owner.tables.iter().copied())
7367            .into_iter()
7368            .map(str::to_owned)
7369            .collect();
7370        let code = failure.code.unwrap_or("client.query_failed").to_owned();
7371        if position.is_some_and(|index| {
7372            let entry = &self.query_failures[index];
7373            entry.tables == tables && entry.code == code && entry.sqlite_code == failure.sqlite_code
7374        }) {
7375            return;
7376        }
7377        if let Some(index) = position {
7378            self.query_failures.remove(index);
7379        }
7380        self.query_failures.push(DiagnosticQueryFailure {
7381            id: owner.id.to_owned(),
7382            tables,
7383            code,
7384            sqlite_code: failure.sqlite_code,
7385            at_ms: self.clock_now_ms(),
7386        });
7387        if self.query_failures.len() > MAX_DIAGNOSTIC_QUERY_FAILURES {
7388            self.query_failures.remove(0);
7389        }
7390    }
7391
7392    // -- request building ---------------------------------------------------------
7393
7394    fn build_request(&self, url_capable: bool) -> Result<(Message, RequestMeta), Box<SyncOutcome>> {
7395        #[cfg(feature = "bench-internals")]
7396        let _phase = self.benchmark_phases.start(Phase::RequestPrepare);
7397        let log_epoch = self.get_meta(LOG_EPOCH_KEY).map_err(|message| {
7398            Box::new(SyncOutcome::Failed {
7399                error_code: "storage.failed".into(),
7400                message,
7401                details: None,
7402                recommended_action: None,
7403            })
7404        })?;
7405        let max_commits = self.limits.max_push_commits_per_request;
7406        let max_ops = self
7407            .limits
7408            .max_push_operations_per_request
7409            .unwrap_or(PUSH_OPS_PER_REQUEST);
7410        let byte_cap = self.limits.max_push_request_bytes;
7411        // §4.2/§5.4: bit 3 is advertised iff the transport can fetch a
7412        // bare URL — capability negotiation, decided per transport.
7413        let accept = self.limits.accept.unwrap_or(if url_capable {
7414            DEFAULT_ACCEPT | ACCEPT_SIGNED_URLS
7415        } else {
7416            DEFAULT_ACCEPT
7417        });
7418        let storage_failed = |message: String| {
7419            Box::new(SyncOutcome::Failed {
7420                error_code: "storage.failed".into(),
7421                message,
7422                details: None,
7423                recommended_action: None,
7424            })
7425        };
7426        let header = Frame::ReqHeader {
7427            client_id: self.client_id.clone(),
7428            schema_version: self.schema.version,
7429            log_epoch: log_epoch.clone(),
7430            outbox_id: self.get_meta(OUTBOX_ID_KEY).map_err(storage_failed)?,
7431        };
7432        // §1.5: the lead frames are REQ_HEADER plus the optional ROUND_FAILURE,
7433        // which stays ahead of every push, pull, unit and progress frame; the
7434        // byte-capacity probes below measure this exact sequence.
7435        let mut lead = vec![header];
7436        lead.extend(
7437            self.round_failure
7438                .iter()
7439                .map(|code| Frame::RoundFailure { code: code.clone() }),
7440        );
7441        let registered: Vec<UnitRecord> = self
7442            .stored(load_units(&self.conn))
7443            .map_err(storage_failed)?
7444            .into_iter()
7445            .filter(|unit| unit.status == UnitStatus::Active)
7446            .collect();
7447        let registration_digest = ssp2::registration::registration_digest(
7448            registered
7449                .iter()
7450                .map(|unit| scope_pairs(&unit.scopes))
7451                .collect::<Vec<_>>()
7452                .iter()
7453                .map(Vec::as_slice),
7454        );
7455        let echo_digest = self
7456            .get_meta(ECHO_DIGEST_KEY)
7457            .map_err(storage_failed)?
7458            .filter(|hex| !hex.is_empty())
7459            .map(|hex| {
7460                crate::values::hex_to_bytes(&hex)
7461                    .ok()
7462                    .and_then(|bytes| <[u8; 32]>::try_from(bytes).ok())
7463                    .ok_or_else(|| {
7464                        "sync.local_corrupt: persisted echo digest is invalid".to_owned()
7465                    })
7466            })
7467            .transpose()
7468            .map_err(storage_failed)?;
7469        let pull = Frame::PullHeader {
7470            limit_commits: self.limits.limit_commits.unwrap_or(0),
7471            limit_snapshot_rows: self.limits.limit_snapshot_rows.unwrap_or(0),
7472            max_snapshot_pages: self.limits.max_snapshot_pages.unwrap_or(0),
7473            accept,
7474            cursor: self
7475                .stored(replica_cursor(&self.conn))
7476                .map_err(storage_failed)?,
7477            registration_digest,
7478            echo_digest,
7479        };
7480        let registration_digest = bytes_to_hex(&registration_digest);
7481        // §4.1: the list travels when it changed since the server stored it,
7482        // after `unregistered` and after a reset (both clear the stored digest).
7483        let travels = self
7484            .get_meta(REGISTERED_DIGEST_KEY)
7485            .map_err(storage_failed)?
7486            .as_deref()
7487            != Some(registration_digest.as_str());
7488        let mut sub_frames: Vec<Frame> = Vec::new();
7489        if travels {
7490            sub_frames.extend(registered.iter().map(|unit| Frame::Unit {
7491                scopes: scope_pairs(&unit.scopes),
7492            }));
7493        }
7494        // §4.3: one PROGRESS per catching-up unit, in registration order.
7495        sub_frames.extend(registered.iter().filter_map(|unit| {
7496            Some(Frame::Progress {
7497                id: unit.id.clone(),
7498                cursor: unit.cursor?,
7499                bootstrap_state: unit.bootstrap_state.clone().map(RawJson),
7500            })
7501        }));
7502        // The fixed frames (order-independent size) and the header+pull probe
7503        // both encode canonically, so the byte budget is the exact wire size.
7504        let fixed_bytes = match byte_cap {
7505            Some(_) => {
7506                let mut frames = lead.clone();
7507                frames.push(pull.clone());
7508                frames.extend(sub_frames.iter().cloned());
7509                encode_message(&Message {
7510                    wire_version: WIRE_VERSION,
7511                    msg_kind: MsgKind::Request,
7512                    frames,
7513                })
7514                .len()
7515            }
7516            None => 0,
7517        };
7518        if let Some(cap) = byte_cap {
7519            if fixed_bytes > cap {
7520                return Err(Self::push_capacity_outcome("bytes", cap, fixed_bytes, None));
7521            }
7522        }
7523        let base_bytes = match byte_cap {
7524            Some(_) => {
7525                let mut frames = lead.clone();
7526                frames.push(pull.clone());
7527                encode_message(&Message {
7528                    wire_version: WIRE_VERSION,
7529                    msg_kind: MsgKind::Request,
7530                    frames,
7531                })
7532                .len()
7533            }
7534            None => 0,
7535        };
7536        let total_outbox = if log_epoch.is_some() {
7537            self.outbox.len()
7538        } else {
7539            0
7540        };
7541        let mut push_frames: Vec<Frame> = Vec::new();
7542        let mut pushed_ids = Vec::new();
7543        let mut ops_in_request = 0usize;
7544        let mut bytes_in_request = 0usize;
7545        let mut deferred_commits = 0usize;
7546        #[cfg(feature = "bench-internals")]
7547        let outbox_phase = self.benchmark_phases.start(Phase::OutboxEncode);
7548        for (index, commit) in self.outbox.iter().take(total_outbox).enumerate() {
7549            // §6.1/§7.1: the commit-count and operation caps are hard. A
7550            // first commit over the operation cap is a typed capacity error
7551            // and stays queued; a later one defers with the complete suffix.
7552            if max_commits.is_some_and(|cap| push_frames.len() >= cap) {
7553                deferred_commits = total_outbox - index;
7554                break;
7555            }
7556            if ops_in_request + commit.ops.len() > max_ops {
7557                if push_frames.is_empty() {
7558                    return Err(Self::push_capacity_outcome(
7559                        "operations",
7560                        max_ops,
7561                        commit.ops.len(),
7562                        Some(commit.client_commit_id.as_str()),
7563                    ));
7564                }
7565                deferred_commits = total_outbox - index;
7566                break;
7567            }
7568            // §7.1 chained base: the commit waits for the change of the commit
7569            // it chains on, which supplies its rebased version.
7570            if self.outbox[..index]
7571                .iter()
7572                .flat_map(|earlier| &earlier.ops)
7573                .any(|earlier| commit.ops.iter().any(|op| chains_on(earlier, op)))
7574            {
7575                deferred_commits = total_outbox - index;
7576                break;
7577            }
7578            let operations = commit
7579                .ops
7580                .iter()
7581                .map(|op| {
7582                    let payload = match self
7583                        .schema
7584                        .table(&op.table)
7585                        .and_then(|table| Some((table, op.presented_values(table)?)))
7586                    {
7587                        Some((table, values)) => {
7588                            // §0: outbox entries encode at send time with the
7589                            // current codec (validated at mutate()). §6.1: the
7590                            // values map's keys are the sparse presence set; an
7591                            // insert-intent operation presents every column (§7.1).
7592                            // §5.11: encrypted columns are encrypted here, with
7593                            // an absent key-id selector read from the stored row.
7594                            // `sync_inner` runs `drop_unencodable_outbox` first,
7595                            // so an encrypt failure cannot reach this encode; the
7596                            // author seam validated every other failure.
7597                            let fallback =
7598                                self.stored_key_fallback(table, &op.row_id)
7599                                    .map_err(|message| {
7600                                        Box::new(SyncOutcome::Failed {
7601                                            error_code: "storage.failed".into(),
7602                                            message,
7603                                            details: None,
7604                                            recommended_action: None,
7605                                        })
7606                                    })?;
7607                            encode_sparse_row_json(
7608                                table,
7609                                &op.row_id,
7610                                &values,
7611                                &self.encryption,
7612                                fallback.as_ref(),
7613                            )
7614                            .ok()
7615                        }
7616                        None => None,
7617                    };
7618                    Ok(ssp2::model::Operation {
7619                        table: op.table.clone(),
7620                        row_id: op.row_id.clone(),
7621                        op: if op.upsert { Op::Upsert } else { Op::Delete },
7622                        base_version: op.base_version,
7623                        payload,
7624                    })
7625                })
7626                .collect::<Result<Vec<_>, Box<SyncOutcome>>>()?;
7627            // §6.3.2, §7.1: the commit's outbox sequence.
7628            let client_seq = self
7629                .conn
7630                .prepare_cached("SELECT seq FROM _syncular_outbox WHERE commit_id = ?1")
7631                .and_then(|mut statement| {
7632                    statement.query_row([&commit.client_commit_id], |row| row.get(0))
7633                })
7634                .map_err(|error| {
7635                    storage_failed(Self::sqlite_failure(&self.storage_failure, error))
7636                })?;
7637            let frame = Frame::PushCommit {
7638                client_commit_id: commit.client_commit_id.clone(),
7639                client_seq,
7640                operations,
7641            };
7642            let frame_bytes = match byte_cap {
7643                Some(_) => {
7644                    let mut frames = lead.clone();
7645                    frames.push(frame.clone());
7646                    frames.push(pull.clone());
7647                    encode_message(&Message {
7648                        wire_version: WIRE_VERSION,
7649                        msg_kind: MsgKind::Request,
7650                        frames,
7651                    })
7652                    .len()
7653                        - base_bytes
7654                }
7655                None => 0,
7656            };
7657            // The head is always encoded so the caller can report the full
7658            // projected request size; a later commit defers once it no longer
7659            // fits.
7660            if let Some(cap) = byte_cap {
7661                if !push_frames.is_empty() && fixed_bytes + bytes_in_request + frame_bytes > cap {
7662                    deferred_commits = total_outbox - index;
7663                    break;
7664                }
7665            }
7666            push_frames.push(frame);
7667            pushed_ids.push(commit.client_commit_id.clone());
7668            ops_in_request += commit.ops.len();
7669            bytes_in_request += frame_bytes;
7670        }
7671        #[cfg(feature = "bench-internals")]
7672        drop(outbox_phase);
7673        if let Some(cap) = byte_cap {
7674            if fixed_bytes + bytes_in_request > cap {
7675                // The head alone does not fit this budget; the commit stays
7676                // queued and no transport round is attempted.
7677                let blocked = if push_frames.is_empty() {
7678                    None
7679                } else {
7680                    self.outbox.first().map(|c| c.client_commit_id.as_str())
7681                };
7682                return Err(Self::push_capacity_outcome(
7683                    "bytes",
7684                    cap,
7685                    fixed_bytes + bytes_in_request,
7686                    blocked,
7687                ));
7688            }
7689        }
7690        let mut frames = lead;
7691        frames.extend(push_frames);
7692        frames.push(pull);
7693        frames.extend(sub_frames);
7694        let message = Message {
7695            wire_version: WIRE_VERSION,
7696            msg_kind: MsgKind::Request,
7697            frames,
7698        };
7699        Ok((
7700            message,
7701            RequestMeta {
7702                pushed_ids,
7703                registration_digest,
7704                accept,
7705                deferred_commits,
7706                applied: BTreeMap::new(),
7707                deferred: BTreeSet::new(),
7708            },
7709        ))
7710    }
7711
7712    /// §7.1: a configured push capacity limit cannot carry this request.
7713    fn push_capacity_outcome(
7714        kind: &str,
7715        limit: usize,
7716        size: usize,
7717        client_commit_id: Option<&str>,
7718    ) -> Box<SyncOutcome> {
7719        let mut details = Map::new();
7720        details.insert("kind".to_owned(), Value::from(kind));
7721        details.insert("limit".to_owned(), Value::from(limit as u64));
7722        details.insert("size".to_owned(), Value::from(size as u64));
7723        if let Some(id) = client_commit_id {
7724            details.insert("clientCommitId".to_owned(), Value::from(id));
7725        }
7726        Box::new(SyncOutcome::Failed {
7727            error_code: "client.push_request_too_large".into(),
7728            message: "the push request exceeds a configured capacity limit".into(),
7729            details: Some(Value::Object(details)),
7730            recommended_action: None,
7731        })
7732    }
7733
7734    // -- sync -------------------------------------------------------------------
7735
7736    /// Cloneable observation handle; no client lock is needed to read it.
7737    pub fn progress(&self) -> ProgressObserver {
7738        self.progress.clone()
7739    }
7740
7741    pub fn sync(&mut self, transport: &mut dyn Transport) -> SyncOutcome {
7742        self.sync_bounded(transport, None)
7743    }
7744
7745    /// One round; `idle_bound` is the idle drive's round bound (§7.8).
7746    fn sync_bounded(
7747        &mut self,
7748        transport: &mut dyn Transport,
7749        idle_bound: Option<(u64, u32)>,
7750    ) -> SyncOutcome {
7751        let mut prepared = match self.prepare_sync_round(transport.supports_url_fetch()) {
7752            Ok(prepared) => prepared,
7753            Err(outcome) => return *outcome,
7754        };
7755        prepared.idle_bound = idle_bound;
7756        // The round's network deadline rides `PreparedSyncRound` and is scoped
7757        // to each exchange, so uploads, continuations, the main request, and
7758        // every segment fetch share one monotonic anchor without leaving
7759        // deadline state on a reused transport.
7760        let mut next = prepared;
7761        loop {
7762            match self.apply_sync_round(next.exchange(transport)) {
7763                crate::AppliedSyncRound::Continue(prepared) => next = prepared,
7764                crate::AppliedSyncRound::Complete {
7765                    outcome, controls, ..
7766                } => {
7767                    if self.transport_enabled {
7768                        for text in controls {
7769                            let _ = transport.realtime_send(&text);
7770                        }
7771                    } else {
7772                        self.disconnect_realtime(transport);
7773                    }
7774                    return outcome;
7775                }
7776            }
7777        }
7778    }
7779
7780    fn finish_sync_round(&mut self, started_at_ms: i64, mut outcome: SyncOutcome) -> SyncOutcome {
7781        if let SyncOutcome::Failed {
7782            error_code,
7783            message,
7784            details,
7785            ..
7786        } = &mut outcome
7787        {
7788            if let Some((failure, code)) = self
7789                .storage_failure
7790                .borrow()
7791                .as_ref()
7792                .and_then(|failure| failure.code.map(|code| (failure, code)))
7793            {
7794                *error_code = code.to_owned();
7795                *message = match code {
7796                    "client.storage_busy" => "local SQLite storage is busy",
7797                    "client.storage_full" => "local SQLite storage is full",
7798                    "client.storage_io" => "local SQLite storage I/O failed",
7799                    "client.storage_corrupt" => "local SQLite storage is corrupt",
7800                    _ => unreachable!("unrecognized classified storage failure"),
7801                }
7802                .to_owned();
7803                *details = failure.details();
7804            }
7805        }
7806        let details = match &outcome {
7807            SyncOutcome::Failed { details, .. } => details.as_ref(),
7808            _ => None,
7809        };
7810        let cause_kind = details
7811            .and_then(|details| details.get("causeKind"))
7812            .and_then(Value::as_str)
7813            .and_then(|cause| match cause {
7814                "network" => Some(crate::TransportCauseKind::Network),
7815                "connect" => Some(crate::TransportCauseKind::Connect),
7816                "tls" => Some(crate::TransportCauseKind::Tls),
7817                "io" => Some(crate::TransportCauseKind::Io),
7818                "request" => Some(crate::TransportCauseKind::Request),
7819                "protocol" => Some(crate::TransportCauseKind::Protocol),
7820                "timeout" => Some(crate::TransportCauseKind::Timeout),
7821                "round_deadline" => Some(crate::TransportCauseKind::RoundDeadline),
7822                "aborted" => Some(crate::TransportCauseKind::Aborted),
7823                "body" => Some(crate::TransportCauseKind::Body),
7824                "status" => Some(crate::TransportCauseKind::Status),
7825                "redirect" => Some(crate::TransportCauseKind::Redirect),
7826                "unknown" => Some(crate::TransportCauseKind::Unknown),
7827                _ => None,
7828            });
7829        let http_status = if matches!(
7830            cause_kind,
7831            Some(
7832                crate::TransportCauseKind::Status
7833                    | crate::TransportCauseKind::Body
7834                    | crate::TransportCauseKind::Redirect
7835            )
7836        ) {
7837            details
7838                .and_then(|details| details.get("httpStatus"))
7839                .and_then(Value::as_f64)
7840                .filter(|status| status.fract() == 0.0 && (100.0..=599.0).contains(status))
7841                .map(|status| status as u16)
7842        } else {
7843            None
7844        };
7845        let retry_delay_ms = self.round_retry_delay_ms;
7846        self.progress.update(|p| match &outcome {
7847            SyncOutcome::Ok(report) | SyncOutcome::BudgetExhausted(report)
7848                if report.failed.is_empty() =>
7849            {
7850                p.state = ProgressState::Complete
7851            }
7852            SyncOutcome::Ok(_) | SyncOutcome::BudgetExhausted(_) => {
7853                p.state = ProgressState::Failed;
7854                p.error_code = Some("sync.scope_revoked".into());
7855            }
7856            SyncOutcome::Failed { error_code, .. } => {
7857                p.state = ProgressState::Failed;
7858                p.error_code = Some(Self::diagnostic_code(error_code));
7859                p.cause_kind = cause_kind;
7860                p.http_status = http_status;
7861                p.retry_delay_ms = retry_delay_ms;
7862            }
7863            SyncOutcome::RealtimeUnavailable { .. } => {
7864                p.state = ProgressState::Failed;
7865                p.error_code = Some(REALTIME_UNAVAILABLE_CODE.to_owned());
7866                p.retry_delay_ms = retry_delay_ms;
7867            }
7868        });
7869        let completed_at_ms = self.clock_now_ms();
7870        self.last_round = Some(match &outcome {
7871            SyncOutcome::Ok(report) | SyncOutcome::BudgetExhausted(report) => DiagnosticLastRound {
7872                status: "succeeded".to_owned(),
7873                started_at_ms,
7874                completed_at_ms,
7875                duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
7876                counters: Some(DiagnosticRoundCounters {
7877                    pushed: report.pushed,
7878                    applied: report.applied.len(),
7879                    rejected: report.rejected.len(),
7880                    retryable: report.retryable.len(),
7881                    conflicts: report.conflicts,
7882                    commits_applied: report.commits_applied,
7883                    segment_rows_applied: report.segment_rows_applied,
7884                    bootstrapping: report.bootstrapping.len(),
7885                    resets: report.resets.len(),
7886                    revoked: report.revoked.len(),
7887                    failed: report.failed.len(),
7888                    deferred_commits: report.deferred_commits,
7889                }),
7890                error_code: None,
7891                cause_kind: None,
7892                http_status: None,
7893            },
7894            SyncOutcome::Failed { error_code, .. } => DiagnosticLastRound {
7895                status: "failed".to_owned(),
7896                started_at_ms,
7897                completed_at_ms,
7898                duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
7899                counters: None,
7900                error_code: Some(Self::diagnostic_code(error_code)),
7901                cause_kind,
7902                http_status,
7903            },
7904            SyncOutcome::RealtimeUnavailable { .. } => DiagnosticLastRound {
7905                status: "failed".to_owned(),
7906                started_at_ms,
7907                completed_at_ms,
7908                duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
7909                counters: None,
7910                error_code: Some(REALTIME_UNAVAILABLE_CODE.to_owned()),
7911                cause_kind: None,
7912                http_status: None,
7913            },
7914        });
7915        outcome
7916    }
7917
7918    /// Capture one request on the client owner. Mutations made after this
7919    /// call remain in the outbox for a later round; no network I/O occurs here.
7920    pub fn prepare_sync_round(
7921        &mut self,
7922        url_capable: bool,
7923    ) -> Result<crate::PreparedSyncRound, Box<SyncOutcome>> {
7924        if !self.transport_enabled {
7925            return Err(Box::new(SyncOutcome::Failed {
7926                error_code: "sync.offline".into(),
7927                message: "transport gate is closed".into(),
7928                details: None,
7929                recommended_action: None,
7930            }));
7931        }
7932        if self.active_round.is_some() {
7933            return Err(Box::new(SyncOutcome::Failed {
7934                error_code: "client.round_in_progress".into(),
7935                message: "a sync round is already in flight".into(),
7936                details: None,
7937                recommended_action: None,
7938            }));
7939        }
7940        self.storage_failure.borrow_mut().take();
7941        self.progress.start();
7942        self.round_retry_delay_ms = None;
7943        let started_at_ms = self.clock_now_ms();
7944        let prepared = (|| {
7945            if self.stopped {
7946                return Err(Box::new(SyncOutcome::Ok(SyncReport {
7947                    schema_floor: self.schema_floor.clone(),
7948                    ..SyncReport::default()
7949                })));
7950            }
7951            if self.security_preflight {
7952                return Err(Box::new(SyncOutcome::Failed {
7953                    error_code: SECURITY_PREFLIGHT_REQUIRED_CODE.into(),
7954                    message: "security preflight must complete before syncing".into(),
7955                    details: None,
7956                    recommended_action: None,
7957                }));
7958            }
7959            self.set_sync_needed(false, false);
7960            self.drain_pending_evictions()
7961                .and_then(|_| self.drop_incompatible_outbox())
7962                .and_then(|_| self.drop_unencodable_outbox(true, None))
7963                .map_err(|message| {
7964                    Box::new(SyncOutcome::Failed {
7965                        error_code: "storage.failed".into(),
7966                        message,
7967                        details: None,
7968                        recommended_action: None,
7969                    })
7970                })?;
7971            // §5.11: reveal rows sealed under a key the keyring gained since.
7972            self.reveal_sealed_rows().map_err(|error| {
7973                Box::new(SyncOutcome::Failed {
7974                    error_code: error.code,
7975                    message: error.message,
7976                    details: error.details,
7977                    recommended_action: None,
7978                })
7979            })?;
7980            let has_log_epoch = self.get_meta(LOG_EPOCH_KEY).map_err(|message| {
7981                Box::new(SyncOutcome::Failed {
7982                    error_code: "storage.failed".into(),
7983                    message,
7984                    details: None,
7985                    recommended_action: None,
7986                })
7987            })?;
7988            let uploads = if has_log_epoch.is_some() && self.schema_has_blobs() {
7989                self.pending_blob_uploads().map_err(|e| {
7990                    Box::new(SyncOutcome::Failed {
7991                        error_code: e.code,
7992                        message: e.message,
7993                        details: e.details,
7994                        recommended_action: None,
7995                    })
7996                })?
7997            } else {
7998                Vec::new()
7999            };
8000            if self.realtime_state != RealtimeState::Connected
8001                && self.realtime_policy == RealtimePolicy::Required
8002            {
8003                let retry_delay_ms = self.schedule_background_retry();
8004                self.realtime_retry_delay_ms = Some(retry_delay_ms);
8005                return Err(Box::new(SyncOutcome::RealtimeUnavailable {
8006                    state: self.realtime_state,
8007                    reason_code: self.realtime_reason_code.clone(),
8008                    retry_delay_ms,
8009                }));
8010            }
8011            let (message, mut meta) = self.build_request(url_capable)?;
8012            let units = self.stored(load_units(&self.conn)).map_err(|message| {
8013                Box::new(SyncOutcome::Failed {
8014                    error_code: "storage.failed".into(),
8015                    message,
8016                    details: None,
8017                    recommended_action: None,
8018                })
8019            })?;
8020            for unit in &units {
8021                let applied =
8022                    self.stored(applied_rows(&self.conn, &unit.id))
8023                        .map_err(|message| {
8024                            Box::new(SyncOutcome::Failed {
8025                                error_code: "storage.failed".into(),
8026                                message,
8027                                details: None,
8028                                recommended_action: None,
8029                            })
8030                        })?;
8031                if !applied.is_empty() {
8032                    meta.applied.insert(unit.id.clone(), applied);
8033                }
8034            }
8035            let id = uuid::Uuid::new_v4();
8036            self.active_round = Some(id);
8037            let phase = if uploads.is_empty() {
8038                crate::round::RoundPhase::Sync
8039            } else {
8040                crate::round::RoundPhase::UploadGrant
8041            };
8042            Ok(crate::PreparedSyncRound {
8043                id,
8044                started_at_ms,
8045                message,
8046                meta,
8047                uploads: uploads.into(),
8048                phase,
8049                encoded: None,
8050                continuation: None,
8051                completed: None,
8052                realtime: self.realtime_state == RealtimeState::Connected,
8053                // §5.5: the owning unit's scope map, each value a one-element list.
8054                scopes: units
8055                    .into_iter()
8056                    .map(|unit| {
8057                        (
8058                            unit.id,
8059                            canonical_scope_json(&unit_scope_lists(&unit.scopes)),
8060                        )
8061                    })
8062                    .collect(),
8063                #[cfg(feature = "bench-internals")]
8064                benchmark_phases: self.benchmark_phases.clone(),
8065                progress: self.progress.clone(),
8066                fixed_now: self
8067                    .now_ms
8068                    .as_ref()
8069                    .map(|clock| clock.load(std::sync::atomic::Ordering::Relaxed)),
8070                round_deadline_at: None,
8071                idle_bound: None,
8072            })
8073        })();
8074        prepared.map_err(|outcome| Box::new(self.finish_sync_round(started_at_ms, *outcome)))
8075    }
8076
8077    /// Discard the reply of an old authorization or registration context. Its
8078    /// submitted commits stay durable and can be retried with the same ids.
8079    pub fn cancel_sync_round(&mut self) {
8080        self.active_round = None;
8081    }
8082
8083    /// Apply captured network results on the owner, preserving all normal
8084    /// version, schema, content-address and cursor checks. No network I/O.
8085    pub fn apply_sync_round(
8086        &mut self,
8087        completed: crate::CompletedSyncRound,
8088    ) -> crate::AppliedSyncRound {
8089        let mut prepared = completed.prepared;
8090        if self.active_round != Some(prepared.id) || self.security_preflight {
8091            return crate::AppliedSyncRound::Complete {
8092                outcome: SyncOutcome::Failed {
8093                    error_code: "client.round_cancelled".into(),
8094                    message: "sync round belongs to a released client context".into(),
8095                    details: None,
8096                    recommended_action: None,
8097                },
8098                controls: Vec::new(),
8099                more: false,
8100                bootstrap_advanced: false,
8101            };
8102        }
8103        self.storage_failure.borrow_mut().take();
8104        let (response, mut downloads, transport_failed) = match completed.exchange {
8105            crate::round::ExchangeResult::Upload { id, result } => {
8106                let result = match result {
8107                    // §5.9.7: a non-retryable server refusal (`blob.too_large`)
8108                    // is the body's final outcome. The commits naming it take
8109                    // the incompatible-commit path with the server's code and
8110                    // leave this round's request; the round continues.
8111                    Err(error)
8112                        if error
8113                            .details
8114                            .as_ref()
8115                            .and_then(|details| details.get("retryable"))
8116                            == Some(&Value::Bool(false)) =>
8117                    {
8118                        self.conn
8119                            .prepare(
8120                                "SELECT commit_id FROM _syncular_blob_commit_refs WHERE blob_id = ?",
8121                            )
8122                            .and_then(|mut statement| {
8123                                statement
8124                                    .query_map(rusqlite::params![id], |row| row.get::<_, String>(0))?
8125                                    .collect::<Result<HashSet<_>, _>>()
8126                            })
8127                            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
8128                            .and_then(|refused| {
8129                                let commits = self
8130                                    .outbox
8131                                    .iter()
8132                                    .filter(|commit| refused.contains(&commit.client_commit_id))
8133                                    .cloned()
8134                                    .collect::<Vec<_>>();
8135                                if !commits.is_empty() {
8136                                    self.drop_outbox_commits(&commits, &error.code, &error.message)?;
8137                                }
8138                                prepared.message.frames.retain(|frame| {
8139                                    !matches!(frame, Frame::PushCommit { client_commit_id, .. }
8140                                        if refused.contains(client_commit_id))
8141                                });
8142                                prepared.meta.pushed_ids.retain(|id| !refused.contains(id));
8143                                prepared.continuation = Some(crate::round::RoundPhase::UploadGrant);
8144                                Ok(())
8145                            })
8146                            .map_err(|message| TransportError::new("client.failed", message))
8147                    }
8148                    result => result,
8149                };
8150                let result = result.and_then(|()| {
8151                    self.conn
8152                        .execute(
8153                            "DELETE FROM _syncular_blob_uploads WHERE blob_id = ?",
8154                            rusqlite::params![id],
8155                        )
8156                        .map(|_| ())
8157                        .map_err(|error| {
8158                            TransportError::new(
8159                                "client.failed",
8160                                Self::sqlite_failure(&self.storage_failure, error),
8161                            )
8162                        })
8163                });
8164                match result {
8165                    Ok(()) => return crate::AppliedSyncRound::Continue(prepared),
8166                    Err(TransportError {
8167                        code,
8168                        message,
8169                        details,
8170                    }) => {
8171                        self.active_round = None;
8172                        if Self::unreachable_transport(&details) {
8173                            // The round's work is still owed; the host's
8174                            // connectivity resume runs it.
8175                            self.set_sync_needed(true, false);
8176                        } else if Self::retryable_failure_code(&code) {
8177                            self.schedule_background_retry();
8178                        }
8179                        let recommended_action = recommended_action_of(&details);
8180                        let outcome = self.finish_sync_round(
8181                            prepared.started_at_ms,
8182                            SyncOutcome::Failed {
8183                                error_code: code,
8184                                message,
8185                                details,
8186                                recommended_action,
8187                            },
8188                        );
8189                        return crate::AppliedSyncRound::Complete {
8190                            outcome,
8191                            controls: Vec::new(),
8192                            more: false,
8193                            bootstrap_advanced: false,
8194                        };
8195                    }
8196                }
8197            }
8198            crate::round::ExchangeResult::Reply {
8199                response,
8200                downloads,
8201                transport_failed,
8202            } => (response, downloads, transport_failed),
8203        };
8204        self.active_round = None;
8205        let before = self.resume_tokens();
8206        let outcome = (|| {
8207            let response = match response {
8208                Ok(response) => {
8209                    // §1.5: a complete, decodable response ends the report, an
8210                    // in-band ERROR included (the server already recorded it).
8211                    self.round_failure = None;
8212                    response
8213                }
8214                Err(TransportError {
8215                    code,
8216                    message,
8217                    details,
8218                }) => {
8219                    // §1.5: the server cannot see a round whose transport
8220                    // failed or whose response never decoded, so its code rides
8221                    // the following requests. A code the frame grammar rejects
8222                    // is not reportable, and the previous code stays; the
8223                    // client-local drive timeout never goes on the wire (§7.8).
8224                    if code != "sync.idle_timeout" && ssp2::model::valid_round_failure_code(&code) {
8225                        self.round_failure = Some(code.clone());
8226                    }
8227                    if transport_failed {
8228                        if prepared.realtime {
8229                            self.set_realtime_state(RealtimeState::Lost, Some(&code));
8230                        }
8231                        if let Err(message) = self.record_lease_error(&code) {
8232                            return SyncOutcome::Failed {
8233                                error_code: "storage.failed".to_owned(),
8234                                message,
8235                                details: None,
8236                                recommended_action: None,
8237                            };
8238                        }
8239                    }
8240                    if Self::retryable_failure_code(&code) && !Self::unreachable_transport(&details)
8241                    {
8242                        let delay = self.schedule_background_retry();
8243                        if prepared.realtime && transport_failed {
8244                            self.realtime_retry_delay_ms = Some(delay);
8245                        }
8246                    }
8247                    let recommended_action = recommended_action_of(&details);
8248                    return SyncOutcome::Failed {
8249                        error_code: code,
8250                        message,
8251                        details,
8252                        recommended_action,
8253                    };
8254                }
8255            };
8256            let mut outcome = self.process_response(&mut downloads, response, &prepared.meta);
8257            if let SyncOutcome::Ok(report) = &mut outcome {
8258                report.deferred_commits = prepared.meta.deferred_commits;
8259            }
8260            match &outcome {
8261                SyncOutcome::Ok(_) => self.reset_background_retry(),
8262                SyncOutcome::Failed {
8263                    error_code,
8264                    details,
8265                    ..
8266                } if Self::retryable_failure_code(error_code)
8267                    && !Self::unreachable_transport(details) =>
8268                {
8269                    self.schedule_background_retry();
8270                }
8271                _ => {}
8272            }
8273            if prepared.meta.deferred_commits > 0 {
8274                self.set_sync_needed(true, true);
8275            }
8276            self.previous_version_lifetime_check();
8277            outcome
8278        })();
8279        match &outcome {
8280            // §7.1: a unit still catching up needs another round, also when the
8281            // drive that started it ends early.
8282            SyncOutcome::Ok(report) if !report.bootstrapping.is_empty() => {
8283                self.set_sync_needed(true, false)
8284            }
8285            // The round's work is still owed; the host's connectivity resume
8286            // runs it.
8287            SyncOutcome::Failed { details, .. } if Self::unreachable_transport(details) => {
8288                self.set_sync_needed(true, false)
8289            }
8290            _ => {}
8291        }
8292        let more = self.transport_enabled
8293            && matches!(&outcome, SyncOutcome::Ok(report)
8294                if report.schema_floor.is_none()
8295                    && (!report.bootstrapping.is_empty()
8296                        || report.commits_applied > 0
8297                        || report.segment_rows_applied > 0
8298                        || !report.resets.is_empty()
8299                        || report.deferred_commits > 0
8300                        || self.sync_needed));
8301        let bootstrap_advanced = matches!(&outcome, SyncOutcome::Ok(report) if report.segment_rows_applied > 0 && !report.bootstrapping.is_empty())
8302            && before != self.resume_tokens();
8303        crate::AppliedSyncRound::Complete {
8304            outcome: self.finish_sync_round(prepared.started_at_ms, outcome),
8305            controls: if self.transport_enabled {
8306                downloads.controls
8307            } else {
8308                Vec::new()
8309            },
8310            more,
8311            bootstrap_advanced,
8312        }
8313    }
8314
8315    /// Every unit's resume token and applied-segment record: a bootstrap
8316    /// that advanced changes one (§4.7), also when it applied images of a
8317    /// section it deferred (§5.3, §5.6).
8318    #[allow(clippy::type_complexity)]
8319    fn resume_tokens(&self) -> Option<Vec<(String, Option<String>, Option<String>)>> {
8320        self.conn
8321            .prepare_cached(
8322                "SELECT id, bootstrap_state, applied FROM _syncular_units ORDER BY rowid",
8323            )
8324            .and_then(|mut stmt| {
8325                stmt.query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))?
8326                    .collect()
8327            })
8328            .ok()
8329    }
8330
8331    /// §7.8: one idle drive's default total deadline, the TS client's value.
8332    pub const SYNC_IDLE_TIMEOUT_MS: u64 = 60_000;
8333
8334    pub fn sync_until_idle(
8335        &mut self,
8336        transport: &mut dyn Transport,
8337        max_rounds: Option<u32>,
8338    ) -> SyncOutcome {
8339        self.sync_until_idle_with_deadline(transport, max_rounds, Some(Self::SYNC_IDLE_TIMEOUT_MS))
8340    }
8341
8342    /// As [`SyncClient::sync_until_idle`], with an explicit total deadline in
8343    /// milliseconds (`None` disables it). The deadline is checked against the
8344    /// client clock between rounds: a drive past it after a completed round
8345    /// ends as `BudgetExhausted`. Each round is bounded by `timeout_ms` on its
8346    /// own, or the transport's tighter whole-round deadline (§1.8); a round
8347    /// that reaches the drive bound fails with `sync.idle_timeout`.
8348    pub fn sync_until_idle_with_deadline(
8349        &mut self,
8350        transport: &mut dyn Transport,
8351        max_rounds: Option<u32>,
8352        timeout_ms: Option<u64>,
8353    ) -> SyncOutcome {
8354        // §7.8: an explicit budget is a strict positive round count. Zero is a
8355        // caller error, not an exhaustion.
8356        if max_rounds == Some(0) {
8357            return SyncOutcome::Failed {
8358                details: None,
8359                error_code: "sync.invalid_request".into(),
8360                message: "maxRounds must be a positive integer".into(),
8361                recommended_action: None,
8362            };
8363        }
8364        if timeout_ms.is_some_and(|ms| ms == 0 || i64::try_from(ms).is_err()) {
8365            return SyncOutcome::Failed {
8366                details: None,
8367                error_code: "sync.invalid_request".into(),
8368                message: "timeoutMs must be a positive integer".into(),
8369                recommended_action: None,
8370            };
8371        }
8372        let started_at_ms = self.clock_now_ms();
8373        let rounds = max_rounds.unwrap_or(20);
8374        let mut aggregate = SyncReport::default();
8375        let mut spent = 0;
8376        let mut completed = false;
8377        while spent < rounds {
8378            // §7.8: a deadline that elapsed across completed rounds ends the
8379            // drive like an exhausted round budget; the next drive resumes.
8380            if completed
8381                && timeout_ms.is_some_and(|ms| {
8382                    self.clock_now_ms() >= started_at_ms.saturating_add(ms as i64)
8383                })
8384            {
8385                return SyncOutcome::BudgetExhausted(aggregate);
8386            }
8387            let idle_bound = timeout_ms.map(|ms| (ms, spent));
8388            spent += 1;
8389            let before = self.resume_tokens();
8390            match self.sync_bounded(transport, idle_bound) {
8391                SyncOutcome::Failed {
8392                    error_code,
8393                    message,
8394                    details,
8395                    recommended_action,
8396                } => {
8397                    return SyncOutcome::Failed {
8398                        details,
8399                        error_code,
8400                        message,
8401                        recommended_action,
8402                    };
8403                }
8404                outcome @ SyncOutcome::RealtimeUnavailable { .. } => return outcome,
8405                SyncOutcome::BudgetExhausted(_) => {
8406                    unreachable!("sync() never exhausts a round budget")
8407                }
8408                SyncOutcome::Ok(report) => {
8409                    completed = true;
8410                    if max_rounds.is_none()
8411                        && report.segment_rows_applied > 0
8412                        && !report.bootstrapping.is_empty()
8413                        && before != self.resume_tokens()
8414                    {
8415                        spent = 0;
8416                    }
8417                    // TS parity: a disabled transport or a schema floor ends
8418                    // the loop. The latest round still merges first.
8419                    let stop = !self.transport_enabled() || report.schema_floor.is_some();
8420                    aggregate.merge(&report);
8421                    if stop {
8422                        return SyncOutcome::Ok(aggregate);
8423                    }
8424                    // §4.5: pull again whenever the response contained
8425                    // commits or segments; resets re-bootstrap; a pending
8426                    // resume token continues paging (§4.7); deferred outbox
8427                    // commits and a raised sync-needed signal push on the next
8428                    // round.
8429                    let more = !report.bootstrapping.is_empty()
8430                        || report.commits_applied > 0
8431                        || report.segment_rows_applied > 0
8432                        || !report.resets.is_empty()
8433                        || report.deferred_commits > 0
8434                        || self.sync_needed;
8435                    if !more {
8436                        return SyncOutcome::Ok(aggregate);
8437                    }
8438                }
8439            }
8440        }
8441        // §7.8: partial success. The aggregate report is retained and the run
8442        // is explicitly not idle.
8443        SyncOutcome::BudgetExhausted(aggregate)
8444    }
8445
8446    fn process_response(
8447        &mut self,
8448        transport: &mut dyn Transport,
8449        response: Message,
8450        meta: &RequestMeta,
8451    ) -> SyncOutcome {
8452        #[cfg(feature = "bench-internals")]
8453        let _phase = self.benchmark_phases.start(Phase::ResponseApply);
8454        let mut report = SyncReport {
8455            pushed: meta.pushed_ids.len() as u32,
8456            ..SyncReport::default()
8457        };
8458        let mut rejection_details_by_commit: HashMap<String, BTreeMap<i32, RejectionDetails>> =
8459            HashMap::new();
8460        let pushed_ids = meta
8461            .pushed_ids
8462            .iter()
8463            .map(String::as_str)
8464            .collect::<HashSet<_>>();
8465        let mut last_final_push_result_id: Option<String> = None;
8466        for frame in &response.frames {
8467            match frame {
8468                Frame::PushResultDetails {
8469                    client_commit_id,
8470                    entries,
8471                } => {
8472                    let details = rejection_details_by_commit
8473                        .entry(client_commit_id.clone())
8474                        .or_default();
8475                    for entry in entries {
8476                        let parsed = match RejectionDetails::parse(&entry.details.0) {
8477                            Ok(value) => value,
8478                            Err(message) => {
8479                                return SyncOutcome::Failed {
8480                                    details: None,
8481                                    error_code: "sync.invalid_request".to_owned(),
8482                                    message,
8483                                    recommended_action: None,
8484                                };
8485                            }
8486                        };
8487                        details.insert(entry.op_index, parsed);
8488                    }
8489                }
8490                Frame::PushResult {
8491                    client_commit_id,
8492                    status,
8493                    results,
8494                    ..
8495                } if pushed_ids.contains(client_commit_id.as_str())
8496                    && Self::push_result_is_final(*status, results) =>
8497                {
8498                    last_final_push_result_id = Some(client_commit_id.clone());
8499                }
8500                _ => {}
8501            }
8502        }
8503        let mut frames = response.frames.into_iter().peekable();
8504        match frames.next() {
8505            Some(Frame::RespHeader {
8506                required_schema_version,
8507                latest_schema_version,
8508                log_epoch,
8509                reset_required,
8510            }) => {
8511                if let Some(required) = required_schema_version {
8512                    // §1.6 schema-floor response: nothing else is processed;
8513                    // stop syncing and surface the upgrade requirement.
8514                    let floor = SchemaFloor {
8515                        required_schema_version: Some(required),
8516                        latest_schema_version,
8517                    };
8518                    if let Err(message) = self.set_schema_floor(Some(floor.clone())) {
8519                        return SyncOutcome::Failed {
8520                            details: None,
8521                            error_code: "storage.failed".to_owned(),
8522                            message,
8523                            recommended_action: None,
8524                        };
8525                    }
8526                    report.schema_floor = Some(floor);
8527                    return SyncOutcome::Ok(report);
8528                }
8529                if reset_required {
8530                    if frames.next().is_some() {
8531                        return SyncOutcome::Failed {
8532                            details: None,
8533                            error_code: "client.invalid_host_response".to_owned(),
8534                            message: "log-epoch reset response contains body frames".to_owned(),
8535                            recommended_action: None,
8536                        };
8537                    }
8538                    match self.run_log_epoch_reset(&log_epoch) {
8539                        Ok(resets) => {
8540                            report.resets = resets;
8541                            return SyncOutcome::Ok(report);
8542                        }
8543                        Err(message) => {
8544                            return SyncOutcome::Failed {
8545                                details: None,
8546                                error_code: "sync.local_corrupt".to_owned(),
8547                                message,
8548                                recommended_action: None,
8549                            };
8550                        }
8551                    }
8552                }
8553                let stored_epoch = match self.get_meta(LOG_EPOCH_KEY) {
8554                    Ok(stored) => stored,
8555                    Err(message) => {
8556                        return SyncOutcome::Failed {
8557                            details: None,
8558                            error_code: "storage.failed".to_owned(),
8559                            message,
8560                            recommended_action: None,
8561                        };
8562                    }
8563                };
8564                if stored_epoch.as_deref() != Some(log_epoch.as_str()) {
8565                    return SyncOutcome::Failed {
8566                        details: None,
8567                        error_code: "client.invalid_host_response".to_owned(),
8568                        message: "server changed logEpoch without requiring a reset".to_owned(),
8569                        recommended_action: None,
8570                    };
8571                }
8572            }
8573            _ => {
8574                return SyncOutcome::Failed {
8575                    details: None,
8576                    error_code: "sync.invalid_request".to_owned(),
8577                    message: "response does not start with RESP_HEADER".to_owned(),
8578                    recommended_action: None,
8579                };
8580            }
8581        }
8582
8583        if let Err(message) = self.begin_coalesced_apply() {
8584            return SyncOutcome::Failed {
8585                details: None,
8586                error_code: "storage.failed".into(),
8587                message,
8588                recommended_action: None,
8589            };
8590        }
8591        let mut failure: Option<(String, String, Option<String>, Option<Value>)> = None;
8592        // §4.5 client application: the units whose persisted echo a live
8593        // change must match, fixed at LIVE_START after every SUB_END persisted.
8594        let mut live: Option<(LiveStatus, String, Vec<UnitRecord>)> = None;
8595        let mut live_ended = false;
8596        while let Some(frame) = frames.next() {
8597            match frame {
8598                frame @ Frame::PushResult { .. } => {
8599                    let successful = matches!(
8600                        &frame,
8601                        Frame::PushResult {
8602                            status: PushStatus::Applied | PushStatus::Cached,
8603                            ..
8604                        }
8605                    );
8606                    let mut results = vec![frame];
8607                    if successful {
8608                        while matches!(
8609                            frames.peek(),
8610                            Some(Frame::PushResult {
8611                                status: PushStatus::Applied | PushStatus::Cached,
8612                                ..
8613                            })
8614                        ) {
8615                            if let Some(next) = frames.next() {
8616                                results.push(next);
8617                            }
8618                        }
8619                    }
8620                    if self
8621                        .handle_push_results(
8622                            &results,
8623                            &pushed_ids,
8624                            &rejection_details_by_commit,
8625                            &mut report,
8626                            last_final_push_result_id.as_deref(),
8627                        )
8628                        .is_err()
8629                    {
8630                        failure = Some((
8631                            "client.outcome_persistence_failed".to_owned(),
8632                            "local commit outcome could not be persisted".to_owned(),
8633                            None,
8634                            None,
8635                        ));
8636                        break;
8637                    }
8638                }
8639                Frame::PushResultDetails { .. } => {}
8640                Frame::SubStart {
8641                    id,
8642                    status,
8643                    reason_code,
8644                    bootstrap,
8645                    fenced,
8646                } => {
8647                    // §1.4 rule 5: an ERROR closes the section; its body still
8648                    // applied, its SUB_END values are never persisted.
8649                    let mut body = Vec::new();
8650                    let mut sub_end: Option<(i64, Option<String>)> = None;
8651                    let mut error: Option<Frame> = None;
8652                    for inner in frames.by_ref() {
8653                        match inner {
8654                            Frame::SubEnd {
8655                                next_cursor,
8656                                bootstrap_state,
8657                            } => {
8658                                sub_end = Some((next_cursor, bootstrap_state.map(|r| r.0)));
8659                                break;
8660                            }
8661                            inner @ Frame::Error { .. } => {
8662                                error = Some(inner);
8663                                break;
8664                            }
8665                            Frame::Unknown { .. } => {}
8666                            other => body.push(other),
8667                        }
8668                    }
8669                    let ended = sub_end.is_some();
8670                    let section = SectionStart {
8671                        id,
8672                        status,
8673                        reason_code,
8674                        bootstrap,
8675                        fenced,
8676                    };
8677                    match self.process_section(
8678                        transport,
8679                        &section,
8680                        body,
8681                        sub_end,
8682                        meta,
8683                        &mut report,
8684                    ) {
8685                        Err(SectionError::Abort(code, message)) => {
8686                            failure = Some((code, message, None, None));
8687                            break;
8688                        }
8689                        Err(SectionError::Transport(error)) => {
8690                            failure = Some((error.code, error.message, None, error.details));
8691                            break;
8692                        }
8693                        _ => {}
8694                    }
8695                    if ended {
8696                        // The counters stay: `complete` and `failed` report them.
8697                        self.progress.update(|p| {
8698                            p.unit_id = None;
8699                            p.table = None;
8700                            p.segment_id = None;
8701                        });
8702                    }
8703                    match error {
8704                        Some(Frame::Error {
8705                            code,
8706                            message,
8707                            recommended_action,
8708                            details,
8709                            ..
8710                        }) => {
8711                            failure = Some(Self::error_frame_failure(
8712                                code,
8713                                message,
8714                                recommended_action,
8715                                details,
8716                            ));
8717                            break;
8718                        }
8719                        Some(_) => unreachable!("only an ERROR closes a section early"),
8720                        None if !ended => {
8721                            failure = Some((
8722                                "sync.invalid_request".to_owned(),
8723                                "unit section without SUB_END".to_owned(),
8724                                None,
8725                                None,
8726                            ));
8727                            break;
8728                        }
8729                        None => {}
8730                    }
8731                }
8732                Frame::LiveStart {
8733                    status,
8734                    reason_code,
8735                } => {
8736                    let receivers = match self.stored(load_units(&self.conn)) {
8737                        Ok(units) => units
8738                            .into_iter()
8739                            .filter(|unit| {
8740                                unit.status == UnitStatus::Active
8741                                    && unit.echo.is_some()
8742                                    && !unit.bootstrapping()
8743                            })
8744                            .collect(),
8745                        Err(message) => {
8746                            failure = Some(("storage.failed".to_owned(), message, None, None));
8747                            break;
8748                        }
8749                    };
8750                    live = Some((status, reason_code, receivers));
8751                }
8752                Frame::Commit {
8753                    commit_seq,
8754                    tables,
8755                    changes,
8756                    ..
8757                } => {
8758                    let Some((LiveStatus::Active, _, receivers)) = &live else {
8759                        continue;
8760                    };
8761                    let receivers = receivers.clone();
8762                    match self.apply_live_commit(
8763                        &receivers,
8764                        &tables,
8765                        changes,
8766                        commit_seq,
8767                        AppliedServerCommitSource::Pull,
8768                    ) {
8769                        Ok(applied) => report.commits_applied += u32::from(applied),
8770                        Err(error) => {
8771                            failure = Some((error.code, error.message, None, error.details));
8772                            break;
8773                        }
8774                    }
8775                }
8776                Frame::LiveEnd {
8777                    through_seq,
8778                    echo_digest,
8779                } => {
8780                    let Some((status, reason_code, _)) = live.take() else {
8781                        continue;
8782                    };
8783                    live_ended = true;
8784                    live = Some((status, String::new(), Vec::new()));
8785                    if let Err(message) = self.finish_live_section(
8786                        status,
8787                        &reason_code,
8788                        through_seq,
8789                        echo_digest,
8790                        &meta.registration_digest,
8791                        &mut report,
8792                    ) {
8793                        failure = Some(("storage.failed".to_owned(), message, None, None));
8794                        break;
8795                    }
8796                }
8797                Frame::Lease {
8798                    lease_id,
8799                    expires_at_ms,
8800                } => {
8801                    // §7.3.5: persist the opaque lease; a fresh lease clears
8802                    // any prior lease error (the outage/revocation is over).
8803                    if let Err(message) = self.set_lease_state(Some(LeaseState {
8804                        lease_id: Some(lease_id),
8805                        expires_at_ms: Some(expires_at_ms),
8806                        error_code: None,
8807                    })) {
8808                        failure = Some(("storage.failed".to_owned(), message, None, None));
8809                        break;
8810                    }
8811                }
8812                Frame::Error {
8813                    code,
8814                    message,
8815                    recommended_action,
8816                    details,
8817                    ..
8818                } => {
8819                    // §1.6: the whole request failed; already-completed
8820                    // units keep their applied data and cursors.
8821                    failure = Some(Self::error_frame_failure(
8822                        code,
8823                        message,
8824                        recommended_action,
8825                        details,
8826                    ));
8827                    break;
8828                }
8829                Frame::Unknown { .. } => {}
8830                _ => {
8831                    failure = Some((
8832                        "sync.invalid_request".to_owned(),
8833                        "unexpected frame in response".to_owned(),
8834                        None,
8835                        None,
8836                    ));
8837                    break;
8838                }
8839            }
8840        }
8841
8842        let applied = match failure {
8843            Some(failure) => Err(failure),
8844            // Local apply boundaries already reconciled their affected overlay.
8845            // Refuse round success if any remaining dirty work fails to
8846            // reconcile. §4.8 E1: the push half may have drained commits that
8847            // pinned rows of a shrunk window unit, so retry deferred evictions.
8848            None => self
8849                .rebuild_overlay_if_dirty()
8850                .and_then(|()| self.drain_pending_evictions())
8851                .map_err(|message| ("storage.failed".to_owned(), message, None, None)),
8852        };
8853        // §1.4 rule 4: frames completed before a failure stay committed; a
8854        // failed COMMIT of the coalesced transaction fails the round.
8855        if let Err(message) = self.finish_coalesced_apply() {
8856            return SyncOutcome::Failed {
8857                details: None,
8858                error_code: "storage.failed".into(),
8859                message,
8860                recommended_action: None,
8861            };
8862        }
8863        if let Err((error_code, message, recommended_action, details)) = applied {
8864            return SyncOutcome::Failed {
8865                details,
8866                error_code,
8867                message,
8868                recommended_action,
8869            };
8870        }
8871        let catching_up = match self.stored(load_units(&self.conn)) {
8872            Ok(units) => units
8873                .into_iter()
8874                .filter(|unit| unit.status == UnitStatus::Active && unit.cursor.is_some())
8875                .map(|unit| unit.id)
8876                .collect::<Vec<_>>(),
8877            Err(message) => {
8878                return SyncOutcome::Failed {
8879                    details: None,
8880                    error_code: "storage.failed".into(),
8881                    message,
8882                    recommended_action: None,
8883                }
8884            }
8885        };
8886        // §7.1: a response without LIVE_END, one answered `unregistered` or
8887        // `reset`, or a unit still catching up leaves sync-needed raised;
8888        // after `unregistered` the next round starts at once.
8889        let status = live.map(|(status, _, _)| status);
8890        if !live_ended || status != Some(LiveStatus::Active) || !catching_up.is_empty() {
8891            self.set_sync_needed(true, status == Some(LiveStatus::Unregistered));
8892        }
8893        self.ack_after_pull(transport);
8894        // §7.4.5: the reset is over once the first post-reset pull round
8895        // leaves no unit catching up: the tables are rebuilt and current.
8896        if self.upgrading && catching_up.is_empty() {
8897            self.set_upgrading(false);
8898        }
8899        report.bootstrapping = catching_up;
8900        SyncOutcome::Ok(report)
8901    }
8902
8903    /// §1.6 / §10.1: the failure an in-band `ERROR` frame reports.
8904    fn error_frame_failure(
8905        code: String,
8906        message: String,
8907        recommended_action: String,
8908        details: Option<RawJson>,
8909    ) -> (String, String, Option<String>, Option<Value>) {
8910        let recommended_action = (!recommended_action.is_empty()).then_some(recommended_action);
8911        let details = details
8912            .and_then(|details| serde_json::from_str::<Value>(&details.0).ok())
8913            .filter(Value::is_object);
8914        (code, message, recommended_action, details)
8915    }
8916
8917    /// §4.5 at `LIVE_END` of a pull: join rule J for an `active` section, the
8918    /// §4.6 replica reset for `reset`, nothing for `unregistered`; then the
8919    /// replica cursor, the echo digest and the digest of the list the server
8920    /// now stores.
8921    fn finish_live_section(
8922        &mut self,
8923        status: LiveStatus,
8924        reason_code: &str,
8925        through_seq: i64,
8926        echo_digest: Option<[u8; 32]>,
8927        registration_digest: &str,
8928        report: &mut SyncReport,
8929    ) -> Result<(), String> {
8930        let rollback = self.capture();
8931        let resets = report.resets.len();
8932        self.begin_observation("syncular_live_end")?;
8933        let finished = (|| {
8934            let mut batch = ChangeAccumulator::default();
8935            let before = self.stored(replica_cursor(&self.conn))?;
8936            let mut cursor = if status == LiveStatus::Unregistered {
8937                before
8938            } else {
8939                -1
8940            };
8941            let joined = before.max(through_seq);
8942            for unit in self.stored(load_units(&self.conn))? {
8943                if unit.status != UnitStatus::Active || unit.bootstrapping() {
8944                    continue;
8945                }
8946                match status {
8947                    // §4.6 replica reset: every live unit bootstraps again;
8948                    // post-bootstrap units keep their cursors.
8949                    LiveStatus::Reset if unit.cursor.is_none() => {
8950                        for (base_key, value) in self.stored(window_refs(&self.conn, &unit.id))? {
8951                            batch.window(&base_key, &value);
8952                        }
8953                        report.resets.push(unit.id.clone());
8954                        self.stored(save_unit(
8955                            &self.conn,
8956                            &UnitRecord {
8957                                reason_code: Some(reason_code.to_owned()),
8958                                cursor: Some(-1),
8959                                seeded: Vec::new(),
8960                                ..unit
8961                            },
8962                        ))?;
8963                    }
8964                    // §4.5 join rule J.
8965                    LiveStatus::Active => {
8966                        let next = unit.cursor.unwrap_or(joined).max(through_seq);
8967                        if next >= joined {
8968                            cursor = joined;
8969                        }
8970                        if unit.cursor.is_some() {
8971                            self.stored(save_unit(
8972                                &self.conn,
8973                                &UnitRecord {
8974                                    cursor: (next < joined).then_some(next),
8975                                    ..unit
8976                                },
8977                            ))?;
8978                        }
8979                    }
8980                    _ => {}
8981                }
8982            }
8983            self.set_meta(REPLICA_CURSOR_KEY, &cursor.to_string())?;
8984            self.set_meta(
8985                ECHO_DIGEST_KEY,
8986                &echo_digest
8987                    .map(|digest| bytes_to_hex(&digest))
8988                    .unwrap_or_default(),
8989            )?;
8990            if status == LiveStatus::Unregistered {
8991                self.delete_meta(REGISTERED_DIGEST_KEY)?;
8992            } else {
8993                self.set_meta(REGISTERED_DIGEST_KEY, registration_digest)?;
8994            }
8995            self.finish_observation("syncular_live_end", batch)
8996        })();
8997        if finished.is_err() {
8998            self.rollback_observation("syncular_live_end");
8999            self.restore(rollback);
9000            report.resets.truncate(resets);
9001        }
9002        finished
9003    }
9004
9005    // -- push results (§6.3, §7.2) ------------------------------------------------
9006
9007    fn handle_push_results(
9008        &mut self,
9009        frames: &[Frame],
9010        pushed_ids: &HashSet<&str>,
9011        rejection_details_by_commit: &HashMap<String, BTreeMap<i32, RejectionDetails>>,
9012        report: &mut SyncReport,
9013        last_final_push_result_id: Option<&str>,
9014    ) -> Result<(), String> {
9015        self.begin_observation("syncular_push_results")?;
9016        let rollback = self.capture();
9017        let conflict_count = self.conflicts.len();
9018        let applied_count = report.applied.len();
9019        let rejected_count = report.rejected.len();
9020        let retryable_count = report.retryable.len();
9021        let mut drained = Vec::new();
9022        let mut batch = ChangeAccumulator::default();
9023        let persisted = (|| {
9024            for frame in frames {
9025                let Frame::PushResult {
9026                    client_commit_id,
9027                    status,
9028                    results,
9029                    commit_seq,
9030                } = frame
9031                else {
9032                    return Err("unexpected acknowledgement frame".to_owned());
9033                };
9034                if !pushed_ids.contains(client_commit_id.as_str()) {
9035                    continue;
9036                }
9037                let Some(index) = self
9038                    .outbox
9039                    .iter()
9040                    .position(|commit| commit.client_commit_id == *client_commit_id)
9041                else {
9042                    continue;
9043                };
9044                let status = *status;
9045                let rejection_details = rejection_details_by_commit.get(client_commit_id);
9046                let operations = self.outbox[index].ops.clone();
9047                match status {
9048                    PushStatus::Applied | PushStatus::Cached => {
9049                        // §7.2: a lost ack replays as `cached` — proceed as if the
9050                        // ack had arrived.
9051                        let sequence =
9052                            commit_seq.ok_or_else(|| "ACK lacks a commit sequence".to_owned())?;
9053                        let affected: HashSet<_> = operations
9054                            .iter()
9055                            .map(|op| (op.table.as_str(), op.row_id.as_str()))
9056                            .collect();
9057                        let prior = self.acknowledged_operations(Some(&affected), None)?;
9058                        for (op_index, operation) in operations.iter().enumerate() {
9059                            let delivered: bool = self.conn.query_row("SELECT EXISTS(SELECT 1 FROM _syncular_row_deliveries WHERE tbl=?1 AND id=?2 AND commit_seq>=?3)", rusqlite::params![operation.table, operation.row_id, sequence], |row| row.get(0))
9060                                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9061                            if delivered {
9062                                continue;
9063                            }
9064                            let table = self
9065                                .schema
9066                                .table(&operation.table)
9067                                .ok_or_else(|| "sync.unknown_table".to_owned())?;
9068                            let mut intended: Map<String, Value> = self
9069                                .stored_row(table, &operation.row_id, true)?
9070                                .map(|(row, _)| {
9071                                    table
9072                                        .columns
9073                                        .iter()
9074                                        .zip(&row)
9075                                        .map(|(column, value)| {
9076                                            (column.name.clone(), column_value_to_json(value))
9077                                        })
9078                                        .collect()
9079                                })
9080                                .unwrap_or_default();
9081                            let mut scope_image = intended.clone();
9082                            for op in prior
9083                                .iter()
9084                                .chain(
9085                                    self.failed_commits
9086                                        .iter()
9087                                        .flat_map(|(commit, _)| &commit.ops),
9088                                )
9089                                .chain(
9090                                    self.outbox
9091                                        .iter()
9092                                        .take(index)
9093                                        .flat_map(|commit| &commit.ops),
9094                                )
9095                                .chain(operations.iter().take(op_index + 1))
9096                            {
9097                                if op.table != operation.table || op.row_id != operation.row_id {
9098                                    continue;
9099                                }
9100                                if let Some(values) = &op.values {
9101                                    intended.extend(values.clone());
9102                                } else {
9103                                    scope_image = intended.clone();
9104                                    intended.clear();
9105                                }
9106                            }
9107                            self.conn.execute("INSERT INTO _syncular_acked_rows(commit_id,idx,tbl,id,commit_seq,op_json,intent_json) VALUES (?1,?2,?3,?4,?5,?6,?7)",
9108                                rusqlite::params![client_commit_id, op_index as i64, operation.table, operation.row_id, sequence,
9109                                    serde_json::to_string(&CommitOperation::from(operation)).map_err(|_| "sync.local_corrupt".to_owned())?,
9110                                    serde_json::to_string(if operation.upsert { &intended } else { &scope_image }).map_err(|_| "sync.local_corrupt".to_owned())?])
9111                                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9112                        }
9113                        let journal_results = results
9114                            .iter()
9115                            .map(|result| {
9116                                let op_index = match result {
9117                                    OpResult::Applied { op_index }
9118                                    | OpResult::Conflict { op_index, .. }
9119                                    | OpResult::Error { op_index, .. } => *op_index,
9120                                };
9121                                CommitOperationOutcome::Applied { op_index }
9122                            })
9123                            .collect::<Vec<_>>();
9124                        let outcome_status = if status == PushStatus::Applied {
9125                            CommitOutcomeStatus::Applied
9126                        } else {
9127                            CommitOutcomeStatus::Cached
9128                        };
9129                        let persisted = self
9130                            .persist_commit_outcome(
9131                                client_commit_id,
9132                                outcome_status,
9133                                &journal_results,
9134                                None,
9135                            )
9136                            .and_then(|()| self.delete_outbox_persisted(client_commit_id));
9137                        persisted?;
9138                        self.sync_needed = true;
9139                        batch.status = true;
9140                        batch.outcomes = true;
9141                    }
9142                    PushStatus::Rejected => {
9143                        if results.iter().any(|result| {
9144                            matches!(
9145                                result,
9146                                OpResult::Error {
9147                                    code,
9148                                    retryable: true,
9149                                    ..
9150                                } if code == "sync.idempotency_cache_miss"
9151                            )
9152                        }) {
9153                            // §6.3/§7.2: a serving failure, not an outcome — keep the
9154                            // exact commit queued for an identical retry.
9155                            report.retryable.push(client_commit_id.to_owned());
9156                            continue;
9157                        }
9158
9159                        let mut journal_results = Vec::with_capacity(results.len());
9160                        let mut conflicts = Vec::new();
9161                        let mut rejections = Vec::new();
9162                        for result in results {
9163                            match result {
9164                                OpResult::Applied { op_index } => {
9165                                    journal_results.push(CommitOperationOutcome::Applied {
9166                                        op_index: *op_index,
9167                                    });
9168                                }
9169                                OpResult::Conflict {
9170                                    op_index,
9171                                    code,
9172                                    message,
9173                                    server_version,
9174                                    server_row,
9175                                    conflict_columns: conflict_columns_bitmap,
9176                                } => {
9177                                    let operation = operations
9178                                        .get(*op_index as usize)
9179                                        .map(CommitOperation::from);
9180                                    let (table, row_id) = operation
9181                                        .as_ref()
9182                                        .map(|op| (op.table.clone(), op.row_id.clone()))
9183                                        .unwrap_or_default();
9184                                    let server_row_json = self
9185                                        .schema
9186                                        .table(&table)
9187                                        .and_then(|t| {
9188                                            decode_row_bytes(t, server_row, &self.encryption)
9189                                                .ok()
9190                                                .map(|(row, _)| (t, row))
9191                                        })
9192                                        .map(|(t, row)| {
9193                                            let mut map = Map::new();
9194                                            for (i, column) in t.columns.iter().enumerate() {
9195                                                map.insert(
9196                                                    column.name.clone(),
9197                                                    column_value_to_json(
9198                                                        row.get(i).unwrap_or(&None),
9199                                                    ),
9200                                                );
9201                                            }
9202                                            map
9203                                        })
9204                                        .unwrap_or_default();
9205                                    // §6.3: decode the presence-layout bitmap
9206                                    // into the marked column names.
9207                                    let conflict_columns = self
9208                                        .schema
9209                                        .table(&table)
9210                                        .map(|t| {
9211                                            t.columns
9212                                                .iter()
9213                                                .enumerate()
9214                                                .filter(|(index, _)| {
9215                                                    conflict_columns_bitmap
9216                                                        .get(index / 8)
9217                                                        .is_some_and(|byte| {
9218                                                            byte & (1 << (index % 8)) != 0
9219                                                        })
9220                                                })
9221                                                .map(|(_, column)| column.name.clone())
9222                                                .collect::<Vec<_>>()
9223                                        })
9224                                        .unwrap_or_default();
9225                                    let conflict = ConflictRecord {
9226                                        client_commit_id: client_commit_id.to_owned(),
9227                                        op_index: *op_index,
9228                                        table,
9229                                        row_id,
9230                                        code: code.clone(),
9231                                        message: message.clone(),
9232                                        server_version: *server_version,
9233                                        server_row: server_row_json,
9234                                        conflict_columns,
9235                                        operation,
9236                                    };
9237                                    journal_results.push(CommitOperationOutcome::Conflict {
9238                                        conflict: conflict.clone(),
9239                                    });
9240                                    conflicts.push(conflict);
9241                                }
9242                                OpResult::Error {
9243                                    op_index,
9244                                    code,
9245                                    message,
9246                                    retryable,
9247                                } => {
9248                                    let rejection = RejectionRecord {
9249                                        client_commit_id: client_commit_id.to_owned(),
9250                                        op_index: *op_index,
9251                                        code: code.clone(),
9252                                        message: message.clone(),
9253                                        retryable: *retryable,
9254                                        details: rejection_details
9255                                            .and_then(|details| details.get(op_index))
9256                                            .cloned(),
9257                                        operation: operations
9258                                            .get(*op_index as usize)
9259                                            .map(CommitOperation::from),
9260                                    };
9261                                    journal_results.push(CommitOperationOutcome::Error {
9262                                        rejection: rejection.clone(),
9263                                    });
9264                                    rejections.push(rejection);
9265                                }
9266                            }
9267                        }
9268                        if self.retain_failed_commits {
9269                            self.retain_failed_commit(client_commit_id, &operations)?;
9270                        }
9271                        let outcome_status = if conflicts.is_empty() {
9272                            CommitOutcomeStatus::Rejected
9273                        } else {
9274                            CommitOutcomeStatus::Conflict
9275                        };
9276                        let persisted = self
9277                            .persist_commit_outcome(
9278                                client_commit_id,
9279                                outcome_status,
9280                                &journal_results,
9281                                Some(&operations),
9282                            )
9283                            .and_then(|()| self.delete_outbox_persisted(client_commit_id));
9284                        persisted?;
9285                        batch.conflicts = !conflicts.is_empty();
9286                        batch.rejections = !rejections.is_empty();
9287                        batch.status = true;
9288                        batch.outcomes = true;
9289                        self.conflicts.extend(conflicts);
9290                        self.rejections.extend(rejections);
9291                    }
9292                }
9293                drained.push(self.outbox.remove(index));
9294                if batch.status {
9295                    for operation in &operations {
9296                        if !self.record_row_scopes(
9297                            &mut batch,
9298                            &operation.table,
9299                            &operation.row_id,
9300                            false,
9301                        ) {
9302                            batch.table(&operation.table);
9303                        }
9304                    }
9305                }
9306                match status {
9307                    PushStatus::Applied | PushStatus::Cached => {
9308                        report.applied.push(client_commit_id.to_owned())
9309                    }
9310                    PushStatus::Rejected => report.rejected.push(client_commit_id.to_owned()),
9311                }
9312            }
9313            if batch.outcomes && frames.iter().any(|frame| matches!(frame, Frame::PushResult { client_commit_id, .. } if Some(client_commit_id.as_str()) == last_final_push_result_id)) {
9314                self.prune_commit_outcomes()?;
9315            }
9316            self.reconcile_rows(&drained)?;
9317            self.rebuild_overlay_if_dirty()?;
9318            self.finish_observation("syncular_push_results", batch)
9319        })();
9320        if let Err(error) = persisted {
9321            self.rollback_observation("syncular_push_results");
9322            self.restore(rollback);
9323            report.applied.truncate(applied_count);
9324            report.rejected.truncate(rejected_count);
9325            report.retryable.truncate(retryable_count);
9326            return Err(error);
9327        }
9328        if !rollback.sync_needed && self.sync_needed {
9329            self.sync_intent_queue.push_back(SyncIntent::Interactive);
9330        }
9331        report.conflicts += (self.conflicts.len() - conflict_count) as u32;
9332        Ok(())
9333    }
9334
9335    fn push_result_is_final(status: PushStatus, results: &[OpResult]) -> bool {
9336        status != PushStatus::Rejected
9337            || !results.iter().any(|result| {
9338                matches!(
9339                    result,
9340                    OpResult::Error {
9341                        code,
9342                        retryable: true,
9343                        ..
9344                    } if code == "sync.idempotency_cache_miss"
9345                )
9346            })
9347    }
9348
9349    // -- unit sections and the live section ------------------------------------------
9350
9351    /// One unit section (§4.4): apply its segments, then persist its
9352    /// `SUB_END` (§4.6 reset, §3.3 revocation and purge, the echo, cursor and
9353    /// resume token). A section of a unit the app released while the round was
9354    /// in flight is skipped; a fail-closed first page fails the unit and skips
9355    /// the rest of its section (§5.6).
9356    fn process_section(
9357        &mut self,
9358        transport: &mut dyn Transport,
9359        start: &SectionStart,
9360        body: Vec<Frame>,
9361        sub_end: Option<(i64, Option<String>)>,
9362        meta: &RequestMeta,
9363        report: &mut SyncReport,
9364    ) -> Result<(), SectionError> {
9365        let storage = |message| SectionError::Abort("storage.failed".to_owned(), message);
9366        let unit = self
9367            .stored(get_unit(&self.conn, &start.id))
9368            .map_err(storage)?
9369            .filter(|unit| unit.status == UnitStatus::Active);
9370        // §7.7: a section boundary ends the previous payload.
9371        self.progress.update(|p| {
9372            p.phase = ProgressPhase::Import;
9373            p.unit_id = unit.as_ref().map(|unit| unit.id.clone());
9374            p.table = None;
9375            p.segment_id = None;
9376            p.bytes_received = 0;
9377            p.bytes_total = None;
9378            p.rows_processed = 0;
9379            p.rows_total = None;
9380            p.import_write_ms = None;
9381            p.import_commit_ms = None;
9382        });
9383        let Some(unit) = unit else {
9384            return Ok(());
9385        };
9386        let echo: Vec<EchoRecord> = start
9387            .fenced
9388            .iter()
9389            .map(|record| EchoRecord {
9390                table: record.table.clone(),
9391                status: record.status.name().to_owned(),
9392                scopes: record.scopes.iter().cloned().collect(),
9393            })
9394            .collect();
9395        // §5.6: rows earlier attempts at this position applied.
9396        let applied_record = self
9397            .stored(applied_rows(&self.conn, &unit.id))
9398            .map_err(storage)?;
9399        // §5.6: a resumed bootstrap keeps the tables it already seeded.
9400        let mut seeded: BTreeSet<String> = if unit.bootstrap_state.is_none() {
9401            BTreeSet::new()
9402        } else {
9403            unit.seeded.iter().cloned().collect()
9404        };
9405        let applied = (|| {
9406            for frame in body {
9407                match frame {
9408                    Frame::SegmentInline { payload } => {
9409                        let segment = decode_rows_segment(&payload).map_err(|e| {
9410                            SectionError::Abort(e.code.as_str().to_owned(), e.detail)
9411                        })?;
9412                        self.progress.update(|p| {
9413                            p.phase = ProgressPhase::Import;
9414                            p.unit_id = Some(unit.id.clone());
9415                            p.table = Some(segment.table.clone());
9416                            p.segment_id = None;
9417                            p.bytes_received = payload.len() as u64;
9418                            p.bytes_total = Some(payload.len() as u64);
9419                            p.rows_processed = 0;
9420                            p.rows_total =
9421                                Some(segment.blocks.iter().map(|block| block.len() as u64).sum());
9422                            p.import_write_ms = None;
9423                            p.import_commit_ms = None;
9424                        });
9425                        let (pair, clear, effective) =
9426                            self.segment_pair(&unit, &echo, &seeded, &segment.table)?;
9427                        report.segment_rows_applied +=
9428                            self.apply_segment(&pair, &segment, clear, &effective, &unit.id)?;
9429                        seeded.insert(segment.table);
9430                    }
9431                    Frame::SegmentRef {
9432                        segment_id,
9433                        byte_length,
9434                        media_type,
9435                        table,
9436                        row_count,
9437                        as_of_commit_seq,
9438                        scope_digest,
9439                        row_cursor,
9440                        next_row_cursor,
9441                        url,
9442                        url_expires_at_ms,
9443                        ..
9444                    } => {
9445                        // §4.2: reject a descriptor whose mediaType was not
9446                        // advertised — never skip or guess.
9447                        let advertised = match media_type {
9448                            MediaType::Rows => {
9449                                meta.accept & ACCEPT_EXTERNAL_ROWS != 0
9450                                    || meta.accept & ACCEPT_INLINE_ROWS != 0
9451                            }
9452                            MediaType::Sqlite => meta.accept & ACCEPT_SQLITE != 0,
9453                        };
9454                        if !advertised {
9455                            return Err(SectionError::Abort(
9456                                "sync.invalid_request".to_owned(),
9457                                format!(
9458                                    "SEGMENT_REF mediaType {} was not advertised in accept (§4.2)",
9459                                    media_type.name()
9460                                ),
9461                            ));
9462                        }
9463                        // §5.6 applied-segment rule: a partly applied image
9464                        // already ran its table's first-page clear.
9465                        let resumed = applied_record.get(&segment_id).copied();
9466                        if resumed.is_some() {
9467                            seeded.insert(table.clone());
9468                        }
9469                        if resumed == Some(row_count) {
9470                            continue;
9471                        }
9472                        if meta
9473                            .deferred
9474                            .contains(&(unit.id.clone(), segment_id.clone()))
9475                        {
9476                            return Err(SectionError::Deferred);
9477                        }
9478                        self.progress.update(|p| {
9479                            p.phase = ProgressPhase::Download;
9480                            p.unit_id = Some(unit.id.clone());
9481                            p.table = Some(table.clone());
9482                            p.segment_id = Some(segment_id.clone());
9483                            p.bytes_received = 0;
9484                            p.bytes_total = u64::try_from(byte_length).ok();
9485                            p.rows_processed = 0;
9486                            p.rows_total = u64::try_from(row_count).ok();
9487                            p.import_write_ms = None;
9488                            p.import_commit_ms = None;
9489                        });
9490                        let observer = self.progress.clone();
9491                        let mut on_progress = |bytes| observer.update(|p| p.bytes_received = bytes);
9492                        let bytes = if let Some(url) = url {
9493                            // §5.4: a url-carrying descriptor MUST be fetched
9494                            // from that URL; failure invalidates the whole
9495                            // descriptor (no fall-through to §5.5 — re-pull
9496                            // recovers, §1.4 rule 5).
9497                            if meta.accept & ACCEPT_SIGNED_URLS == 0 {
9498                                return Err(SectionError::Abort(
9499                                    "sync.invalid_request".to_owned(),
9500                                    "SEGMENT_REF carries a url but accept bit 3 was not advertised (§5.4)"
9501                                        .to_owned(),
9502                                ));
9503                            }
9504                            // §5.4: MUST NOT start a fetch at/past expiry.
9505                            if url_expires_at_ms.is_some_and(|exp| {
9506                                exp <= transport
9507                                    .segment_fetch_started_at(&url)
9508                                    .unwrap_or_else(|| self.clock_now_ms())
9509                            }) {
9510                                return Err(SectionError::Abort(
9511                                    "sync.segment_expired".to_owned(),
9512                                    format!(
9513                                        "signed URL for segment {segment_id} expired before fetch — re-pull mints fresh descriptors (§5.4)"
9514                                    ),
9515                                ));
9516                            }
9517                            transport
9518                                .fetch_url(
9519                                    &url,
9520                                    u64::try_from(byte_length).unwrap_or(0),
9521                                    &mut on_progress,
9522                                )
9523                                .map_err(|e| self.segment_fetch_failed(e))?
9524                        } else {
9525                            // §5.5: the owning unit's scope map, each value a one-element list.
9526                            let requested_scopes_json =
9527                                canonical_scope_json(&unit_scope_lists(&unit.scopes));
9528                            transport
9529                                .download_segment(
9530                                    &SegmentRequest {
9531                                        segment_id: segment_id.clone(),
9532                                        table: table.clone(),
9533                                        requested_scopes_json,
9534                                        byte_length: u64::try_from(byte_length).unwrap_or(0),
9535                                    },
9536                                    &mut on_progress,
9537                                )
9538                                .map_err(|e| {
9539                                    // §5.5: access changed since the response;
9540                                    // the next round's SUB_START settles the unit.
9541                                    if e.code == "sync.forbidden" {
9542                                        SectionError::Refused
9543                                    } else {
9544                                        self.segment_fetch_failed(e)
9545                                    }
9546                                })?
9547                        };
9548                        // §5.4: the descriptor's byteLength caps the body before
9549                        // §5.1 verifies the content address; the client discards
9550                        // the segment and re-pulls on either failure.
9551                        if bytes.len() as i64 > byte_length {
9552                            self.schedule_background_retry();
9553                            return Err(SectionError::Abort(
9554                                "sync.invalid_request".to_owned(),
9555                                "segment exceeds its byteLength".to_owned(),
9556                            ));
9557                        }
9558                        on_progress(bytes.len() as u64);
9559                        let digest = Sha256::digest(&bytes);
9560                        let expected = segment_id
9561                            .strip_prefix("sha256:")
9562                            .unwrap_or(segment_id.as_str());
9563                        if bytes_to_hex(&digest) != expected {
9564                            // §5.1: the client discards the segment and re-pulls.
9565                            self.schedule_background_retry();
9566                            return Err(SectionError::Abort(
9567                                "sync.invalid_request".to_owned(),
9568                                "segment bytes do not match the content address (§5.1)".to_owned(),
9569                            ));
9570                        }
9571                        self.progress.update(|p| p.phase = ProgressPhase::Import);
9572                        if media_type == MediaType::Sqlite {
9573                            // §5.3: a range image keeps its position in the resume
9574                            // token; a descriptor row cursor is invalid.
9575                            if row_cursor.is_some() || next_row_cursor.is_some() {
9576                                return Err(SectionError::Abort(
9577                                    "sync.invalid_request".to_owned(),
9578                                    "sqlite segments are whole-table: rowCursor/nextRowCursor must be absent (§5.3)"
9579                                        .to_owned(),
9580                                ));
9581                            }
9582                            let (pair, clear, effective) =
9583                                self.segment_pair(&unit, &echo, &seeded, &table)?;
9584                            report.segment_rows_applied += self.apply_sqlite_segment(
9585                                &pair,
9586                                &bytes,
9587                                clear,
9588                                &effective,
9589                                &unit.id,
9590                                &segment_id,
9591                                resumed.unwrap_or(0),
9592                                row_count,
9593                                as_of_commit_seq,
9594                                &scope_digest,
9595                            )?;
9596                            seeded.insert(table);
9597                        } else {
9598                            let segment = decode_rows_segment(&bytes).map_err(|e| {
9599                                SectionError::Abort(e.code.as_str().to_owned(), e.detail)
9600                            })?;
9601                            let (pair, clear, effective) =
9602                                self.segment_pair(&unit, &echo, &seeded, &segment.table)?;
9603                            report.segment_rows_applied +=
9604                                self.apply_segment(&pair, &segment, clear, &effective, &unit.id)?;
9605                            seeded.insert(segment.table);
9606                            // An image records its rows with each chunk.
9607                            self.stored(record_applied_rows(
9608                                &self.conn,
9609                                &unit.id,
9610                                &segment_id,
9611                                row_count,
9612                            ))
9613                            .map_err(storage)?;
9614                        }
9615                    }
9616                    _ => {
9617                        return Err(SectionError::Abort(
9618                            "sync.invalid_request".to_owned(),
9619                            "unexpected frame inside a unit section".to_owned(),
9620                        ));
9621                    }
9622                }
9623            }
9624            Ok(())
9625        })();
9626        match applied {
9627            Ok(()) => {}
9628            // The unit keeps catching up with its cursor unpersisted.
9629            Err(SectionError::Refused | SectionError::Deferred) => return Ok(()),
9630            Err(SectionError::FailClosed) => {
9631                // §5.6: the failure is unit-local; the rest of the response
9632                // still applies and the failed unit leaves the list.
9633                self.begin_observation("syncular_section_failure")
9634                    .map_err(storage)?;
9635                let failed = (|| {
9636                    let mut batch = ChangeAccumulator::default();
9637                    for (base_key, value) in self.stored(window_refs(&self.conn, &unit.id))? {
9638                        batch.window(&base_key, &value);
9639                    }
9640                    batch.status = true;
9641                    self.stored(save_unit(
9642                        &self.conn,
9643                        &UnitRecord {
9644                            status: UnitStatus::Failed,
9645                            reason_code: Some("sync.scope_revoked".to_owned()),
9646                            ..unit.clone()
9647                        },
9648                    ))?;
9649                    self.finish_observation("syncular_section_failure", batch)
9650                })();
9651                if let Err(message) = failed {
9652                    self.rollback_observation("syncular_section_failure");
9653                    return Err(storage(message));
9654                }
9655                report.failed.push(unit.id);
9656                return Ok(());
9657            }
9658            Err(error) => return Err(error),
9659        }
9660        let Some((next_cursor, bootstrap_state)) = sub_end else {
9661            return Ok(());
9662        };
9663        let rollback = self.capture();
9664        let counts = (
9665            report.resets.len(),
9666            report.revoked.len(),
9667            report.failed.len(),
9668        );
9669        self.begin_observation("syncular_sub_end")
9670            .map_err(storage)?;
9671        let finished = self
9672            .finish_section(start, echo, next_cursor, bootstrap_state, seeded, report)
9673            .and_then(|batch| {
9674                self.rebuild_overlay_if_dirty()?;
9675                self.finish_observation("syncular_sub_end", batch)
9676            });
9677        if let Err(message) = finished {
9678            self.rollback_observation("syncular_sub_end");
9679            self.restore(rollback);
9680            report.resets.truncate(counts.0);
9681            report.revoked.truncate(counts.1);
9682            report.failed.truncate(counts.2);
9683            return Err(storage(message));
9684        }
9685        Ok(())
9686    }
9687
9688    /// §5.6: the table, first-page flag and effective scopes of one segment
9689    /// of `unit`'s section.
9690    #[allow(clippy::type_complexity)]
9691    fn segment_pair(
9692        &self,
9693        unit: &UnitRecord,
9694        echo: &[EchoRecord],
9695        seeded: &BTreeSet<String>,
9696        name: &str,
9697    ) -> Result<(TableSchema, bool, Vec<(String, Vec<String>)>), SectionError> {
9698        let table = self.schema.table(name).cloned().ok_or_else(|| {
9699            SectionError::Abort(
9700                "sync.schema_mismatch".to_owned(),
9701                format!("segment table {name:?} is not in the client schema"),
9702            )
9703        })?;
9704        let effective = covers(&table, &unit.scopes)
9705            .then(|| pair_scopes(&unit.scopes, name, Some(echo)))
9706            .flatten()
9707            .ok_or_else(|| {
9708                SectionError::Abort(
9709                    "sync.invalid_request".to_owned(),
9710                    "a segment names no active pair of its unit".to_owned(),
9711                )
9712            })?;
9713        Ok((table, !seeded.contains(name), effective))
9714    }
9715
9716    /// §4.4 `SUB_END` of an active, revoked or reset section, inside the
9717    /// caller's observation: a reset discards cursor, resume token and
9718    /// first-page record and keeps the echo; a revoked pair purges by its last
9719    /// echo (§3.3); otherwise the echo replaces the persisted one, a revoked
9720    /// fenced record that turns active again re-bootstraps the unit, and a
9721    /// bootstrap section persists its pin, resume token and seeded tables.
9722    fn finish_section(
9723        &mut self,
9724        start: &SectionStart,
9725        echo: Vec<EchoRecord>,
9726        next_cursor: i64,
9727        bootstrap_state: Option<String>,
9728        seeded: BTreeSet<String>,
9729        report: &mut SyncReport,
9730    ) -> Result<ChangeAccumulator, String> {
9731        #[cfg(feature = "bench-internals")]
9732        let _phase = self.benchmark_phases.start(Phase::CursorPersist);
9733        let mut batch = ChangeAccumulator::default();
9734        // §4.4: SUB_END writes only into a unit that is still registered.
9735        let Some(unit) = self
9736            .stored(get_unit(&self.conn, &start.id))?
9737            .filter(|unit| unit.status == UnitStatus::Active)
9738        else {
9739            return Ok(batch);
9740        };
9741        for (base_key, value) in self.stored(window_refs(&self.conn, &unit.id))? {
9742            batch.window(&base_key, &value);
9743        }
9744        if start.status == SubStatus::Reset {
9745            // §4.6: staleness, never a purge; the rows stay until the
9746            // bootstrap replaces them.
9747            report.resets.push(unit.id.clone());
9748            self.stored(save_unit(
9749                &self.conn,
9750                &UnitRecord {
9751                    reason_code: Some(start.reason_code.clone()),
9752                    cursor: Some(-1),
9753                    bootstrap_state: None,
9754                    seeded: Vec::new(),
9755                    ..unit
9756                },
9757            ))?;
9758            return Ok(batch);
9759        }
9760        let revoked = start.status == SubStatus::Revoked;
9761        let mut units = None;
9762        let mut failed = false;
9763        let mut rearmed = false;
9764        for table in self.schema.tables.clone() {
9765            if !covers(&table, &unit.scopes) {
9766                continue;
9767            }
9768            let last = unit
9769                .echo
9770                .as_deref()
9771                .and_then(|last| pair_scopes(&unit.scopes, &table.name, Some(last)));
9772            let next = (!revoked)
9773                .then(|| pair_scopes(&unit.scopes, &table.name, Some(&echo)))
9774                .flatten();
9775            // §3.3: a revoked fenced record that turns active again re-bootstraps.
9776            rearmed |= unit.echo.is_some() && last.is_none() && next.is_some();
9777            // §3.3: purge a pair the response revokes, keyed on the echo the
9778            // client persisted while the pair was active.
9779            if let (Some(last), None) = (&last, &next) {
9780                let units = match units {
9781                    Some(ref units) => units,
9782                    None => units.insert(self.stored(load_units(&self.conn))?),
9783                };
9784                let owners = unit_owners(units, &table, &unit.id);
9785                failed |= !self.purge_pair(&mut batch, &table, last, &owners)?;
9786            }
9787        }
9788        if revoked || failed {
9789            if revoked {
9790                report.revoked.push(unit.id.clone());
9791            }
9792            if failed {
9793                report.failed.push(unit.id.clone());
9794            }
9795            batch.status = true;
9796            self.stored(save_unit(
9797                &self.conn,
9798                &UnitRecord {
9799                    status: if failed {
9800                        UnitStatus::Failed
9801                    } else {
9802                        UnitStatus::Revoked
9803                    },
9804                    reason_code: Some(if failed {
9805                        "sync.scope_revoked".to_owned()
9806                    } else {
9807                        start.reason_code.clone()
9808                    }),
9809                    cursor: Some(-1),
9810                    bootstrap_state: None,
9811                    seeded: Vec::new(),
9812                    echo: None,
9813                    ..unit
9814                },
9815            ))?;
9816            return Ok(batch);
9817        }
9818        let completed = start.bootstrap && !rearmed && bootstrap_state.is_none();
9819        let (cursor, bootstrap_state, seeded) = if rearmed {
9820            (Some(-1), None, Vec::new())
9821        } else if start.bootstrap {
9822            let seeded = if bootstrap_state.is_none() {
9823                Vec::new()
9824            } else {
9825                seeded.into_iter().collect()
9826            };
9827            (Some(next_cursor), bootstrap_state, seeded)
9828        } else {
9829            (
9830                unit.cursor,
9831                unit.bootstrap_state.clone(),
9832                unit.seeded.clone(),
9833            )
9834        };
9835        self.stored(save_unit(
9836            &self.conn,
9837            &UnitRecord {
9838                reason_code: None,
9839                echo: Some(echo.clone()),
9840                cursor,
9841                bootstrap_state,
9842                seeded,
9843                ..unit.clone()
9844            },
9845        ))?;
9846        if !completed {
9847            return Ok(batch);
9848        }
9849        // §7.1: a completed bootstrap covers acknowledged intent of its pairs
9850        // at or below its pin.
9851        for table in self.schema.tables.clone() {
9852            let Some(effective) = covers(&table, &unit.scopes)
9853                .then(|| pair_scopes(&unit.scopes, &table.name, Some(&echo)))
9854                .flatten()
9855            else {
9856                continue;
9857            };
9858            let mut statement = self
9859                .conn
9860                .prepare("SELECT id,intent_json FROM _syncular_acked_rows WHERE tbl=?1 AND commit_seq<=?2")
9861                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9862            let rows = statement
9863                .query_map(rusqlite::params![table.name, next_cursor], |row| {
9864                    Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
9865                })
9866                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9867            let mut covered = HashSet::new();
9868            for row in rows {
9869                let (id, intent) =
9870                    row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9871                let values: Map<String, Value> =
9872                    serde_json::from_str(&intent).map_err(|_| "sync.local_corrupt".to_owned())?;
9873                if owns_local_row(&table, &effective, &[], &values) {
9874                    covered.insert(id);
9875                }
9876            }
9877            drop(statement);
9878            for id in covered {
9879                self.conn
9880                    .execute(
9881                        "DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2 AND commit_seq<=?3",
9882                        rusqlite::params![table.name, id, next_cursor],
9883                    )
9884                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9885                self.overlay_dirty.table(&table.name);
9886                batch.table(&table.name);
9887            }
9888        }
9889        Ok(batch)
9890    }
9891
9892    /// §3.3: purge one revoked pair inside the caller's observation: delete
9893    /// the rows matching its last effective scopes that no owner holds, drop
9894    /// the retained intent and every pending commit that provably writes into
9895    /// them, and delete now-unauthorized blob bodies. `Ok(false)` when the
9896    /// table has no local scope-column mapping (fail closed: nothing deleted).
9897    fn purge_pair(
9898        &mut self,
9899        batch: &mut ChangeAccumulator,
9900        table: &TableSchema,
9901        last: &[(String, Vec<String>)],
9902        owners: &[Vec<(String, Vec<String>)>],
9903    ) -> Result<bool, String> {
9904        if local_scope_predicate(table, last, owners).is_err() {
9905            return Ok(false);
9906        }
9907        for commit in self.drop_failed_commits(|name, values| {
9908            name == table.name && owns_local_row(table, last, owners, values)
9909        })? {
9910            batch.conflicts = true;
9911            batch.rejections = true;
9912            batch.outcomes = true;
9913            for operation in commit.ops {
9914                batch.table(&operation.table);
9915            }
9916        }
9917        if !self.purge_scope_rows(&table.name, last, owners)? {
9918            return Err("storage.failed: the revoked scope purge failed".to_owned());
9919        }
9920        self.record_scope_map(batch, &table.name, last);
9921        if self.drop_doomed_outbox(table, last, owners)? {
9922            batch.status = true;
9923            batch.rejections = true;
9924            batch.outcomes = true;
9925        }
9926        // §5.9.7 B2: revocation deletes now-unauthorized blob bodies.
9927        self.delete_unreferenced_cached_blobs()?;
9928        Ok(true)
9929    }
9930
9931    /// §4.5 client application: apply one live-section or delta `COMMIT`
9932    /// with only the changes that match the persisted echo of a receiving
9933    /// unit; every applied change then goes through the version guard.
9934    /// Returns whether any change of the frame applied.
9935    fn apply_live_commit(
9936        &mut self,
9937        receivers: &[UnitRecord],
9938        tables: &[String],
9939        changes: Vec<ssp2::model::Change>,
9940        commit_seq: i64,
9941        source: AppliedServerCommitSource,
9942    ) -> Result<bool, TransportError> {
9943        let changes: Vec<ssp2::model::Change> = changes
9944            .into_iter()
9945            .filter(|change| {
9946                // An unknown table fails loud in the apply below.
9947                let Some(table) = tables
9948                    .get(change.table_index as usize)
9949                    .and_then(|name| self.schema.table(name))
9950                else {
9951                    return true;
9952                };
9953                receivers.iter().any(|unit| {
9954                    covers(table, &unit.scopes)
9955                        && pair_scopes(&unit.scopes, &table.name, unit.echo.as_deref()).is_some_and(
9956                            |scopes| {
9957                                scopes.iter().all(|(variable, values)| {
9958                                    change
9959                                        .scopes
9960                                        .iter()
9961                                        .find(|(key, _)| key == variable)
9962                                        .is_some_and(|(_, value)| values.contains(value))
9963                                })
9964                            },
9965                        )
9966                })
9967            })
9968            .collect();
9969        if changes.is_empty() {
9970            return Ok(false);
9971        }
9972        let affected_tables = changes
9973            .iter()
9974            .filter_map(|change| tables.get(change.table_index as usize))
9975            .cloned()
9976            .collect::<BTreeSet<_>>();
9977        let previous_batches = self.change_queue.len();
9978        self.apply_commit_frame(tables, &changes, commit_seq)?;
9979        if self.change_queue.len() > previous_batches {
9980            if let Some(batch) = self.change_queue.back_mut() {
9981                batch
9982                    .applied_commits
9983                    .extend(
9984                        affected_tables
9985                            .into_iter()
9986                            .map(|table| AppliedServerCommit {
9987                                table,
9988                                commit_seq,
9989                                source,
9990                            }),
9991                    );
9992            }
9993        }
9994        Ok(true)
9995    }
9996
9997    // Both pull and realtime frames use the same durable observation boundary.
9998    fn apply_commit_frame(
9999        &mut self,
10000        tables: &[String],
10001        changes: &[ssp2::model::Change],
10002        sequence: i64,
10003    ) -> Result<(), TransportError> {
10004        #[cfg(feature = "bench-internals")]
10005        let _phase = self.benchmark_phases.start(Phase::CommitApply);
10006        let rollback = self.capture();
10007        self.coalesce_unit(changes.len())
10008            .and_then(|()| self.begin_observation("syncular_commit"))
10009            .map_err(|message| TransportError::new("storage.failed", message))?;
10010        let applied = (|| {
10011            let mut batch = ChangeAccumulator::default();
10012            self.record_commit_changes(&mut batch, tables, changes);
10013            // §7.1 chained base: a change names the version an acknowledged own
10014            // commit wrote; pending upserts chained on it move to that version.
10015            for change in changes {
10016                let Some(table) = tables.get(change.table_index as usize) else {
10017                    continue;
10018                };
10019                let acknowledged = self
10020                    .conn
10021                    .prepare_cached("SELECT op_json FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2 AND commit_seq=?3")
10022                    .and_then(|mut statement| {
10023                        statement
10024                            .query_map(rusqlite::params![table, change.row_id, sequence], |row| row.get::<_, String>(0))?
10025                            .collect::<Result<Vec<_>, _>>()
10026                    })
10027                    .map_err(|error| TransportError::new("storage.failed", Self::sqlite_failure(&self.storage_failure, error)))?
10028                    .iter()
10029                    .map(|op| serde_json::from_str::<CommitOperation>(op).map(OutboxOp::from))
10030                    .collect::<Result<Vec<_>, _>>()
10031                    .map_err(|_| TransportError::new("storage.failed", "sync.local_corrupt: invalid acknowledged operation".to_owned()))?;
10032                for index in 0..self.outbox.len() {
10033                    if !self.outbox[index]
10034                        .ops
10035                        .iter()
10036                        .any(|op| acknowledged.iter().any(|earlier| chains_on(earlier, op)))
10037                    {
10038                        continue;
10039                    }
10040                    let commit = Arc::make_mut(&mut self.outbox[index]);
10041                    for op in &mut commit.ops {
10042                        if acknowledged.iter().any(|earlier| chains_on(earlier, op)) {
10043                            op.base_version = Some(change.row_version);
10044                        }
10045                    }
10046                    let ops = serde_json::to_string(
10047                        &commit
10048                            .ops
10049                            .iter()
10050                            .map(CommitOperation::from)
10051                            .collect::<Vec<_>>(),
10052                    )
10053                    .map_err(|_| {
10054                        TransportError::new(
10055                            "storage.failed",
10056                            "sync.local_corrupt: unencodable outbox".to_owned(),
10057                        )
10058                    })?;
10059                    let id = commit.client_commit_id.clone();
10060                    self.conn
10061                        .execute(
10062                            "UPDATE _syncular_outbox SET ops_json=?1 WHERE commit_id=?2",
10063                            rusqlite::params![ops, id],
10064                        )
10065                        .map_err(|error| {
10066                            TransportError::new(
10067                                "storage.failed",
10068                                Self::sqlite_failure(&self.storage_failure, error),
10069                            )
10070                        })?;
10071                }
10072            }
10073            {
10074                let storage_failed = |error| {
10075                    TransportError::new(
10076                        "storage.failed",
10077                        Self::sqlite_failure(&self.storage_failure, error),
10078                    )
10079                };
10080                let pinned: HashSet<(&str, &str)> = self
10081                    .outbox
10082                    .iter()
10083                    .flat_map(|commit| &commit.ops)
10084                    .map(|op| (op.table.as_str(), op.row_id.as_str()))
10085                    .collect();
10086                for change in changes {
10087                    let table = tables.get(change.table_index as usize).ok_or_else(|| {
10088                        TransportError::new(
10089                            "sync.invalid_request",
10090                            "change tableIndex out of range".to_owned(),
10091                        )
10092                    })?;
10093                    if pinned.contains(&(table.as_str(), change.row_id.as_str())) {
10094                        self.conn.prepare_cached("INSERT INTO _syncular_row_deliveries(tbl,id,commit_seq) VALUES (?1,?2,?3) ON CONFLICT(tbl,id) DO UPDATE SET commit_seq=MAX(commit_seq,excluded.commit_seq)")
10095                            .and_then(|mut statement| statement.execute(rusqlite::params![table, change.row_id, sequence]))
10096                            .map_err(storage_failed)?;
10097                    }
10098                    self.conn.prepare_cached("DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2 AND commit_seq<=?3")
10099                        .and_then(|mut statement| statement.execute(rusqlite::params![table, change.row_id, sequence]))
10100                        .map_err(storage_failed)?;
10101                }
10102            }
10103            self.apply_commit_changes(tables, changes)?;
10104            self.rebuild_overlay_if_dirty()
10105                .map_err(|message| TransportError::new("storage.failed", message))?;
10106            self.finish_observation("syncular_commit", batch)
10107                .map_err(|message| TransportError::new("storage.failed", message))
10108        })();
10109        if applied.is_err() {
10110            self.rollback_observation("syncular_commit");
10111            self.restore(rollback);
10112        }
10113        applied
10114    }
10115
10116    fn apply_commit_changes(
10117        &mut self,
10118        tables: &[String],
10119        changes: &[ssp2::model::Change],
10120    ) -> Result<(), TransportError> {
10121        // A clean overlay can reconcile changed primary keys independently
10122        // when their tables have no secondary unique constraints. Such a
10123        // constraint can change whether another row's write succeeds, so a
10124        // table with one and any local intent takes the table rebuild.
10125        let changed_tables: BTreeSet<&str> = changes
10126            .iter()
10127            .filter_map(|change| tables.get(change.table_index as usize))
10128            .map(String::as_str)
10129            .collect();
10130        let mut mirror_visible = !self.overlay_dirty.get();
10131        for name in &changed_tables {
10132            if mirror_visible
10133                && self
10134                    .schema
10135                    .table(name)
10136                    .is_some_and(|table| table.indexes.iter().any(|index| index.unique))
10137            {
10138                let (pending, protected) = self
10139                    .table_intent(name)
10140                    .map_err(|message| TransportError::new("storage.failed", message))?;
10141                mirror_visible = !pending && !protected;
10142            }
10143        }
10144        for change in changes {
10145            let table_name = tables.get(change.table_index as usize).ok_or_else(|| {
10146                TransportError::new(
10147                    "sync.invalid_request",
10148                    "change tableIndex out of range".to_owned(),
10149                )
10150            })?;
10151            let table = self.schema.table(table_name).ok_or_else(|| {
10152                TransportError::new(
10153                    "sync.schema_mismatch",
10154                    format!("change targets unknown table {table_name:?}"),
10155                )
10156            })?;
10157            let landed = match change.op {
10158                Op::Upsert => {
10159                    let payload = change.row.as_ref().ok_or_else(|| {
10160                        TransportError::new(
10161                            "sync.invalid_request",
10162                            "upsert change without row payload".to_owned(),
10163                        )
10164                    })?;
10165                    // §5.11: decrypt encrypted columns on apply, or seal the row.
10166                    let (row, sealed) = {
10167                        #[cfg(feature = "bench-internals")]
10168                        let _phase = self.benchmark_phases.start(Phase::RowDecode);
10169                        // §5.11: a decrypt failure keeps its own identity.
10170                        decode_row_bytes(table, payload, &self.encryption)?
10171                    };
10172                    let table_name = table.name.clone();
10173                    // §4.5 version guard: only a newer upsert lands.
10174                    let landed = self
10175                        .write_base_row(&table_name, &row, change.row_version, sealed.as_ref())
10176                        .map_err(|m| TransportError::new("sync.invalid_request", m))?;
10177                    if mirror_visible && landed {
10178                        self.write_row(
10179                            &visible_table(&table_name),
10180                            &table_name,
10181                            &row,
10182                            change.row_version,
10183                            sealed.as_ref(),
10184                        )
10185                        .map_err(|m| TransportError::new("sync.invalid_request", m))?;
10186                    }
10187                    landed
10188                }
10189                // §4.5 version guard: a delete at or below the local version
10190                // is skipped, a delete of an absent row is a no-op.
10191                Op::Delete => self
10192                    .delete_base_row(table_name, &change.row_id, change.row_version)
10193                    .map_err(|m| TransportError::new("sync.invalid_request", m))?,
10194            };
10195            // The change dropped the row's acknowledged intent either way
10196            // (§7.1), so the overlay of a skipped change rebuilds from the
10197            // unchanged base.
10198            self.overlay_dirty.table(table_name);
10199            if mirror_visible && !(landed && change.op == Op::Upsert) {
10200                let predicate = row_id_predicate(table);
10201                let visible = visible_table(table_name);
10202                let mut statements = vec![format!("DELETE FROM {visible} WHERE {predicate}")];
10203                if !landed {
10204                    statements.push(format!(
10205                        "INSERT INTO {visible} SELECT * FROM {} WHERE {predicate}",
10206                        base_table(table_name)
10207                    ));
10208                }
10209                for statement in &statements {
10210                    self.conn
10211                        .prepare_cached(statement)
10212                        .and_then(|mut statement| {
10213                            statement.execute(rusqlite::params![change.row_id])
10214                        })
10215                        .map_err(|error| {
10216                            TransportError::new("sync.invalid_request", error.to_string())
10217                        })?;
10218                }
10219            }
10220        }
10221        if mirror_visible {
10222            let changed_rows: HashSet<(&str, &str)> = changes
10223                .iter()
10224                .map(|change| {
10225                    (
10226                        tables[change.table_index as usize].as_str(),
10227                        change.row_id.as_str(),
10228                    )
10229                })
10230                .collect();
10231            self.replay_overlay(&changed_tables, Some(&changed_rows))
10232                .map_err(|message| TransportError::new("storage.failed", message))?;
10233            self.overlay_dirty.set(false);
10234        }
10235        Ok(())
10236    }
10237
10238    /// §5.6 segment application: validate against the generated schema,
10239    /// clear the pair's scope on the table's first page (fail closed
10240    /// without a mapping), then upsert each row with its segment-carried
10241    /// server version under the §4.5 version guard (§5.2).
10242    fn apply_segment(
10243        &mut self,
10244        table: &TableSchema,
10245        segment: &RowsSegment,
10246        first_page: bool,
10247        effective: &[(String, Vec<String>)],
10248        unit_id: &str,
10249    ) -> Result<u32, SectionError> {
10250        // §5.2: the column table validates against the generated schema —
10251        // order, names, types, nullability; mismatch is fatal. §5.11: the
10252        // server sends the WIRE types (bytes for an encrypted column), so
10253        // validate against wire_columns.
10254        let column = if segment.columns.len() != table.wire_columns.len() {
10255            Some(Value::from("count"))
10256        } else {
10257            segment
10258                .columns
10259                .iter()
10260                .zip(table.wire_columns.iter())
10261                .position(|(a, b)| a.name != b.name || a.ty != b.ty || a.nullable != b.nullable)
10262                .map(Value::from)
10263        };
10264        if segment.table != table.name
10265            || segment.schema_version != self.schema.version
10266            || column.is_some()
10267        {
10268            let mut details = Map::new();
10269            details.insert("table".to_owned(), Value::from(table.name.as_str()));
10270            if let Some(column) = column {
10271                details.insert("column".to_owned(), column);
10272            }
10273            return Err(SectionError::Transport(TransportError {
10274                code: "sync.schema_mismatch".to_owned(),
10275                message: "rows segment columns do not match the generated schema (§5.2)".to_owned(),
10276                details: Some(Value::Object(details)),
10277            }));
10278        }
10279        let mut applied = 0u32;
10280        let empty = Vec::new();
10281        let blocks = if segment.blocks.is_empty() {
10282            std::slice::from_ref(&empty)
10283        } else {
10284            &segment.blocks
10285        };
10286        for (index, block) in blocks.iter().enumerate() {
10287            let clear = first_page && index == 0;
10288            if !clear && block.is_empty() {
10289                continue;
10290            }
10291            let rollback = self.capture();
10292            let (pending, protected) = self
10293                .table_intent(&table.name)
10294                .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10295            let reconcile_pending =
10296                !protected && pending && table.indexes.iter().any(|index| index.unique);
10297            let mirror_visible = !protected && !reconcile_pending;
10298            self.coalesce_unit(block.len())
10299                .and_then(|()| self.begin_observation("syncular_segment_block"))
10300                .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10301            let outcome = (|| {
10302                self.rebuild_overlay_if_dirty()
10303                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10304                let mut changed_keys = Vec::new();
10305                let mut batch = ChangeAccumulator::default();
10306                if clear && self.clear_scope(table, effective, unit_id)? || !block.is_empty() {
10307                    batch.table(&table.name);
10308                }
10309                for (row_index, row) in block.iter().enumerate() {
10310                    let decrypted;
10311                    let mut sealed = None;
10312                    let values = if table.has_encrypted_columns() {
10313                        let mut values = row.values.clone();
10314                        sealed = crate::values::decrypt_segment_row(
10315                            table,
10316                            &mut values,
10317                            &self.encryption,
10318                        )
10319                        .map_err(SectionError::Transport)?;
10320                        decrypted = values;
10321                        &decrypted
10322                    } else {
10323                        &row.values
10324                    };
10325                    if !self
10326                        .write_base_row(&table.name, values, row.server_version, sealed.as_ref())
10327                        .map_err(|message| {
10328                            SectionError::Abort("sync.invalid_request".into(), message)
10329                        })?
10330                    {
10331                        continue;
10332                    }
10333                    if reconcile_pending {
10334                        changed_keys.push(
10335                            owned_sql_value(RowParam::Cell(&values[table.pk_index])).map_err(
10336                                |message| SectionError::Abort("storage.failed".into(), message),
10337                            )?,
10338                        );
10339                    }
10340                    if mirror_visible {
10341                        self.write_row(
10342                            &visible_table(&table.name),
10343                            &table.name,
10344                            values,
10345                            row.server_version,
10346                            sealed.as_ref(),
10347                        )
10348                        .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10349                    }
10350                    let processed = u64::from(applied) + row_index as u64 + 1;
10351                    if processed.is_multiple_of(1024) {
10352                        self.progress.update(|p| p.rows_processed = processed);
10353                    }
10354                }
10355                self.reconcile_import(table, &mut changed_keys, protected, reconcile_pending)
10356                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10357                self.finish_observation("syncular_segment_block", batch)
10358                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))
10359            })();
10360            if let Err(error) = outcome {
10361                self.rollback_observation("syncular_segment_block");
10362                self.restore(rollback);
10363                return Err(error);
10364            }
10365            applied += block.len() as u32;
10366            self.progress
10367                .update(|p| p.rows_processed = u64::from(applied));
10368            std::thread::yield_now();
10369        }
10370        self.progress
10371            .update(|p| p.rows_processed = u64::from(applied));
10372        Ok(applied)
10373    }
10374
10375    /// §5.3 sqlite-image application: validate the in-file metadata
10376    /// against the descriptor, validate column names/order against the
10377    /// generated schema, run the §5.6 first-page clear when fresh, then
10378    /// replace-or-upsert every image row with its `_syncular_version`.
10379    /// Mechanics: the image lands in a temp file read through a second
10380    /// rusqlite connection, equivalent to ATTACH + INSERT…SELECT.
10381    /// Each image chunk commits its writes and revision before readers continue.
10382    #[allow(clippy::too_many_arguments)]
10383    fn apply_sqlite_segment(
10384        &mut self,
10385        table: &TableSchema,
10386        bytes: &[u8],
10387        first_page: bool,
10388        effective: &[(String, Vec<String>)],
10389        unit_id: &str,
10390        segment_id: &str,
10391        resumed: i64,
10392        row_count: i64,
10393        as_of_commit_seq: i64,
10394        scope_digest: &str,
10395    ) -> Result<u32, SectionError> {
10396        let invalid = |detail: &str| {
10397            SectionError::Abort(
10398                "sync.invalid_request".to_owned(),
10399                format!("sqlite segment rejected: {detail} (§5.3)"),
10400            )
10401        };
10402        // §5.11: an image bypasses the decrypt seam, so a server cannot serve
10403        // one for a table with encrypted columns.
10404        if table.has_encrypted_columns() {
10405            return Err(invalid("table has encrypted columns"));
10406        }
10407
10408        // The image lands beside the replica so it shares the replica's
10409        // directory permissions and volume; an in-memory replica has no
10410        // directory and uses the system temp dir. The file is owner-only.
10411        let directory = self
10412            .previous_version_replica_path()
10413            .and_then(|replica| std::path::Path::new(&replica).parent().map(Into::into))
10414            .unwrap_or_else(std::env::temp_dir);
10415        let path = directory.join(format!(".syncular-image-{}.tmp", uuid::Uuid::new_v4()));
10416        let outcome = (|| {
10417            let mut options = std::fs::OpenOptions::new();
10418            options.write(true).create_new(true);
10419            #[cfg(unix)]
10420            std::os::unix::fs::OpenOptionsExt::mode(&mut options, 0o600);
10421            std::io::Write::write_all(
10422                &mut options
10423                    .open(&path)
10424                    .map_err(|_| invalid("image temp file write failed"))?,
10425                bytes,
10426            )
10427            .map_err(|_| invalid("image temp file write failed"))?;
10428            let img = rusqlite::Connection::open_with_flags(
10429                &path,
10430                rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY,
10431            )
10432            .map_err(|_| invalid("bytes do not open as a SQLite database"))?;
10433            self.apply_sqlite_image(
10434                &img,
10435                table,
10436                first_page,
10437                effective,
10438                unit_id,
10439                segment_id,
10440                resumed,
10441                row_count,
10442                as_of_commit_seq,
10443                scope_digest,
10444            )
10445        })();
10446        let _ = std::fs::remove_file(&path);
10447        outcome
10448    }
10449
10450    #[allow(clippy::too_many_arguments)]
10451    fn apply_sqlite_image(
10452        &mut self,
10453        img: &rusqlite::Connection,
10454        table: &crate::schema::TableSchema,
10455        first_fresh_page: bool,
10456        effective: &[(String, Vec<String>)],
10457        unit_id: &str,
10458        segment_id: &str,
10459        resumed: i64,
10460        row_count: i64,
10461        as_of_commit_seq: i64,
10462        scope_digest: &str,
10463    ) -> Result<u32, SectionError> {
10464        let invalid = |detail: String| {
10465            SectionError::Abort(
10466                "sync.invalid_request".to_owned(),
10467                format!("sqlite segment rejected: {detail} (§5.3)"),
10468            )
10469        };
10470
10471        // 1. Metadata vs descriptor + client state (§5.3 rule 2; exactly
10472        //    one row).
10473        type MetaRow = (i64, String, i64, i64, String, i64, i64);
10474        let meta: MetaRow = img
10475            .query_row(
10476                "SELECT format, \"table\", \"schemaVersion\", \"asOfCommitSeq\",
10477                        \"scopeDigest\", \"rowCount\",
10478                        (SELECT count(*) FROM _syncular_segment)
10479                 FROM _syncular_segment",
10480                [],
10481                |row| {
10482                    Ok((
10483                        row.get(0)?,
10484                        row.get(1)?,
10485                        row.get(2)?,
10486                        row.get(3)?,
10487                        row.get(4)?,
10488                        row.get(5)?,
10489                        row.get(6)?,
10490                    ))
10491                },
10492            )
10493            .map_err(|_| invalid("missing or unreadable _syncular_segment metadata".to_owned()))?;
10494        let (format, meta_table, schema_version, pin, digest, meta_rows, meta_count) = meta;
10495        if meta_count != 1 {
10496            return Err(invalid(format!(
10497                "_syncular_segment must contain exactly one row, found {meta_count}"
10498            )));
10499        }
10500        if format != 1 {
10501            return Err(invalid(format!("format {format}")));
10502        }
10503        if meta_table != table.name {
10504            return Err(invalid(format!("image table {meta_table:?}")));
10505        }
10506        if schema_version != i64::from(self.schema.version) {
10507            return Err(invalid(format!("schemaVersion {schema_version}")));
10508        }
10509        if pin != as_of_commit_seq {
10510            return Err(invalid(format!("asOfCommitSeq {pin}")));
10511        }
10512        if digest != scope_digest {
10513            return Err(invalid("scopeDigest mismatch".to_owned()));
10514        }
10515        if meta_rows != row_count {
10516            return Err(invalid(format!("rowCount {meta_rows}")));
10517        }
10518
10519        // 2. Column names and order vs the generated schema (§5.3 rule 3).
10520        let mut names: Vec<String> = Vec::new();
10521        let mut primary_keys: Vec<String> = Vec::new();
10522        {
10523            let mut stmt = img
10524                .prepare(&format!("PRAGMA table_info({})", quote_ident(&table.name)))
10525                .map_err(|_| invalid("image data table missing".to_owned()))?;
10526            let mut rows = stmt
10527                .query([])
10528                .map_err(|_| invalid("image data table unreadable".to_owned()))?;
10529            while let Some(row) = rows
10530                .next()
10531                .map_err(|_| invalid("image data table unreadable".to_owned()))?
10532            {
10533                if row
10534                    .get::<_, i64>(5)
10535                    .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?
10536                    > 0
10537                {
10538                    primary_keys.push(row.get(1).map_err(|error| {
10539                        invalid(Self::sqlite_failure(&self.storage_failure, error))
10540                    })?);
10541                }
10542                names.push(
10543                    row.get::<_, String>(1)
10544                        .map_err(|_| invalid("image data table unreadable".to_owned()))?,
10545                );
10546            }
10547        }
10548        let mut expected: Vec<&str> = table.columns.iter().map(|c| c.name.as_str()).collect();
10549        expected.push("_syncular_version");
10550        if names.len() != expected.len() || names.iter().zip(expected.iter()).any(|(a, b)| a != b) {
10551            return Err(SectionError::Transport(TransportError {
10552                code: "sync.schema_mismatch".to_owned(),
10553                message: "sqlite segment columns do not match the generated schema (§5.3)"
10554                    .to_owned(),
10555                details: Some(serde_json::json!({ "table": table.name })),
10556            }));
10557        }
10558
10559        if primary_keys != vec![table.primary_key.clone()] {
10560            return Err(invalid(
10561                "image primary key does not match generated schema".into(),
10562            ));
10563        }
10564        let actual_count: i64 = img
10565            .query_row(
10566                &format!("SELECT count(*) FROM {}", quote_ident(&table.name)),
10567                [],
10568                |row| row.get(0),
10569            )
10570            .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
10571        if actual_count != row_count {
10572            return Err(invalid("image row count does not match descriptor".into()));
10573        }
10574        let insert = self.insert_row_sql(&base_table(&table.name), table);
10575        let visible_insert = self.insert_row_sql(&visible_table(&table.name), table);
10576        let column_list = names
10577            .iter()
10578            .map(|n| quote_ident(n))
10579            .collect::<Vec<_>>()
10580            .join(", ");
10581        // §5.6: an earlier attempt at this position committed the first
10582        // `resumed` rows in primary-key order; the content address pins them.
10583        let mut stmt = img
10584            .prepare(&format!(
10585                "SELECT {column_list} FROM {} ORDER BY {} LIMIT -1 OFFSET ?1",
10586                quote_ident(&table.name),
10587                quote_ident(&table.primary_key)
10588            ))
10589            .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
10590        let mut rows = stmt
10591            .query([resumed])
10592            .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
10593        let resumed = u32::try_from(resumed)
10594            .ok()
10595            .filter(|rows| i64::from(*rows) <= actual_count)
10596            .ok_or_else(|| invalid("applied-segment record exceeds the image".into()))?;
10597        let mut applied = resumed;
10598        let mut limit = if first_fresh_page {
10599            1024
10600        } else {
10601            self.image_chunk_rows
10602                .get(&table.name)
10603                .copied()
10604                .unwrap_or(1024)
10605        };
10606        let (mut write, mut commit) = (std::time::Duration::ZERO, std::time::Duration::ZERO);
10607        loop {
10608            let clear = applied == 0 && first_fresh_page;
10609            let rollback = self.capture();
10610            let (pending, protected) = self
10611                .table_intent(&table.name)
10612                .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10613            let reconcile_pending =
10614                !protected && pending && table.indexes.iter().any(|index| index.unique);
10615            let mirror_visible = !protected && !reconcile_pending;
10616            // §5.3: a table's first chunk holds 1,024 rows; a later range image
10617            // starts at the size its previous range reached. A later chunk holds at
10618            // least 1,024 rows, at most twice the rows of the one before, at
10619            // most 65,536, and at most the rows the one before applied per
10620            // 250 ms from its start to its COMMIT; a chunk that replays local
10621            // writes holds at most 1,024.
10622            let chunk = if pending || protected {
10623                limit.min(1024)
10624            } else {
10625                limit
10626            };
10627            self.coalesce_unit(chunk as usize)
10628                .and_then(|()| self.begin_observation("syncular_image_chunk"))
10629                .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10630            let started = std::time::Instant::now();
10631            let result = (|| {
10632                self.rebuild_overlay_if_dirty()
10633                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10634                let mut changed_keys = Vec::new();
10635                let mut batch = ChangeAccumulator::default();
10636                if clear && self.clear_scope(table, effective, unit_id)? {
10637                    batch.table(&table.name);
10638                }
10639                let mut processed = 0u32;
10640                {
10641                    let mut ins = self.conn.prepare_cached(&insert).map_err(|error| {
10642                        invalid(Self::sqlite_failure(&self.storage_failure, error))
10643                    })?;
10644                    let mut mirror = if mirror_visible {
10645                        Some(self.conn.prepare_cached(&visible_insert).map_err(|error| {
10646                            invalid(Self::sqlite_failure(&self.storage_failure, error))
10647                        })?)
10648                    } else {
10649                        None
10650                    };
10651                    for _ in 0..chunk {
10652                        let Some(row) = rows.next().map_err(|error| {
10653                            invalid(Self::sqlite_failure(&self.storage_failure, error))
10654                        })?
10655                        else {
10656                            break;
10657                        };
10658                        for (i, column) in table.columns.iter().enumerate() {
10659                            let cell = row.get_ref(i).map_err(|error| {
10660                                invalid(Self::sqlite_failure(&self.storage_failure, error))
10661                            })?;
10662                            let param = image_cell_param(column, cell).map_err(invalid)?;
10663                            ins.raw_bind_parameter(i + 1, &param).map_err(|error| {
10664                                invalid(Self::sqlite_failure(&self.storage_failure, error))
10665                            })?;
10666                            if let Some(mirror) = &mut mirror {
10667                                mirror.raw_bind_parameter(i + 1, &param).map_err(|error| {
10668                                    invalid(Self::sqlite_failure(&self.storage_failure, error))
10669                                })?;
10670                            }
10671                        }
10672                        let version_index = table.columns.len();
10673                        let version: i64 = row.get(version_index).map_err(|error| {
10674                            invalid(Self::sqlite_failure(&self.storage_failure, error))
10675                        })?;
10676                        if version < 1 {
10677                            return Err(invalid("image row version must be positive".into()));
10678                        }
10679                        ins.raw_bind_parameter(version_index + 1, version)
10680                            .map_err(|error| {
10681                                invalid(Self::sqlite_failure(&self.storage_failure, error))
10682                            })?;
10683                        // §4.5 version guard: the base upsert decides.
10684                        let landed = ins.raw_execute().map_err(|error| {
10685                            invalid(Self::sqlite_failure(&self.storage_failure, error))
10686                        })? > 0;
10687                        processed += 1;
10688                        if processed.is_multiple_of(1024) && processed < chunk {
10689                            self.progress
10690                                .update(|p| p.rows_processed = u64::from(applied + processed));
10691                        }
10692                        if !landed {
10693                            continue;
10694                        }
10695                        if let Some(mirror) = &mut mirror {
10696                            mirror
10697                                .raw_bind_parameter(version_index + 1, version)
10698                                .map_err(|error| {
10699                                    invalid(Self::sqlite_failure(&self.storage_failure, error))
10700                                })?;
10701                            mirror.raw_execute().map_err(|error| {
10702                                invalid(Self::sqlite_failure(&self.storage_failure, error))
10703                            })?;
10704                        }
10705                        if reconcile_pending {
10706                            changed_keys.push(row.get::<_, SqlValue>(table.pk_index).map_err(
10707                                |error| invalid(Self::sqlite_failure(&self.storage_failure, error)),
10708                            )?);
10709                        }
10710                    }
10711                }
10712                if processed > 0 {
10713                    batch.table(&table.name);
10714                    self.overlay_dirty.table(&table.name);
10715                }
10716                self.stored(record_applied_rows(
10717                    &self.conn,
10718                    unit_id,
10719                    segment_id,
10720                    i64::from(applied + processed),
10721                ))
10722                .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10723                self.reconcile_import(table, &mut changed_keys, protected, reconcile_pending)
10724                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10725                let written = std::time::Instant::now();
10726                self.finish_observation("syncular_image_chunk", batch)
10727                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10728                Ok((processed, written))
10729            })();
10730            let (processed, written) = match result {
10731                Ok(done) => done,
10732                Err(error) => {
10733                    self.rollback_observation("syncular_image_chunk");
10734                    self.restore(rollback);
10735                    return Err(error);
10736                }
10737            };
10738            // The chunk commits before the yield, also inside a coalesced apply.
10739            let coalesced = self.coalesced.is_some();
10740            if coalesced {
10741                self.commit_coalesced_apply()
10742                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10743            }
10744            write += written - started;
10745            commit += written.elapsed();
10746            applied += processed;
10747            let last = processed < chunk || i64::from(applied) == actual_count;
10748            // A file replica's WAL keeps its high-water size until truncated.
10749            if last && self.conn.is_autocommit() {
10750                self.conn
10751                    .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))
10752                    .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
10753            }
10754            if coalesced {
10755                self.begin_coalesced_apply()
10756                    .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10757            }
10758            self.progress.update(|p| {
10759                p.rows_processed = u64::from(applied);
10760                p.import_write_ms = u64::try_from(write.as_millis()).ok();
10761                p.import_commit_ms = u64::try_from(commit.as_millis()).ok();
10762            });
10763            let duration = started.elapsed().as_secs_f64() * 1000.0;
10764            #[cfg(test)]
10765            let duration = self.image_chunk_ms.get().unwrap_or(duration);
10766            std::thread::yield_now();
10767            if last {
10768                break;
10769            }
10770            // A zero duration saturates the cast and leaves the doubling.
10771            limit = (processed * 2)
10772                .min(65_536)
10773                .min((f64::from(processed) * 250.0 / duration) as u32)
10774                .max(1024);
10775            self.image_chunk_rows.insert(table.name.clone(), limit);
10776        }
10777        if i64::from(applied) != actual_count {
10778            return Err(invalid(format!(
10779                "image yielded {applied} rows, expected {actual_count}"
10780            )));
10781        }
10782        Ok(applied - resumed)
10783    }
10784
10785    /// §7.1 keyed reconciliation of the rows `drained` commits touch: restore
10786    /// them from the base and replay their remaining intent. A table with a
10787    /// secondary unique index restores and replays every row with intent,
10788    /// since one row's value can admit or defer another row's write.
10789    fn reconcile_rows(&self, drained: &[Arc<OutboxCommit>]) -> Result<(), String> {
10790        let names: BTreeSet<&str> = drained
10791            .iter()
10792            .flat_map(|commit| &commit.ops)
10793            .map(|op| op.table.as_str())
10794            .collect();
10795        for name in names {
10796            let Some(table) = self.schema.table(name) else {
10797                continue;
10798            };
10799            let ops = drained.iter().flat_map(|commit| &commit.ops);
10800            let tables = BTreeSet::from([name]);
10801            let rows: HashSet<_> = ops
10802                .clone()
10803                .map(|op| (op.table.as_str(), op.row_id.as_str()))
10804                .collect();
10805            let unique = table.indexes.iter().any(|index| index.unique);
10806            let acknowledged = if unique {
10807                self.acknowledged_operations(None, Some(&tables))?
10808            } else {
10809                Vec::new()
10810            };
10811            let peers = acknowledged
10812                .iter()
10813                .chain(
10814                    self.failed_commits
10815                        .iter()
10816                        .flat_map(|(commit, _)| &commit.ops),
10817                )
10818                .chain(self.outbox.iter().flat_map(|commit| &commit.ops))
10819                .filter(|_| unique);
10820            self.restore_base_rows(table, &mut Vec::new(), peers.chain(ops.clone()))?;
10821            self.replay_overlay(&tables, (!unique).then_some(&rows))?;
10822        }
10823        Ok(())
10824    }
10825
10826    /// Whether `table` has pending outbox intent, and whether it has protected
10827    /// (acknowledged or retained failed) intent.
10828    fn table_intent(&self, table: &str) -> Result<(bool, bool), String> {
10829        let touches = |op: &OutboxOp| op.table == table;
10830        Ok((
10831            self.outbox
10832                .iter()
10833                .flat_map(|commit| &commit.ops)
10834                .any(touches),
10835            self.failed_commits
10836                .iter()
10837                .flat_map(|(commit, _)| &commit.ops)
10838                .any(touches)
10839                || self
10840                    .conn
10841                    .query_row(
10842                        "SELECT EXISTS(SELECT 1 FROM _syncular_acked_rows WHERE tbl=?1)",
10843                        [table],
10844                        |row| row.get(0),
10845                    )
10846                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
10847        ))
10848    }
10849
10850    /// §7.1 reconciliation after an import block or chunk. Keyed replay covers
10851    /// only pending intent, so a table with protected intent rebuilds; a
10852    /// `restore` restores the imported `keys` and every pending row first.
10853    fn reconcile_import(
10854        &mut self,
10855        table: &TableSchema,
10856        keys: &mut Vec<SqlValue>,
10857        protected: bool,
10858        restore: bool,
10859    ) -> Result<(), String> {
10860        if protected {
10861            return self.rebuild_overlay_if_dirty();
10862        }
10863        let pending = self.outbox.iter().flat_map(|commit| &commit.ops);
10864        if restore {
10865            self.restore_base_rows(table, keys, pending.clone())?;
10866        }
10867        self.apply_outbox_ops(pending.filter(|op| op.table == table.name))?;
10868        self.overlay_dirty.set(false);
10869        Ok(())
10870    }
10871
10872    /// Restore the visible rows of `keys` and of the rows `ops` touch in
10873    /// `table` from the base before the caller replays their intent. The
10874    /// caller owns the transaction and publishes its revision after replay.
10875    fn restore_base_rows<'a>(
10876        &self,
10877        table: &TableSchema,
10878        keys: &mut Vec<SqlValue>,
10879        ops: impl IntoIterator<Item = &'a OutboxOp>,
10880    ) -> Result<(), String> {
10881        let visible = visible_table(&table.name);
10882        let base = base_table(&table.name);
10883        let pk = quote_ident(&table.primary_key);
10884        let predicate = row_id_predicate(table);
10885        let mut lookup = self.conn.prepare_cached(&format!(
10886            "SELECT {pk} FROM {base} WHERE {predicate} UNION SELECT {pk} FROM {visible} WHERE {predicate}"
10887        )).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
10888        let mut deleted = HashSet::new();
10889        for op in ops.into_iter().filter(|op| op.table == table.name) {
10890            if op.upsert {
10891                if let Some(values) = &op.values {
10892                    let key = json_to_column_value(
10893                        &table.columns[table.pk_index],
10894                        values.get(&table.primary_key),
10895                    )?;
10896                    keys.push(owned_sql_value(RowParam::Cell(&key))?);
10897                }
10898            } else if deleted.insert(&op.row_id) {
10899                keys.extend(
10900                    lookup
10901                        .query_map([&op.row_id], |row| row.get::<_, SqlValue>(0))
10902                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
10903                        .collect::<Result<Vec<_>, _>>()
10904                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
10905                );
10906            }
10907        }
10908        // Delete the complete affected set before restoring any base value, so
10909        // old optimistic unique values cannot collide with the new server rows.
10910        for chunk in keys.chunks(400) {
10911            let holes = vec!["?"; chunk.len()].join(",");
10912            self.conn
10913                .prepare_cached(&format!("DELETE FROM {visible} WHERE {pk} IN ({holes})"))
10914                .and_then(|mut statement| statement.execute(rusqlite::params_from_iter(chunk)))
10915                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
10916        }
10917        let updates = table
10918            .columns
10919            .iter()
10920            .map(|column| column.name.as_str())
10921            .chain(std::iter::once("_syncular_version"))
10922            .chain(table.sealed_columns().iter().copied())
10923            .filter(|name| *name != table.primary_key)
10924            .map(|name| {
10925                let name = quote_ident(name);
10926                format!("{name}=excluded.{name}")
10927            })
10928            .collect::<Vec<_>>()
10929            .join(",");
10930        for chunk in keys.chunks(400) {
10931            let holes = vec!["?"; chunk.len()].join(",");
10932            self.conn.prepare_cached(&format!("INSERT INTO {visible} SELECT * FROM {base} WHERE {pk} IN ({holes}) ON CONFLICT ({pk}) DO UPDATE SET {updates}"))
10933                .and_then(|mut statement| statement.execute(rusqlite::params_from_iter(chunk)))
10934                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
10935        }
10936        Ok(())
10937    }
10938
10939    // -- application-authorized local purge ---------------------------------------
10940
10941    fn compile_local_data_purge(
10942        &self,
10943        input: &LocalDataPurgeInput,
10944    ) -> Result<(Vec<CompiledLocalDataPurgeTarget>, String), String> {
10945        let invalid = |message: String| format!("sync.invalid_request: {message}");
10946        if input.purge_id.is_empty()
10947            || input.purge_id.len() > 128
10948            || !is_local_operation_code_like(&input.purge_id)
10949        {
10950            return Err(invalid(
10951                "local purge purgeId must be a 1–128 character code-like identifier".to_owned(),
10952            ));
10953        }
10954        if input.targets.is_empty() || input.targets.len() > MAX_LOCAL_PURGE_TARGETS {
10955            return Err(invalid(format!(
10956                "local purge needs between 1 and {MAX_LOCAL_PURGE_TARGETS} targets"
10957            )));
10958        }
10959
10960        let mut deduplicated: BTreeMap<
10961            String,
10962            (CompiledLocalDataPurgeTarget, LocalDataPurgeTarget),
10963        > = BTreeMap::new();
10964        for target in &input.targets {
10965            let table = self.schema.table(&target.table).ok_or_else(|| {
10966                invalid(format!(
10967                    "local purge names unknown table {:?}",
10968                    target.table
10969                ))
10970            })?;
10971            if target.selectors.is_empty() || target.selectors.len() > MAX_LOCAL_PURGE_SELECTORS {
10972                return Err(invalid(format!(
10973                    "local purge target {:?} needs between 1 and {MAX_LOCAL_PURGE_SELECTORS} selectors",
10974                    target.table
10975                )));
10976            }
10977            let mut selectors = Vec::with_capacity(target.selectors.len());
10978            let mut canonical_selectors = BTreeMap::new();
10979            for (column_name, raw_values) in &target.selectors {
10980                let Some((column_index, column)) = table
10981                    .columns
10982                    .iter()
10983                    .enumerate()
10984                    .find(|(_, column)| column.name == *column_name)
10985                else {
10986                    return Err(invalid(format!(
10987                        "local purge target {:?} names unknown column {:?}",
10988                        target.table, column_name
10989                    )));
10990                };
10991                let encrypted = table
10992                    .encrypted_columns
10993                    .iter()
10994                    .any(|candidate| candidate.index == column_index);
10995                if column.ty != ColumnType::String || encrypted {
10996                    return Err(invalid(format!(
10997                        "local purge selector {:?}.{:?} must be a plaintext string column",
10998                        target.table, column_name
10999                    )));
11000                }
11001                if raw_values.is_empty() || raw_values.len() > MAX_LOCAL_PURGE_VALUES {
11002                    return Err(invalid(format!(
11003                        "local purge selector {:?}.{:?} needs between 1 and {MAX_LOCAL_PURGE_VALUES} values",
11004                        target.table, column_name
11005                    )));
11006                }
11007                let mut values = raw_values.clone();
11008                values.sort();
11009                values.dedup();
11010                if values.iter().any(|value| {
11011                    value.is_empty()
11012                        || value.len() > MAX_LOCAL_PURGE_VALUE_LENGTH
11013                        || !is_local_operation_code_like(value)
11014                }) {
11015                    return Err(invalid(format!(
11016                        "local purge selector values must be 1–{MAX_LOCAL_PURGE_VALUE_LENGTH} character code-like identifiers"
11017                    )));
11018                }
11019                selectors.push((column_name.clone(), values.clone()));
11020                canonical_selectors.insert(column_name.clone(), values);
11021            }
11022            let canonical = LocalDataPurgeTarget {
11023                table: target.table.clone(),
11024                selectors: canonical_selectors,
11025            };
11026            let key = serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
11027            deduplicated.insert(
11028                key,
11029                (
11030                    CompiledLocalDataPurgeTarget {
11031                        table: target.table.clone(),
11032                        selectors,
11033                    },
11034                    canonical,
11035                ),
11036            );
11037        }
11038        let targets = deduplicated
11039            .values()
11040            .map(|(compiled, _)| compiled.clone())
11041            .collect::<Vec<_>>();
11042        let canonical = deduplicated
11043            .values()
11044            .map(|(_, target)| target.clone())
11045            .collect::<Vec<_>>();
11046        let canonical_plan =
11047            serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
11048        Ok((targets, canonical_plan))
11049    }
11050
11051    fn local_purge_base_row_ids(
11052        &self,
11053        targets: &[CompiledLocalDataPurgeTarget],
11054    ) -> Result<BTreeMap<String, BTreeSet<String>>, String> {
11055        let mut by_table: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
11056        for target in targets {
11057            let table = self
11058                .schema
11059                .table(&target.table)
11060                .ok_or_else(|| format!("sync.invalid_request: unknown table {:?}", target.table))?;
11061            let mut clauses = Vec::with_capacity(target.selectors.len());
11062            let mut params = Vec::new();
11063            for (column, values) in &target.selectors {
11064                clauses.push(format!(
11065                    "{} IN ({})",
11066                    quote_ident(column),
11067                    values.iter().map(|_| "?").collect::<Vec<_>>().join(", ")
11068                ));
11069                params.extend(values.iter().cloned().map(SqlValue::Text));
11070            }
11071            let sql = format!(
11072                "SELECT CAST({} AS TEXT) FROM {} WHERE {}",
11073                quote_ident(&table.primary_key),
11074                base_table(&target.table),
11075                clauses.join(" AND ")
11076            );
11077            let mut statement = self
11078                .conn
11079                .prepare(&sql)
11080                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11081            let rows = statement
11082                .query_map(rusqlite::params_from_iter(params), |row| {
11083                    let value = sql_ref_to_json(&table.columns[table.pk_index], row.get_ref(0)?);
11084                    render_row_id_json(Some(&value))
11085                        .map_err(|message| rusqlite::Error::ToSqlConversionFailure(message.into()))
11086                })
11087                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11088            let ids = by_table.entry(target.table.clone()).or_default();
11089            for row in rows {
11090                ids.insert(
11091                    row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
11092                );
11093            }
11094        }
11095        Ok(by_table)
11096    }
11097
11098    fn local_purge_values_match(
11099        target: &CompiledLocalDataPurgeTarget,
11100        values: &Map<String, Value>,
11101    ) -> bool {
11102        target.selectors.iter().all(|(column, allowed)| {
11103            values
11104                .get(column)
11105                .and_then(Value::as_str)
11106                .is_some_and(|value| allowed.iter().any(|candidate| candidate == value))
11107        })
11108    }
11109
11110    /// Apply one host-authorized local security purge. The host MUST gate the
11111    /// corresponding subscriptions first; this primitive owns local SQLite
11112    /// cleanup and never grants/revokes server authority by itself.
11113    pub fn purge_local_data(
11114        &mut self,
11115        input: &LocalDataPurgeInput,
11116    ) -> Result<LocalDataPurgeResult, String> {
11117        let (targets, canonical_plan) = self.compile_local_data_purge(input)?;
11118        let meta_key = format!("localPurge:{}", input.purge_id);
11119        let applied_plan = self
11120            .conn
11121            .query_row(
11122                "SELECT value FROM _syncular_meta WHERE key = ?1",
11123                rusqlite::params![meta_key],
11124                |row| row.get::<_, String>(0),
11125            )
11126            .optional()
11127            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11128        if let Some(applied_plan) = applied_plan {
11129            if applied_plan != canonical_plan {
11130                return Err(format!(
11131                    "sync.invalid_request: local purge id {:?} was already used with a different plan",
11132                    input.purge_id
11133                ));
11134            }
11135            // A retry after a failed rewrite finishes it.
11136            self.erase_purged_pages()?;
11137            return Ok(LocalDataPurgeResult {
11138                already_applied: true,
11139                purged_rows: 0,
11140                dropped_commits: 0,
11141            });
11142        }
11143
11144        // RFC 0005 D7: the container file cannot be a purge target (its name is
11145        // not in the running schema), so purge drops it as a fixed step, before
11146        // any mirror row is touched and unconditionally.
11147        let _ = self.drop_previous_version();
11148
11149        let rollback = self.capture();
11150        self.begin_observation("syncular_local_purge")?;
11151        let applied = (|| -> Result<(ChangeAccumulator, LocalDataPurgeResult), String> {
11152            let row_ids = self.local_purge_base_row_ids(&targets)?;
11153            let retained_dropped = self.drop_failed_commits(|table, values| {
11154                targets.iter().any(|target| {
11155                    target.table == table && Self::local_purge_values_match(target, values)
11156                })
11157            })?;
11158            let doomed = self
11159                .outbox
11160                .iter()
11161                .filter(|commit| {
11162                    commit.ops.iter().any(|operation| {
11163                        let matching_targets = targets
11164                            .iter()
11165                            .filter(|target| target.table == operation.table)
11166                            .collect::<Vec<_>>();
11167                        if matching_targets.is_empty() {
11168                            return false;
11169                        }
11170                        if row_ids
11171                            .get(&operation.table)
11172                            .is_some_and(|ids| ids.contains(&operation.row_id))
11173                        {
11174                            return true;
11175                        }
11176                        operation.values.as_ref().is_some_and(|values| {
11177                            matching_targets
11178                                .iter()
11179                                .any(|target| Self::local_purge_values_match(target, values))
11180                        })
11181                    })
11182                })
11183                .cloned()
11184                .collect::<Vec<_>>();
11185            let doomed_ids = doomed
11186                .iter()
11187                .map(|commit| commit.client_commit_id.clone())
11188                .collect::<BTreeSet<_>>();
11189            let mut batch = ChangeAccumulator::default();
11190            for commit in &retained_dropped {
11191                batch.conflicts = true;
11192                batch.rejections = true;
11193                batch.outcomes = true;
11194                for operation in &commit.ops {
11195                    batch.table(&operation.table);
11196                }
11197            }
11198            let mut rejections = Vec::new();
11199            // §5.11 purge erasure: the journal keeps a dropped commit's identity and
11200            // codes, never its operations, so no purged value survives in it.
11201            for commit in &doomed {
11202                for operation in &commit.ops {
11203                    batch.table(&operation.table);
11204                    self.overlay_dirty.table(&operation.table);
11205                }
11206                let results = (0..commit.ops.len())
11207                    .map(|op_index| {
11208                        let rejection = RejectionRecord {
11209                            client_commit_id: commit.client_commit_id.clone(),
11210                            op_index: op_index as i32,
11211                            code: "client.local_data_purged".to_owned(),
11212                            message: "the commit was dropped by an application-authorized local data purge".to_owned(),
11213                            retryable: false,
11214                            details: None,
11215                            operation: None,
11216                        };
11217                        rejections.push(rejection.clone());
11218                        CommitOperationOutcome::Error { rejection }
11219                    })
11220                    .collect::<Vec<_>>();
11221                self.persist_commit_outcome(
11222                    &commit.client_commit_id,
11223                    CommitOutcomeStatus::Rejected,
11224                    &results,
11225                    None,
11226                )?;
11227                self.delete_outbox_persisted(&commit.client_commit_id)?;
11228            }
11229            if !doomed.is_empty() {
11230                self.outbox
11231                    .retain(|commit| !doomed_ids.contains(&commit.client_commit_id));
11232                self.rejections.extend(rejections);
11233                self.prune_commit_outcomes()?;
11234                batch.status = true;
11235                batch.rejections = true;
11236                batch.outcomes = true;
11237            }
11238
11239            let mut purged_rows = 0usize;
11240            for (table, ids) in &row_ids {
11241                if ids.is_empty() {
11242                    continue;
11243                }
11244                batch.table(table);
11245                self.overlay_dirty.table(table);
11246                for row_id in ids {
11247                    self.delete_base_row(table, row_id, i64::MAX)?;
11248                    purged_rows += 1;
11249                }
11250            }
11251            // §5.11 purge erasure: a journaled outcome with an operation or
11252            // conflict row the purge reaches keeps its identity, status, ids
11253            // and codes only.
11254            let touches = |table: &str, row_id: &str, values: Option<&Map<String, Value>>| {
11255                targets.iter().any(|target| {
11256                    target.table == table
11257                        && (row_ids.get(table).is_some_and(|ids| ids.contains(row_id))
11258                            || values.is_some_and(|values| {
11259                                Self::local_purge_values_match(target, values)
11260                            }))
11261                })
11262            };
11263            let entries = self
11264                .conn
11265                .prepare("SELECT client_commit_id, results_json, operations_json FROM _syncular_commit_outcomes")
11266                .and_then(|mut statement| {
11267                    statement
11268                        .query_map([], |row| {
11269                            Ok((
11270                                row.get::<_, String>(0)?,
11271                                row.get::<_, String>(1)?,
11272                                row.get::<_, Option<String>>(2)?,
11273                            ))
11274                        })?
11275                        .collect::<Result<Vec<_>, _>>()
11276                })
11277                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11278            for (client_commit_id, results_json, operations_json) in entries {
11279                let corrupt = |_| "sync.local_corrupt: persisted commit outcome is invalid".to_owned();
11280                let mut results: Vec<CommitOperationOutcome> =
11281                    serde_json::from_str(&results_json).map_err(corrupt)?;
11282                let operations: Vec<CommitOperation> = operations_json
11283                    .map(|json| serde_json::from_str(&json))
11284                    .transpose()
11285                    .map_err(corrupt)?
11286                    .unwrap_or_default();
11287                let named = operations
11288                    .iter()
11289                    .chain(results.iter().filter_map(|result| match result {
11290                        CommitOperationOutcome::Conflict { conflict } => conflict.operation.as_ref(),
11291                        CommitOperationOutcome::Error { rejection } => rejection.operation.as_ref(),
11292                        CommitOperationOutcome::Applied { .. } => None,
11293                    }))
11294                    .any(|operation| touches(&operation.table, &operation.row_id, operation.values.as_ref()))
11295                    || results.iter().any(|result| {
11296                        matches!(result, CommitOperationOutcome::Conflict { conflict }
11297                            if touches(&conflict.table, &conflict.row_id, Some(&conflict.server_row)))
11298                    });
11299                if !named {
11300                    continue;
11301                }
11302                for result in &mut results {
11303                    match result {
11304                        CommitOperationOutcome::Conflict { conflict } => {
11305                            conflict.server_row = Map::new();
11306                            conflict.conflict_columns.clear();
11307                            conflict.operation = None;
11308                        }
11309                        CommitOperationOutcome::Error { rejection } => {
11310                            rejection.details = None;
11311                            rejection.operation = None;
11312                        }
11313                        CommitOperationOutcome::Applied { .. } => {}
11314                    }
11315                }
11316                let results_json = serde_json::to_string(&results).map_err(|error| error.to_string())?;
11317                self.conn
11318                    .execute(
11319                        "UPDATE _syncular_commit_outcomes SET results_json = ?1, operations_json = NULL WHERE client_commit_id = ?2",
11320                        rusqlite::params![results_json, client_commit_id],
11321                    )
11322                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11323                batch.outcomes = true;
11324            }
11325            self.rebuild_overlay_if_dirty()?;
11326            self.delete_unreferenced_cached_blobs()?;
11327            self.conn
11328                .execute(
11329                    "INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
11330                    rusqlite::params![meta_key, canonical_plan],
11331                )
11332                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11333            Ok((
11334                batch,
11335                LocalDataPurgeResult {
11336                    already_applied: false,
11337                    purged_rows,
11338                    dropped_commits: doomed.len(),
11339                },
11340            ))
11341        })()
11342        .and_then(|(batch, result)| {
11343            self.finish_observation("syncular_local_purge", batch)
11344                .map(|()| result)
11345        });
11346        let result = match applied {
11347            Ok(result) => result,
11348            Err(error) => {
11349                self.rollback_observation("syncular_local_purge");
11350                self.restore(rollback);
11351                return Err(error);
11352            }
11353        };
11354        self.cancel_sync_round();
11355        self.erase_purged_pages()?;
11356        Ok(result)
11357    }
11358
11359    /// §5.11 purge erasure: SQLite leaves a deleted value in the freed pages of the
11360    /// database file, and the WAL keeps the frames written before the purge,
11361    /// until either is rewritten. VACUUM rebuilds the file from the remaining
11362    /// rows, and the checkpoint truncates the WAL.
11363    fn erase_purged_pages(&self) -> Result<(), String> {
11364        let erased = self
11365            .conn
11366            .execute_batch("PRAGMA secure_delete = ON; VACUUM;")
11367            .and_then(|()| {
11368                self.conn
11369                    .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))
11370            });
11371        let reset = self.conn.execute_batch("PRAGMA secure_delete = OFF");
11372        erased
11373            .and(reset)
11374            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
11375    }
11376
11377    /// Application-authorized recovery of the replicated projection. Unlike a
11378    /// local security purge, this retains the entire outbox, device identity,
11379    /// lease, outcomes, unit registrations, and protected bookkeeping.
11380    /// The projection reset, unit rewind, optimistic replay, and
11381    /// idempotency marker share one savepoint for interruption safety.
11382    pub fn rebootstrap_local_data(
11383        &mut self,
11384        input: &LocalDataRebootstrapInput,
11385    ) -> Result<LocalDataRebootstrapResult, String> {
11386        if input.rebootstrap_id.is_empty()
11387            || input.rebootstrap_id.len() > 128
11388            || !is_local_operation_code_like(&input.rebootstrap_id)
11389        {
11390            return Err(
11391                "sync.invalid_request: local rebootstrap rebootstrapId must be a 1–128 character code-like identifier"
11392                    .to_owned(),
11393            );
11394        }
11395        let meta_key = format!("localRebootstrap:{}", input.rebootstrap_id);
11396        if let Some(persisted) = meta_get(&self.conn, &meta_key).map_err(|_| {
11397            "sync.local_corrupt: persisted local rebootstrap receipt is unreadable".to_owned()
11398        })? {
11399            let (retained_commits, reset_subscriptions) =
11400                decode_local_rebootstrap_receipt(&persisted)?;
11401            return Ok(LocalDataRebootstrapResult {
11402                already_applied: true,
11403                retained_commits,
11404                reset_subscriptions,
11405            });
11406        }
11407        if self.stopped || self.schema_floor.is_some() {
11408            return Err(
11409                "sync.invalid_request: local rebootstrap cannot bypass an active schema-floor stop; update the application first"
11410                    .to_owned(),
11411            );
11412        }
11413
11414        let retained_commits = self.outbox.len();
11415        let reset_subscriptions = self.stored(load_units(&self.conn))?.len();
11416        let receipt = encode_local_rebootstrap_receipt(retained_commits, reset_subscriptions)?;
11417        let rollback = self.capture();
11418        self.begin_observation("syncular_local_rebootstrap")?;
11419        let mut batch = ChangeAccumulator::default();
11420        let applied = (|| -> Result<(), String> {
11421            self.run_schema_reset_observed(&mut batch, false)?;
11422            self.conn
11423                .execute(
11424                    "INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
11425                    rusqlite::params![meta_key, receipt],
11426                )
11427                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11428            self.sync_needed = true;
11429            self.sync_intent_queue.push_back(SyncIntent::Interactive);
11430            batch.status = true;
11431            Ok(())
11432        })()
11433        .and_then(|()| self.finish_observation("syncular_local_rebootstrap", batch));
11434        if let Err(error) = applied {
11435            self.rollback_observation("syncular_local_rebootstrap");
11436            self.restore(rollback);
11437            return Err(error);
11438        }
11439
11440        Ok(LocalDataRebootstrapResult {
11441            already_applied: false,
11442            retained_commits,
11443            reset_subscriptions,
11444        })
11445    }
11446
11447    // -- scope purge + doomed outbox (§3.3) ----------------------------------------
11448
11449    /// Delete base and visible rows owned by the effective scopes
11450    /// ([`local_scope_predicate`]); `Ok(false)` = no local scope-column
11451    /// mapping for a key (the fail-closed case). A SQLite failure is an error.
11452    fn purge_scope_rows(
11453        &mut self,
11454        table_name: &str,
11455        effective: &[(String, Vec<String>)],
11456        owners: &[Vec<(String, Vec<String>)>],
11457    ) -> Result<bool, String> {
11458        let Some(table) = self.schema.table(table_name) else {
11459            return Ok(false);
11460        };
11461        let Ok(predicate) = local_scope_predicate(table, effective, owners) else {
11462            return Ok(false);
11463        };
11464        let Some((predicate, params)) = predicate else {
11465            return Ok(true);
11466        };
11467        self.overlay_dirty.table(table_name);
11468        for source in [base_table(table_name), visible_table(table_name)] {
11469            self.conn
11470                .execute(
11471                    &format!("DELETE FROM {source} WHERE {predicate}"),
11472                    rusqlite::params_from_iter(&params),
11473                )
11474                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11475        }
11476        Ok(true)
11477    }
11478
11479    /// §3.3: drop pending commits whose upserts provably land in the
11480    /// revoked effective scopes and no other owner covers (row ownership)
11481    /// — whole-commit, never per-operation.
11482    fn drop_doomed_outbox(
11483        &mut self,
11484        table: &TableSchema,
11485        effective: &[(String, Vec<String>)],
11486        owners: &[Vec<(String, Vec<String>)>],
11487    ) -> Result<bool, String> {
11488        let doomed: Vec<_> =
11489            self.outbox
11490                .iter()
11491                .filter(|commit| {
11492                    commit.ops.iter().any(|op| {
11493                        op.upsert
11494                            && op.table == table.name
11495                            && op.values.as_ref().is_some_and(|values| {
11496                                owns_local_row(table, effective, owners, values)
11497                            })
11498                    })
11499                })
11500                .cloned()
11501                .collect();
11502        if doomed.is_empty() {
11503            return Ok(false);
11504        }
11505        let mut rejections = Vec::new();
11506        for commit in &doomed {
11507            let results = commit
11508                .ops
11509                .iter()
11510                .enumerate()
11511                .map(|(op_index, operation)| {
11512                    self.overlay_dirty.table(&operation.table);
11513                    let rejection = RejectionRecord {
11514                        client_commit_id: commit.client_commit_id.clone(),
11515                        op_index: op_index as i32,
11516                        code: "sync.scope_revoked".to_owned(),
11517                        message: "the commit was dropped because its effective scope was revoked"
11518                            .to_owned(),
11519                        retryable: false,
11520                        details: None,
11521                        operation: Some(CommitOperation::from(operation)),
11522                    };
11523                    rejections.push(rejection.clone());
11524                    CommitOperationOutcome::Error { rejection }
11525                })
11526                .collect::<Vec<_>>();
11527            self.persist_commit_outcome(
11528                &commit.client_commit_id,
11529                CommitOutcomeStatus::Rejected,
11530                &results,
11531                Some(&commit.ops),
11532            )?;
11533            self.delete_outbox_persisted(&commit.client_commit_id)?;
11534        }
11535        self.prune_commit_outcomes()?;
11536        let doomed_ids = doomed
11537            .iter()
11538            .map(|commit| commit.client_commit_id.as_str())
11539            .collect::<BTreeSet<_>>();
11540        self.outbox
11541            .retain(|commit| !doomed_ids.contains(commit.client_commit_id.as_str()));
11542        self.rejections.extend(rejections);
11543        Ok(true)
11544    }
11545
11546    // -- blobs (§5.9) ----------------------------------------------------------------
11547
11548    /// §5.9.7: hash bytes into the content address, cache them, queue the
11549    /// upload (flushed before the next push, B4). Returns the canonical
11550    /// BlobRef JSON `{blobId, byteLength, mediaType?}` for a `blob_ref`
11551    /// column value.
11552    pub fn upload_blob(
11553        &mut self,
11554        bytes: &[u8],
11555        media_type: Option<String>,
11556        name: Option<String>,
11557    ) -> Result<Value, String> {
11558        let blob_id = blob_id_for(bytes);
11559        let now = self.clock_now_ms();
11560        let tx = self
11561            .conn
11562            .transaction()
11563            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11564        tx.execute(
11565            "INSERT INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES (?,?,?,?,?)
11566             ON CONFLICT(blob_id) DO NOTHING",
11567            rusqlite::params![blob_id, bytes, bytes.len() as i64, media_type, now],
11568        )
11569        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11570        tx.execute(
11571            "INSERT OR IGNORE INTO _syncular_blob_uploads(blob_id, media_type, created_at_ms) VALUES (?,?,?)",
11572            rusqlite::params![blob_id, media_type, now],
11573        )
11574        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11575        tx.commit()
11576            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11577        self.enforce_blob_cache_cap();
11578        let mut obj = Map::new();
11579        obj.insert("blobId".to_owned(), Value::from(blob_id));
11580        obj.insert("byteLength".to_owned(), Value::from(bytes.len() as i64));
11581        if let Some(mt) = media_type {
11582            obj.insert("mediaType".to_owned(), Value::from(mt));
11583        }
11584        if let Some(n) = name {
11585            obj.insert("name".to_owned(), Value::from(n));
11586        }
11587        Ok(Value::Object(obj))
11588    }
11589
11590    /// §5.9.7: resolve an owned blob body without driver encoding. A
11591    /// content-addressed cache hit serves with no fetch (B1); a miss downloads
11592    /// (§5.9.5), verifies the address, caches the bytes, and returns them.
11593    pub fn fetch_blob_bytes(
11594        &mut self,
11595        transport: &mut dyn Transport,
11596        blob_id_or_ref: &str,
11597    ) -> Result<FetchedBlob, (String, String)> {
11598        let simple = |m: String| ("client.failed".to_owned(), m);
11599        // A BlobRef declares the size that caps the download (§5.9.5); a bare
11600        // blobId declares none.
11601        let (blob_id, byte_length) = if blob_id_or_ref.starts_with("sha256:") {
11602            (blob_id_or_ref.to_owned(), None)
11603        } else {
11604            let value: Value = serde_json::from_str(blob_id_or_ref)
11605                .map_err(|_| simple("blob ref is not JSON".to_owned()))?;
11606            (
11607                value
11608                    .get("blobId")
11609                    .and_then(Value::as_str)
11610                    .ok_or_else(|| simple("blob ref has no blobId".to_owned()))?
11611                    .to_owned(),
11612                value.get("byteLength").and_then(Value::as_u64),
11613            )
11614        };
11615        if let Some(cached) = self.get_cached_blob_bytes(&blob_id).map_err(simple)? {
11616            return Ok(cached);
11617        }
11618        if !self.transport_enabled {
11619            return Err(("sync.offline".into(), "transport gate is closed".into()));
11620        }
11621        // §5.9.5: propagate the server's blob.* code (blob.forbidden /
11622        // blob.not_found) verbatim so the harness can assert on it. The
11623        // authorized endpoint serves bytes inline OR (always-issue, presign
11624        // configured) a signed url the client fetches directly — no host auth,
11625        // no fall-through: failure => re-request (the caller's next fetch_blob_bytes).
11626        #[cfg(feature = "bench-internals")]
11627        let download_phase = self.benchmark_phases.start(Phase::BlobDownload);
11628        let bytes = match transport
11629            .blob_download(&blob_id, byte_length)
11630            .map_err(|e| (e.code, e.message))?
11631        {
11632            BlobDownload::Bytes(bytes) => bytes,
11633            BlobDownload::Url {
11634                url,
11635                url_expires_at_ms,
11636            } => {
11637                // §5.9.5: MUST NOT start a fetch at/past expiry.
11638                if url_expires_at_ms.is_some_and(|exp| exp <= self.clock_now_ms()) {
11639                    return Err((
11640                        "sync.segment_expired".to_owned(),
11641                        format!(
11642                            "blob url for {blob_id} expired before fetch — re-request mints a fresh url (§5.9.5)"
11643                        ),
11644                    ));
11645                }
11646                transport
11647                    .fetch_blob_url(&url, byte_length)
11648                    .map_err(|e| (e.code, e.message))?
11649            }
11650        };
11651        #[cfg(feature = "bench-internals")]
11652        drop(download_phase);
11653        #[cfg(feature = "bench-internals")]
11654        let validate_phase = self.benchmark_phases.start(Phase::BlobValidate);
11655        // §5.9.5 inherits §5.1: the declared size caps the body before the
11656        // content address is verified.
11657        if byte_length.is_some_and(|cap| bytes.len() as u64 > cap) {
11658            return Err((
11659                "sync.invalid_request".to_owned(),
11660                "blob exceeds its byteLength".to_owned(),
11661            ));
11662        }
11663        if blob_id_for(&bytes) != blob_id {
11664            return Err(simple(format!(
11665                "blob content address mismatch for {blob_id}"
11666            )));
11667        }
11668        #[cfg(feature = "bench-internals")]
11669        drop(validate_phase);
11670        #[cfg(feature = "bench-internals")]
11671        let insert_phase = self.benchmark_phases.start(Phase::BlobCacheInsert);
11672        let now = self.clock_now_ms();
11673        self.conn
11674            .execute(
11675                "INSERT OR IGNORE INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES (?,?,?,NULL,?)",
11676                rusqlite::params![blob_id, bytes, bytes.len() as i64, now],
11677            )
11678            .map_err(|e| simple(e.to_string()))?;
11679        self.conn
11680            .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))
11681            .map_err(|e| simple(e.to_string()))?;
11682        self.enforce_blob_cache_cap();
11683        #[cfg(feature = "bench-internals")]
11684        drop(insert_phase);
11685        self.get_cached_blob_bytes(&blob_id)
11686            .map_err(simple)?
11687            .ok_or_else(|| simple("blob cache write failed".to_owned()))
11688    }
11689
11690    fn get_cached_blob_bytes(&self, blob_id: &str) -> Result<Option<FetchedBlob>, String> {
11691        #[cfg(feature = "bench-internals")]
11692        let _phase = self.benchmark_phases.start(Phase::BlobCacheRead);
11693        let mut stmt = self
11694            .conn
11695            .prepare("SELECT bytes, byte_length, media_type FROM _syncular_blobs WHERE blob_id = ?")
11696            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11697        let mut rows = stmt
11698            .query(rusqlite::params![blob_id])
11699            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11700        if let Some(row) = rows
11701            .next()
11702            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
11703        {
11704            let bytes: Vec<u8> = row
11705                .get(0)
11706                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11707            let byte_length: i64 = row
11708                .get(1)
11709                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11710            let media_type: Option<String> = row
11711                .get(2)
11712                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11713            return Ok(Some(FetchedBlob {
11714                blob_id: blob_id.to_owned(),
11715                byte_length,
11716                bytes,
11717                media_type,
11718            }));
11719        }
11720        Ok(None)
11721    }
11722
11723    /// §5.9.7 B4: upload every queued blob before push.
11724    fn pending_blob_uploads(&self) -> Result<Vec<crate::round::PendingUpload>, TransportError> {
11725        let pending: Vec<(String, Option<String>)> = {
11726            let mut stmt = self
11727                .conn
11728                .prepare(
11729                    "SELECT blob_id, media_type FROM (
11730                      SELECT blob_id, media_type, created_at_ms FROM _syncular_blob_uploads
11731                      UNION ALL
11732                      SELECT DISTINCT r.blob_id, b.media_type, 9223372036854775807 AS created_at_ms
11733                      FROM _syncular_blob_commit_refs r LEFT JOIN _syncular_blobs b ON b.blob_id = r.blob_id
11734                      WHERE NOT EXISTS (SELECT 1 FROM _syncular_blob_uploads u WHERE u.blob_id = r.blob_id)
11735                    ) ORDER BY created_at_ms, blob_id",
11736                )
11737                .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
11738            let rows = stmt
11739                .query_map([], |row| {
11740                    Ok((row.get::<_, String>(0)?, row.get::<_, Option<String>>(1)?))
11741                })
11742                .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
11743            rows.collect::<Result<_, _>>()
11744                .map_err(|error| match error {
11745                    rusqlite::Error::InvalidColumnType(..)
11746                    | rusqlite::Error::FromSqlConversionFailure(..) => TransportError::new(
11747                        "sync.local_corrupt",
11748                        "Pending blob upload metadata is invalid",
11749                    ),
11750                    _ => TransportError::new("client.failed", error.to_string()),
11751                })?
11752        };
11753        let mut uploads = Vec::new();
11754        for (blob_id, media_type) in pending {
11755            let bytes = (|| {
11756                let stored: Option<(SqlValue, SqlValue)> = self
11757                    .conn
11758                    .query_row(
11759                        "SELECT bytes, byte_length FROM _syncular_blobs WHERE blob_id = ?",
11760                        rusqlite::params![blob_id],
11761                        |row| Ok((row.get(0)?, row.get(1)?)),
11762                    )
11763                    .optional()
11764                    .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
11765                let Some((SqlValue::Blob(bytes), SqlValue::Integer(byte_length))) = stored else {
11766                    return Err(TransportError::new(
11767                        "sync.local_corrupt",
11768                        "Pending blob upload body is missing or corrupt",
11769                    ));
11770                };
11771                if byte_length != bytes.len() as i64 || blob_id_for(&bytes) != blob_id {
11772                    return Err(TransportError::new(
11773                        "sync.local_corrupt",
11774                        "Pending blob upload body is missing or corrupt",
11775                    ));
11776                }
11777                Ok(bytes)
11778            })();
11779            let failed = bytes.is_err();
11780            uploads.push(crate::round::PendingUpload {
11781                id: blob_id,
11782                bytes,
11783                media_type,
11784            });
11785            if failed {
11786                break;
11787            }
11788        }
11789        Ok(uploads)
11790    }
11791
11792    fn visible_blob_ids_sql(&self) -> String {
11793        let mut selects = Vec::new();
11794        for table in &self.schema.tables {
11795            for column in table
11796                .columns
11797                .iter()
11798                .filter(|column| column.ty == ColumnType::BlobRef)
11799            {
11800                let identifier = quote_ident(&column.name);
11801                selects.push(format!(
11802                    "SELECT json_extract({identifier}, '$.blobId') AS blob_id FROM {} \
11803                     WHERE typeof({identifier}) = 'text' AND json_valid({identifier}) \
11804                     AND json_type({identifier}, '$.blobId') = 'text'",
11805                    quote_ident(&table.name),
11806                ));
11807            }
11808        }
11809        if selects.is_empty() {
11810            "SELECT NULL AS blob_id WHERE 0".to_owned()
11811        } else {
11812            selects.join(" UNION ")
11813        }
11814    }
11815
11816    /// §5.9.7 B1 size cap. Visible rows, pending commits, and
11817    /// staged uploads pin bodies directly; no derived refcount is stored.
11818    fn enforce_blob_cache_cap(&self) {
11819        let Some(max_bytes) = self.limits.blob_cache_max_bytes else {
11820            return;
11821        };
11822        let mut total: i64 = self
11823            .conn
11824            .query_row(
11825                "SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs",
11826                [],
11827                |row| row.get(0),
11828            )
11829            .unwrap_or(0);
11830        if total <= max_bytes {
11831            return;
11832        }
11833        let query = format!(
11834            "SELECT blob_id, byte_length FROM _syncular_blobs
11835             WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
11836               AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
11837               AND blob_id NOT IN ({})
11838             ORDER BY created_at_ms ASC, blob_id ASC",
11839            self.visible_blob_ids_sql(),
11840        );
11841        let candidates: Vec<(String, i64)> = {
11842            let Ok(mut stmt) = self.conn.prepare(&query) else {
11843                return;
11844            };
11845            let Ok(rows) = stmt.query_map([], |row| {
11846                Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?))
11847            }) else {
11848                return;
11849            };
11850            rows.filter_map(Result::ok).collect()
11851        };
11852        for (blob_id, byte_length) in candidates {
11853            if total <= max_bytes {
11854                break;
11855            }
11856            let _ = self.conn.execute(
11857                "DELETE FROM _syncular_blobs WHERE blob_id = ?",
11858                rusqlite::params![blob_id],
11859            );
11860            total -= byte_length;
11861        }
11862    }
11863
11864    fn delete_unreferenced_cached_blobs(&mut self) -> Result<(), String> {
11865        #[cfg(feature = "bench-internals")]
11866        let _phase = self.benchmark_phases.start(Phase::BlobRetention);
11867        if !self.schema_has_blobs() {
11868            return Ok(());
11869        }
11870        self.rebuild_overlay_if_dirty()?;
11871        self.conn
11872            .execute(
11873                &format!(
11874                    "DELETE FROM _syncular_blobs
11875                 WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
11876                   AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
11877                   AND blob_id NOT IN ({})",
11878                    self.visible_blob_ids_sql(),
11879                ),
11880                [],
11881            )
11882            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11883        Ok(())
11884    }
11885
11886    // -- local row storage ----------------------------------------------------------
11887
11888    /// The cached per-table primary-key upsert SQL for `full_table` (see
11889    /// the `insert_sql` field: built once, reused per row).
11890    fn insert_row_sql(&self, full_table: &str, table: &crate::schema::TableSchema) -> String {
11891        if let Some(sql) = self.insert_sql.borrow().get(full_table) {
11892            return sql.clone();
11893        }
11894        let mut columns: Vec<String> = table.columns.iter().map(|c| quote_ident(&c.name)).collect();
11895        columns.push(quote_ident("_syncular_version"));
11896        columns.extend(table.sealed_columns().iter().map(|name| quote_ident(name)));
11897        let placeholders: Vec<&str> = columns.iter().map(|_| "?").collect();
11898        let primary_key = quote_ident(&table.primary_key);
11899        let updates = columns
11900            .iter()
11901            .filter(|column| **column != primary_key)
11902            .map(|column| format!("{column}=excluded.{column}"))
11903            .collect::<Vec<_>>()
11904            .join(", ");
11905        // §4.5 version guard: server state lands in the base table only above
11906        // the stored version; the visible overlay takes every write.
11907        let guard = if full_table == base_table(&table.name) {
11908            format!(" WHERE excluded._syncular_version > {full_table}._syncular_version")
11909        } else {
11910            String::new()
11911        };
11912        let sql = format!(
11913            "INSERT INTO {full_table} ({}) VALUES ({}) ON CONFLICT ({primary_key}) DO UPDATE SET {updates}{guard}",
11914            columns.join(", "),
11915            placeholders.join(", ")
11916        );
11917        self.insert_sql
11918            .borrow_mut()
11919            .insert(full_table.to_owned(), sql.clone());
11920        sql
11921    }
11922
11923    /// §4.5: write a server row under the version guard; returns whether it
11924    /// landed. `sealed` is its §5.11 sealed state, `None` for an unsealed row.
11925    fn write_base_row(
11926        &self,
11927        table_name: &str,
11928        row: &Row,
11929        version: i64,
11930        sealed: Option<&SealedRow>,
11931    ) -> Result<bool, String> {
11932        let landed = self.write_row(&base_table(table_name), table_name, row, version, sealed)?;
11933        if landed {
11934            self.overlay_dirty.table(table_name);
11935        }
11936        Ok(landed)
11937    }
11938
11939    fn write_row(
11940        &self,
11941        full_table: &str,
11942        table_name: &str,
11943        row: &Row,
11944        version: i64,
11945        sealed: Option<&SealedRow>,
11946    ) -> Result<bool, String> {
11947        #[cfg(feature = "bench-internals")]
11948        let _phase = self.benchmark_phases.start(Phase::RowWrite);
11949        let table = self
11950            .schema
11951            .table(table_name)
11952            .ok_or_else(|| format!("unknown table {table_name:?}"))?;
11953        let sql = self.insert_row_sql(full_table, table);
11954        let mut stmt = self
11955            .conn
11956            .prepare_cached(&sql)
11957            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11958        let params = row
11959            .iter()
11960            .map(RowParam::Cell)
11961            .chain(std::iter::once(RowParam::Version(version)))
11962            .chain(
11963                [
11964                    sealed.map(|sealed| sealed.key_id.as_str()),
11965                    sealed.map(|sealed| sealed.envelopes.as_str()),
11966                ]
11967                .into_iter()
11968                .take(table.sealed_columns().len())
11969                .map(RowParam::Text),
11970            );
11971        stmt.execute(rusqlite::params_from_iter(params))
11972            .map(|changed| changed > 0)
11973            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
11974    }
11975
11976    /// §4.5: delete a server row whose stored version lies below `version`;
11977    /// returns whether a row left.
11978    fn delete_base_row(
11979        &self,
11980        table_name: &str,
11981        row_id: &str,
11982        version: i64,
11983    ) -> Result<bool, String> {
11984        let table = self
11985            .schema
11986            .table(table_name)
11987            .ok_or_else(|| format!("unknown table {table_name:?}"))?;
11988        let sql = format!(
11989            "DELETE FROM {} WHERE {} AND _syncular_version < ?2",
11990            base_table(table_name),
11991            row_id_predicate(table)
11992        );
11993        let mut stmt = self
11994            .conn
11995            .prepare_cached(&sql)
11996            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11997        let landed = stmt
11998            .execute(rusqlite::params![row_id, version])
11999            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
12000            > 0;
12001        if landed {
12002            self.overlay_dirty.table(table_name);
12003        }
12004        Ok(landed)
12005    }
12006
12007    fn has_acknowledged_rows(&self) -> Result<bool, String> {
12008        self.conn
12009            .query_row(
12010                "SELECT EXISTS(SELECT 1 FROM _syncular_acked_rows)",
12011                [],
12012                |row| row.get(0),
12013            )
12014            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
12015    }
12016
12017    fn acknowledged_operations(
12018        &self,
12019        rows: Option<&HashSet<(&str, &str)>>,
12020        tables: Option<&BTreeSet<&str>>,
12021    ) -> Result<Vec<OutboxOp>, String> {
12022        let keys = rows.map(|rows| serde_json::to_string(&rows).expect("row keys"));
12023        let mut statement = self
12024            .conn
12025            .prepare_cached(if keys.is_some() {
12026                ACKED_ROWS_BY_ROW
12027            } else {
12028                "SELECT op_json,intent_json FROM _syncular_acked_rows WHERE ?1 IS NULL AND (?2 IS NULL OR tbl IN(SELECT value FROM json_each(?2))) ORDER BY commit_seq,idx"
12029            })
12030            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12031        let rows = statement
12032            .query_map(
12033                rusqlite::params![
12034                    keys,
12035                    tables.map(|tables| serde_json::to_string(tables).expect("table keys"))
12036                ],
12037                |row| {
12038                    #[cfg(test)]
12039                    self.acknowledged_replay_count
12040                        .set(self.acknowledged_replay_count.get() + 1);
12041                    Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
12042                },
12043            )
12044            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12045        rows.map(|row| {
12046            let (op, intent) =
12047                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12048            let op: CommitOperation =
12049                serde_json::from_str(&op).map_err(|_| "sync.local_corrupt".to_owned())?;
12050            let table = self
12051                .schema
12052                .table(&op.table)
12053                .ok_or_else(|| "sync.unknown_table".to_owned())?;
12054            let values = if op.op == "upsert" && self.visible_row(table, &op.row_id)?.is_none() {
12055                Some(serde_json::from_str(&intent).map_err(|_| "sync.local_corrupt".to_owned())?)
12056            } else {
12057                op.values.clone()
12058            };
12059            Ok(OutboxOp {
12060                values,
12061                ..op.into()
12062            })
12063        })
12064        .collect()
12065    }
12066
12067    fn drop_acknowledged_rows(
12068        &self,
12069        matches: impl Fn(&str, &Map<String, Value>) -> bool,
12070    ) -> Result<Vec<OutboxCommit>, String> {
12071        let mut statement = self
12072            .conn
12073            .prepare("SELECT commit_id,tbl,intent_json,id FROM _syncular_acked_rows")
12074            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12075        let rows = statement
12076            .query_map([], |row| {
12077                Ok((
12078                    row.get::<_, String>(0)?,
12079                    row.get::<_, String>(1)?,
12080                    row.get::<_, String>(2)?,
12081                    row.get::<_, String>(3)?,
12082                ))
12083            })
12084            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12085        let mut doomed = HashSet::new();
12086        for row in rows {
12087            let (id, table, intent, row_id) =
12088                row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12089            let values =
12090                serde_json::from_str(&intent).map_err(|_| "sync.local_corrupt".to_owned())?;
12091            let base_matches = self
12092                .schema
12093                .table(&table)
12094                .map(|schema| self.stored_row(schema, &row_id, true))
12095                .transpose()?
12096                .flatten()
12097                .is_some_and(|(base, _)| {
12098                    let schema = self.schema.table(&table).expect("acknowledged table");
12099                    let values = schema
12100                        .columns
12101                        .iter()
12102                        .zip(base.iter())
12103                        .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
12104                        .collect();
12105                    matches(&table, &values)
12106                });
12107            if matches(&table, &values) || base_matches {
12108                doomed.insert(id);
12109            }
12110        }
12111        let mut dropped = Vec::new();
12112        for id in doomed {
12113            let mut statement = self
12114                .conn
12115                .prepare("SELECT op_json FROM _syncular_acked_rows WHERE commit_id=?1 ORDER BY idx")
12116                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12117            let operations = statement
12118                .query_map([&id], |row| row.get::<_, String>(0))
12119                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12120            let mut ops = Vec::new();
12121            for operation in operations {
12122                let json = operation
12123                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12124                let op: CommitOperation =
12125                    serde_json::from_str(&json).map_err(|_| "sync.local_corrupt".to_owned())?;
12126                self.overlay_dirty.table(&op.table);
12127                ops.push(OutboxOp {
12128                    table: op.table,
12129                    row_id: op.row_id,
12130                    upsert: op.op == "upsert",
12131                    base_version: op.base_version,
12132                    values: op.values,
12133                    insert: op.insert,
12134                });
12135            }
12136            dropped.push(OutboxCommit {
12137                client_commit_id: id.clone(),
12138                ops,
12139            });
12140            self.conn
12141                .execute("DELETE FROM _syncular_acked_rows WHERE commit_id=?1", [id])
12142                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12143        }
12144        Ok(dropped)
12145    }
12146
12147    /// [`Self::rebuild_overlay`], skipped when neither the base tables nor
12148    /// the outbox changed since the last rebuild (the no-op sync round).
12149    fn rebuild_overlay_if_dirty(&mut self) -> Result<(), String> {
12150        if self.overlay_dirty.get() {
12151            self.rebuild_overlay()?;
12152        }
12153        Ok(())
12154    }
12155
12156    /// §7.1: local reads see outbox state applied optimistically — rebuild
12157    /// every visible table as (base server state) + (pending outbox replay
12158    /// on top). Optimistic rows carry version `-1`.
12159    fn rebuild_overlay(&mut self) -> Result<(), String> {
12160        #[cfg(feature = "bench-internals")]
12161        let _phase = self.benchmark_phases.start(Phase::OverlayRebuild);
12162        #[cfg(test)]
12163        self.overlay_rebuild_count
12164            .set(self.overlay_rebuild_count.get() + 1);
12165        let dirty = self.overlay_dirty.snapshot();
12166        let tables: Vec<_> = self
12167            .schema
12168            .tables
12169            .iter()
12170            .filter(|table| {
12171                dirty
12172                    .as_ref()
12173                    .is_none_or(|names| names.is_empty() || names.contains(&table.name))
12174            })
12175            .cloned()
12176            .collect();
12177        let names: BTreeSet<_> = tables.iter().map(|table| table.name.as_str()).collect();
12178        self.begin_observation("syncular_overlay")?;
12179        let rebuilt = (|| {
12180            for table in &tables {
12181                for index in &table.fts_indexes {
12182                    self.drop_fts_triggers(index)?;
12183                }
12184                let visible = visible_table(&table.name);
12185                let base = base_table(&table.name);
12186                self.conn
12187                    .execute_batch(&format!(
12188                        "DELETE FROM {visible}; INSERT INTO {visible} SELECT * FROM {base}"
12189                    ))
12190                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12191            }
12192            self.replay_overlay(&names, None)?;
12193            for table in &tables {
12194                for index in &table.fts_indexes {
12195                    self.rebuild_fts_projection(table, index)?;
12196                    self.create_fts_triggers(table, index)?;
12197                }
12198            }
12199            self.conn
12200                .execute_batch("RELEASE syncular_overlay")
12201                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12202            Ok(())
12203        })();
12204        if rebuilt.is_err() {
12205            self.conn
12206                .execute_batch("ROLLBACK TO syncular_overlay; RELEASE syncular_overlay")
12207                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12208        } else {
12209            self.overlay_dirty.set(false);
12210        }
12211        rebuilt
12212    }
12213
12214    /// §7.1 replay of acknowledged, retained failed and pending intent, in
12215    /// that order, for the operations of `tables` on `rows` (every row for
12216    /// `None`) over their current visible state.
12217    fn replay_overlay(
12218        &self,
12219        tables: &BTreeSet<&str>,
12220        rows: Option<&HashSet<(&str, &str)>>,
12221    ) -> Result<(), String> {
12222        let replays = |op: &&OutboxOp| {
12223            tables.contains(op.table.as_str())
12224                && rows.is_none_or(|rows| rows.contains(&(op.table.as_str(), op.row_id.as_str())))
12225        };
12226        self.apply_outbox_ops(&self.acknowledged_operations(rows, Some(tables))?)?;
12227        for (commit, initial) in self.failed_commits.iter() {
12228            for operation in commit.ops.iter().filter(replays) {
12229                let Some(table) = self.schema.table(&operation.table) else {
12230                    continue;
12231                };
12232                if operation.upsert
12233                    && self.visible_row(table, &operation.row_id)?.is_none()
12234                    && operation.presented_values(table).is_some_and(|values| {
12235                        table
12236                            .columns
12237                            .iter()
12238                            .all(|column| values.contains_key(&column.name))
12239                    })
12240                {
12241                    let key = serde_json::to_string(&(&operation.table, &operation.row_id))
12242                        .expect("row key serialization");
12243                    if let Some(Value::Object(values)) = initial.get(&key) {
12244                        self.apply_outbox_ops(&[OutboxOp {
12245                            values: Some(values.clone()),
12246                            ..operation.clone()
12247                        }])?;
12248                    }
12249                }
12250            }
12251            self.apply_outbox_ops(commit.ops.iter().filter(replays))?;
12252        }
12253        self.apply_outbox_ops(
12254            self.outbox
12255                .iter()
12256                .flat_map(|commit| &commit.ops)
12257                .filter(replays),
12258        )
12259    }
12260
12261    /// §7.1 replay: every operation is applied over the current visible
12262    /// state. An operation that no longer applies (a server row now holds its
12263    /// unique value) stays pending and invisible until a later replay admits
12264    /// it or the server answers its push.
12265    fn apply_outbox_ops<'a>(
12266        &self,
12267        ops: impl IntoIterator<Item = &'a OutboxOp>,
12268    ) -> Result<(), String> {
12269        #[cfg(feature = "bench-internals")]
12270        let mut phase = self.benchmark_phases.start(Phase::PendingReplay);
12271        for op in ops {
12272            #[cfg(feature = "bench-internals")]
12273            if let Some(phase) = &mut phase {
12274                phase.unit();
12275            }
12276            if op.upsert {
12277                if let Some((table, values)) = self
12278                    .schema
12279                    .table(&op.table)
12280                    .filter(|table| table.indexes.iter().any(|index| index.unique))
12281                    .and_then(|table| Some((table, op.presented_values(table)?)))
12282                {
12283                    let mut intended: Map<String, Value> = self
12284                        .visible_row(table, &op.row_id)?
12285                        .map(|(row, _)| {
12286                            table
12287                                .columns
12288                                .iter()
12289                                .zip(&row)
12290                                .map(|(column, value)| {
12291                                    (column.name.clone(), column_value_to_json(value))
12292                                })
12293                                .collect()
12294                        })
12295                        .unwrap_or_default();
12296                    intended.extend(values.into_owned());
12297                    if !self.unique_conflicts(table, &intended, false)?.is_empty() {
12298                        continue;
12299                    }
12300                }
12301            }
12302            self.apply_outbox_op(op)?;
12303        }
12304        Ok(())
12305    }
12306
12307    fn apply_outbox_op(&self, op: &OutboxOp) -> Result<(), String> {
12308        let Some(table) = self.schema.table(&op.table) else {
12309            return Ok(());
12310        };
12311        let visible = visible_table(&table.name);
12312        if !op.upsert {
12313            let sql = format!("DELETE FROM {visible} WHERE {}", row_id_predicate(table));
12314            self.conn
12315                .prepare_cached(&sql)
12316                .and_then(|mut statement| statement.execute(rusqlite::params![op.row_id]))
12317                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12318            return Ok(());
12319        }
12320        // §7.1: an insert-intent operation that lacks a non-nullable column
12321        // cannot encode; it stays invisible until the send-time prepass drops
12322        // it as sync.outbox_incompatible (§7.4.4).
12323        let Some(values) = op.presented_values(table) else {
12324            return Ok(());
12325        };
12326        // §7.1: the overlay applies the operation's PRESENT columns
12327        // over the current local row. A partial operation over an
12328        // absent local row leaves it absent — the server answers it
12329        // with sync.row_deleted or sync.row_missing (§6.2). An
12330        // insert-intent operation presents every column.
12331        let full = table
12332            .columns
12333            .iter()
12334            .all(|column| values.contains_key(&column.name));
12335        let local = self.visible_row(table, &op.row_id)?;
12336        if local.is_none() && !full {
12337            return Ok(());
12338        }
12339        let mut row: Row = Vec::with_capacity(table.columns.len());
12340        for (index, column) in table.columns.iter().enumerate() {
12341            let incoming = match values.get(&column.name) {
12342                Some(raw) => json_to_column_value(
12343                    if raw.get("$bytes").is_some()
12344                        && table
12345                            .encrypted_columns
12346                            .iter()
12347                            .any(|encrypted| encrypted.index == index)
12348                    {
12349                        &table.wire_columns[index]
12350                    } else {
12351                        column
12352                    },
12353                    Some(raw),
12354                ),
12355                None => Ok(local
12356                    .as_ref()
12357                    .and_then(|(values, _)| values.get(index).cloned())
12358                    .flatten()),
12359            };
12360            row.push(incoming.map_err(|_| {
12361                "sync.local_corrupt: persisted optimistic row value is invalid".to_owned()
12362            })?);
12363        }
12364        let version = local.as_ref().map_or(-1, |(_, version)| *version);
12365        self.write_row(&visible, &table.name, &row, version, None)
12366            .map(|_| ())
12367            .map_err(|error| {
12368                if error.starts_with("UNIQUE constraint failed:") {
12369                    "sync.constraint_violation: local write violates a unique constraint".to_owned()
12370                } else {
12371                    error
12372                }
12373            })
12374    }
12375
12376    /// Whether the row is present in the overlay. Existence reads only the
12377    /// key: locked encrypted values stay opaque, and decoding the complete row
12378    /// would reject a valid plain write.
12379    fn visible_row_exists(&self, table: &TableSchema, row_id: &str) -> Result<bool, String> {
12380        self.conn
12381            .query_row(
12382                &format!(
12383                    "SELECT EXISTS(SELECT 1 FROM {} WHERE {})",
12384                    visible_table(&table.name),
12385                    row_id_predicate(table)
12386                ),
12387                rusqlite::params![row_id],
12388                |row| row.get(0),
12389            )
12390            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
12391    }
12392
12393    /// §5.11: a sealed row is read-only; a write could otherwise send NULL
12394    /// over ciphertext this device cannot see.
12395    fn refuse_sealed_row(&self, table: &TableSchema, row_id: &str) -> Result<(), ClientError> {
12396        if !table.has_encrypted_columns() {
12397            return Ok(());
12398        }
12399        let key_id: Option<String> = self
12400            .conn
12401            .query_row(
12402                &format!(
12403                    "SELECT \"_sync_sealed_key\" FROM {} WHERE {}",
12404                    visible_table(&table.name),
12405                    row_id_predicate(table)
12406                ),
12407                rusqlite::params![row_id],
12408                |row| row.get(0),
12409            )
12410            .optional()
12411            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
12412            .flatten();
12413        match key_id {
12414            None => Ok(()),
12415            Some(key_id) => Err(ClientError {
12416                code: "client.encryption_key_missing".to_owned(),
12417                message: "the row is sealed under a key this device does not hold".to_owned(),
12418                details: Some(json!({ "table": table.name, "keyId": key_id })),
12419                retryable: false,
12420            }),
12421        }
12422    }
12423
12424    /// §5.11 reveal: decrypt every sealed base row whose envelope keys the
12425    /// keyring now holds into its columns and clear both reserved columns,
12426    /// one change batch per table. `client.decrypt_failed` fails the caller.
12427    fn reveal_sealed_rows(&mut self) -> Result<(), TransportError> {
12428        let storage = |message: String| TransportError::new("storage.failed", message);
12429        let tables: Vec<TableSchema> = self
12430            .schema
12431            .tables
12432            .iter()
12433            .filter(|table| table.has_encrypted_columns())
12434            .cloned()
12435            .collect();
12436        for table in &tables {
12437            let base = base_table(&table.name);
12438            let pk = quote_ident(&table.primary_key);
12439            let sealed: Vec<(SqlValue, i64, String)> = self
12440                .conn
12441                .prepare_cached(&format!(
12442                    "SELECT {pk}, \"_syncular_version\", \"_sync_sealed\" FROM {base} WHERE \"_sync_sealed_key\" IS NOT NULL"
12443                ))
12444                .and_then(|mut statement| {
12445                    statement
12446                        .query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))?
12447                        .collect()
12448                })
12449                .map_err(|error| storage(Self::sqlite_failure(&self.storage_failure, error)))?;
12450            let mut revealed = Vec::new();
12451            for (key, version, envelopes) in sealed {
12452                if let Some(row) =
12453                    crate::values::reveal_sealed_row(table, &envelopes, &self.encryption)?
12454                {
12455                    revealed.push((key, version, row));
12456                }
12457            }
12458            if revealed.is_empty() {
12459                continue;
12460            }
12461            self.begin_observation("syncular_reveal").map_err(storage)?;
12462            let outcome = (|| {
12463                let assignments = table
12464                    .encrypted_columns
12465                    .iter()
12466                    .map(|enc| format!("{}=?", quote_ident(&table.columns[enc.index].name)))
12467                    .collect::<Vec<_>>()
12468                    .join(",");
12469                let mut statement = self
12470                    .conn
12471                    .prepare_cached(&format!(
12472                        "UPDATE {base} SET {assignments},\"_sync_sealed_key\"=NULL,\"_sync_sealed\"=NULL WHERE {pk}=? AND \"_syncular_version\"=?"
12473                    ))
12474                    .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12475                for (key, version, row) in &revealed {
12476                    let mut params = table
12477                        .encrypted_columns
12478                        .iter()
12479                        .map(|enc| owned_sql_value(RowParam::Cell(&row[enc.index])))
12480                        .collect::<Result<Vec<_>, _>>()?;
12481                    params.push(key.clone());
12482                    params.push(SqlValue::Integer(*version));
12483                    statement
12484                        .execute(rusqlite::params_from_iter(params))
12485                        .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12486                }
12487                drop(statement);
12488                self.overlay_dirty.table(&table.name);
12489                self.rebuild_overlay()?;
12490                let mut batch = ChangeAccumulator::default();
12491                batch.table(&table.name);
12492                self.finish_observation("syncular_reveal", batch)
12493            })();
12494            if outcome.is_err() {
12495                self.rollback_observation("syncular_reveal");
12496            }
12497            outcome.map_err(storage)?;
12498        }
12499        Ok(())
12500    }
12501
12502    /// The visible (optimistic) row's values plus version, or `None` when the
12503    /// row is absent from the overlay.
12504    fn visible_row(&self, table: &TableSchema, row_id: &str) -> Result<Option<(Row, i64)>, String> {
12505        self.stored_row(table, row_id, false)
12506    }
12507
12508    fn stored_row(
12509        &self,
12510        table: &TableSchema,
12511        row_id: &str,
12512        base: bool,
12513    ) -> Result<Option<(Row, i64)>, String> {
12514        let sql = format!(
12515            "SELECT * FROM {} WHERE {}",
12516            if base {
12517                base_table(&table.name)
12518            } else {
12519                visible_table(&table.name)
12520            },
12521            row_id_predicate(table)
12522        );
12523        let mut stmt = self
12524            .conn
12525            .prepare_cached(&sql)
12526            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12527        let mut rows = stmt
12528            .query(rusqlite::params![row_id])
12529            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12530        let Some(row) = rows
12531            .next()
12532            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
12533        else {
12534            return Ok(None);
12535        };
12536        let mut values = Vec::with_capacity(table.columns.len());
12537        for (index, column) in table.columns.iter().enumerate() {
12538            let raw = row
12539                .get_ref(index)
12540                .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12541            if let rusqlite::types::ValueRef::Blob(bytes) = raw {
12542                if table
12543                    .encrypted_columns
12544                    .iter()
12545                    .any(|encrypted| encrypted.index == index)
12546                {
12547                    values.push(Some(ColumnValue::Bytes(bytes.to_vec())));
12548                    continue;
12549                }
12550            }
12551            let json = sql_ref_to_json(column, raw);
12552            values.push(json_to_column_value(column, Some(&json)).map_err(|_| {
12553                "sync.local_corrupt: persisted local row value is invalid".to_owned()
12554            })?);
12555        }
12556        let version: i64 = row
12557            .get(table.columns.len())
12558            .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12559        Ok(Some((values, version)))
12560    }
12561
12562    // -- realtime (§8) ---------------------------------------------------------------
12563
12564    /// §8.8: the configured realtime policy.
12565    #[must_use]
12566    pub fn realtime_policy(&self) -> RealtimePolicy {
12567        self.realtime_policy
12568    }
12569
12570    /// §8.8: the explicit realtime availability state.
12571    #[must_use]
12572    pub fn realtime_state(&self) -> RealtimeState {
12573        self.realtime_state
12574    }
12575
12576    /// §8.8: set the realtime policy. Refused while the socket is connected:
12577    /// a policy change never silently re-routes an ownership state.
12578    pub fn set_realtime_policy(&mut self, policy: RealtimePolicy) -> Result<(), String> {
12579        if self.realtime_state == RealtimeState::Connected && policy != self.realtime_policy {
12580            return Err(
12581                "sync.invalid_request: disconnect realtime before changing the realtime policy"
12582                    .to_owned(),
12583            );
12584        }
12585        self.realtime_policy = policy;
12586        match policy {
12587            RealtimePolicy::Off => self.set_realtime_state(RealtimeState::Disabled, None),
12588            RealtimePolicy::Required | RealtimePolicy::Optional
12589                if self.realtime_state == RealtimeState::Disabled =>
12590            {
12591                self.set_realtime_state(RealtimeState::Disconnected, None);
12592            }
12593            RealtimePolicy::Required | RealtimePolicy::Optional => {}
12594        }
12595        Ok(())
12596    }
12597
12598    /// Record an availability transition, keeping the reason code bounded and
12599    /// code-like (§7.6) and clearing it where it cannot apply.
12600    fn set_realtime_state(&mut self, state: RealtimeState, reason_code: Option<&str>) {
12601        self.realtime_state = state;
12602        self.realtime_reason_code = reason_code.map(Self::diagnostic_code);
12603        if matches!(state, RealtimeState::Connected | RealtimeState::Disabled) {
12604            self.realtime_reason_code = None;
12605            self.realtime_retry_delay_ms = None;
12606        }
12607    }
12608
12609    #[must_use]
12610    fn realtime_connected(&self) -> bool {
12611        self.realtime_state == RealtimeState::Connected
12612    }
12613
12614    pub fn connect_realtime(&mut self, transport: &mut dyn Transport) -> Result<(), String> {
12615        if !self.transport_enabled {
12616            return Err("sync.offline: transport gate is closed".into());
12617        }
12618        if self.realtime_policy == RealtimePolicy::Off {
12619            return Err(
12620                "sync.invalid_request: realtime is off; realtime connect is refused".to_owned(),
12621            );
12622        }
12623        if self.realtime_connected() {
12624            return Ok(());
12625        }
12626        self.realtime_state = RealtimeState::Connecting;
12627        match transport.realtime_connect_for_client(&self.client_id) {
12628            Ok(()) => {
12629                self.set_realtime_state(RealtimeState::Connected, None);
12630                Ok(())
12631            }
12632            Err(error) => {
12633                self.set_realtime_state(RealtimeState::Refused, Some(&error.code));
12634                Err(format!("{}: {}", error.code, error.message))
12635            }
12636        }
12637    }
12638
12639    pub fn disconnect_realtime(&mut self, transport: &mut dyn Transport) {
12640        self.cancel_sync_round();
12641        // A deliberately disconnected or policy-disabled client has nothing
12642        // to release; a `lost` socket still does (the host owns the handle).
12643        if self.realtime_state == RealtimeState::Disconnected
12644            || self.realtime_state == RealtimeState::Disabled
12645        {
12646            return;
12647        }
12648        let _ = transport.realtime_close();
12649        self.set_realtime_state(
12650            if self.realtime_policy == RealtimePolicy::Off {
12651                RealtimeState::Disabled
12652            } else {
12653                RealtimeState::Disconnected
12654            },
12655            None,
12656        );
12657        self.presence.clear(); // §8.6.1: presence is per-connection
12658    }
12659
12660    /// §8.8: the host observed the socket end without disconnecting it (a
12661    /// peer close, an I/O error, a failed send; the native transport reports
12662    /// these as `Inbound::Disconnected`). Releases the transport, moves a
12663    /// `Connected` state to `Lost` with the transport's code as the reason, and schedules
12664    /// one background retry intent so the host reconnects and catches up.
12665    pub fn realtime_lost(&mut self, transport: &mut dyn Transport, error: &TransportError) {
12666        if self.realtime_state != RealtimeState::Connected {
12667            return;
12668        }
12669        self.cancel_sync_round();
12670        let _ = transport.realtime_close();
12671        self.set_realtime_state(RealtimeState::Lost, Some(&error.code));
12672        self.realtime_retry_delay_ms = Some(self.schedule_background_retry());
12673        self.presence.clear();
12674    }
12675
12676    /// §8.6.2: publish (or clear, `doc: None`) this client's presence
12677    /// document for `scope_key`. Requires a live socket; the document is
12678    /// ephemeral (lost on disconnect). Authorization is the connection's
12679    /// registration (§8.6.3) — an unheld key is rejected loudly by the
12680    /// server with `presence.forbidden`.
12681    pub fn set_presence(
12682        &mut self,
12683        transport: &mut dyn Transport,
12684        scope_key: &str,
12685        doc: Option<&Value>,
12686    ) -> Result<(), String> {
12687        if !self.transport_enabled {
12688            return Err("sync.offline: transport gate is closed".into());
12689        }
12690        if !self.realtime_connected() {
12691            return Err("setPresence requires a connected realtime socket (§8.6)".to_string());
12692        }
12693        let text = encode_presence_publish(scope_key, doc);
12694        transport
12695            .realtime_send(&text)
12696            .map_err(|e| format!("{}: {}", e.code, e.message))
12697    }
12698
12699    /// §8.6: the peers currently present on a scope key (ephemeral).
12700    pub fn presence(&self, scope_key: &str) -> Vec<PresencePeer> {
12701        self.presence
12702            .get(scope_key)
12703            .map(|peers| peers.values().cloned().collect())
12704            .unwrap_or_default()
12705    }
12706
12707    /// §8.6 apply an inbound presence fanout to the local map.
12708    fn apply_presence(
12709        &mut self,
12710        scope_key: String,
12711        kind: Option<PresenceKind>,
12712        actor_id: Option<String>,
12713        client_id: Option<String>,
12714        doc: Option<Value>,
12715        error: Option<String>,
12716    ) {
12717        // A refused publish (§8.6.2) changes no peer; the native host surfaces
12718        // it as a `presence` event carrying the server's `error` code.
12719        if error.is_some() {
12720            return;
12721        }
12722        let (Some(kind), Some(actor_id), Some(client_id)) = (kind, actor_id, client_id) else {
12723            return;
12724        };
12725        let peer_key = format!("{actor_id} {client_id}");
12726        match kind {
12727            PresenceKind::Leave => {
12728                if let Some(peers) = self.presence.get_mut(&scope_key) {
12729                    peers.remove(&peer_key);
12730                    if peers.is_empty() {
12731                        self.presence.remove(&scope_key);
12732                    }
12733                }
12734            }
12735            _ => {
12736                let doc = match doc {
12737                    Some(Value::Object(_)) => doc.unwrap(),
12738                    _ => return,
12739                };
12740                self.presence.entry(scope_key).or_default().insert(
12741                    peer_key,
12742                    PresencePeer {
12743                        actor_id,
12744                        client_id,
12745                        doc,
12746                    },
12747                );
12748            }
12749        }
12750    }
12751
12752    /// Inbound JSON control message (§8.1). Unknown events are tolerated.
12753    pub fn on_realtime_text(&mut self, text: &str) {
12754        if self.stopped || !self.transport_enabled || self.security_preflight {
12755            return;
12756        }
12757        match parse_control(text) {
12758            Ok(ControlMessage::Hello { requires_sync, .. }) => {
12759                if requires_sync {
12760                    // §8.1: pull before trusting the socket for continuity.
12761                    self.set_sync_needed(true, true);
12762                }
12763            }
12764            Ok(ControlMessage::Presence {
12765                scope_key,
12766                kind,
12767                actor_id,
12768                client_id,
12769                doc,
12770                error,
12771                ..
12772            }) => {
12773                self.apply_presence(scope_key, kind, actor_id, client_id, doc, error);
12774            }
12775            Ok(ControlMessage::Wake { .. }) => {
12776                // §8.3: any wake-up means "run a pull soon", never data.
12777                self.set_sync_needed(true, true);
12778            }
12779            _ => {}
12780        }
12781    }
12782
12783    /// Inbound binary delta: a complete SSP2 response (§8.2), applied like
12784    /// a pull response per section; an unapplied delta is a wake-up.
12785    /// Mailbox hosts send durable acknowledgements on their I/O executor.
12786    pub fn on_realtime_binary_queued(&mut self, bytes: &[u8]) -> Vec<String> {
12787        let mut transport = crate::round::DownloadResults::default();
12788        self.on_realtime_binary(&mut transport, bytes);
12789        transport.controls
12790    }
12791
12792    /// §8.2: a delta applies exactly like a live section and sets the replica
12793    /// cursor to its `throughSeq`; one at or below the replica cursor is a
12794    /// duplicate. A delta the client does not apply (no live unit, a status
12795    /// other than `active`, a decode or apply failure) is a drop and raises
12796    /// sync-needed.
12797    pub fn on_realtime_binary(&mut self, transport: &mut dyn Transport, bytes: &[u8]) {
12798        if self.stopped || !self.transport_enabled || self.security_preflight {
12799            return;
12800        }
12801        #[cfg(feature = "bench-internals")]
12802        let decode_phase = self.benchmark_phases.start(Phase::ResponseDecode);
12803        let message = match decode_message(bytes) {
12804            Ok(m) if m.msg_kind == MsgKind::Response => m,
12805            _ => {
12806                self.set_sync_needed(true, true);
12807                return;
12808            }
12809        };
12810        #[cfg(feature = "bench-internals")]
12811        drop(decode_phase);
12812        // §8.2: a delta's header means what a pull response's header means. A
12813        // schema floor (§1.6) is recorded, stops syncing and applies nothing.
12814        if let Some(Frame::RespHeader {
12815            required_schema_version: Some(required),
12816            latest_schema_version,
12817            ..
12818        }) = message.frames.first()
12819        {
12820            let floor = SchemaFloor {
12821                required_schema_version: Some(*required),
12822                latest_schema_version: *latest_schema_version,
12823            };
12824            if self.set_schema_floor(Some(floor)).is_err() {
12825                self.set_sync_needed(true, true);
12826            }
12827            return;
12828        }
12829        let through = message.frames.iter().find_map(|frame| match frame {
12830            Frame::LiveEnd { through_seq, .. } => Some(*through_seq),
12831            _ => None,
12832        });
12833        let epoch = self.get_meta(LOG_EPOCH_KEY).ok().flatten();
12834        let mut dropped = !matches!(message.frames.first(), Some(Frame::RespHeader {
12835                reset_required: false,
12836                log_epoch,
12837                ..
12838            }) if Some(log_epoch) == epoch.as_ref());
12839        let mut applied: Option<i64> = None;
12840        dropped |= self.begin_coalesced_apply().is_err();
12841        let mut receivers: Option<Vec<UnitRecord>> = None;
12842        for frame in message.frames {
12843            if dropped {
12844                break;
12845            }
12846            match frame {
12847                Frame::LiveStart { status, .. } => {
12848                    let Ok(cursor) = self.stored(replica_cursor(&self.conn)) else {
12849                        dropped = true;
12850                        break;
12851                    };
12852                    if status != LiveStatus::Active || cursor < 0 {
12853                        dropped = true;
12854                    } else if !through.is_some_and(|through| through <= cursor) {
12855                        // A catching-up unit receives no delta (§8.2).
12856                        receivers = self.stored(load_units(&self.conn)).ok().map(|units| {
12857                            units
12858                                .into_iter()
12859                                .filter(|unit| {
12860                                    unit.status == UnitStatus::Active
12861                                        && unit.echo.is_some()
12862                                        && unit.cursor.is_none()
12863                                })
12864                                .collect()
12865                        });
12866                        dropped = receivers.is_none();
12867                    }
12868                }
12869                Frame::Commit {
12870                    commit_seq,
12871                    tables,
12872                    changes,
12873                    ..
12874                } => {
12875                    let Some(receivers) = receivers.clone() else {
12876                        continue;
12877                    };
12878                    dropped = self
12879                        .apply_live_commit(
12880                            &receivers,
12881                            &tables,
12882                            changes,
12883                            commit_seq,
12884                            AppliedServerCommitSource::Delta,
12885                        )
12886                        .is_err();
12887                }
12888                Frame::LiveEnd { through_seq, .. } if receivers.is_some() => {
12889                    dropped = self
12890                        .set_meta(REPLICA_CURSOR_KEY, &through_seq.to_string())
12891                        .is_err();
12892                    applied = Some(through_seq);
12893                }
12894                Frame::Error { .. } => dropped = true,
12895                _ => {}
12896            }
12897        }
12898        // §1.4 rule 4: the ack follows the COMMIT. A failed COMMIT restored
12899        // the replica cursor, so the delta counts as a drop.
12900        if self.finish_coalesced_apply().is_err() || dropped {
12901            applied = None;
12902            dropped = true;
12903        }
12904        if let Some(cursor) = applied {
12905            // §8.2 ack point: the delta's throughSeq.
12906            let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
12907            let _ = transport.realtime_send(&ack);
12908        }
12909        if dropped {
12910            self.set_sync_needed(true, true);
12911        }
12912    }
12913
12914    /// §8.2 ack point after a pull round on a live connection: the replica
12915    /// cursor while it is at least `0`.
12916    fn ack_after_pull(&mut self, transport: &mut dyn Transport) {
12917        if !self.realtime_connected() {
12918            return;
12919        }
12920        if let Ok(cursor @ 0..) = self.stored(replica_cursor(&self.conn)) {
12921            let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
12922            let _ = transport.realtime_send(&ack);
12923        }
12924    }
12925}
12926
12927#[cfg(test)]
12928mod previous_version_wiring_tests;
12929
12930#[cfg(test)]
12931mod identity_tests;
12932
12933#[cfg(test)]
12934mod insert_intent_tests;
12935
12936#[cfg(test)]
12937mod robustness_tests;
12938
12939#[cfg(test)]
12940mod range_image_tests;